blob: 8b44b9e026f3216699b4d32f70871ca325831cc5 [file]
package main
import (
"android/soong/kernel/common"
"archive/zip"
"encoding/json"
"errors"
"flag"
"fmt"
"io"
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
)
var (
soongZip = flag.String("soong_zip", "", "path to soong_zip executable")
zipSync = flag.String("zipsync", "", "path to zipsync executable")
mergeZips = flag.String("merge_zips", "", "path to merge_zips executable")
depmod = flag.String("depmod", "", "path to depmod executable")
llvmStrip = flag.String("llvm-strip", "", "path to llvm-strip executable")
outDir string
installPartition string
)
func main() {
flag.Usage = func() {
fmt.Fprintln(os.Stderr, "usage: kernel_modules_builder <props.json> <install partition> <temp dir> <out load file> <out zip>")
flag.PrintDefaults()
}
flag.Parse()
if len(flag.Args()) != 5 {
flag.Usage()
os.Exit(1)
}
propsFile := flag.Arg(0)
installPartition = flag.Arg(1)
outDir = flag.Arg(2)
outLoadFile := flag.Arg(3)
outInstallZip := flag.Arg(4)
must(os.RemoveAll(outDir))
must(os.MkdirAll(outDir, 0o777))
var props common.PrebuiltKernelModulesPropertiesJSON
must(json.Unmarshal(must2(os.ReadFile(propsFile)), &props))
var loadFile = filepath.Join(outDir, "modules.load")
modules := createLoadFile(&props, loadFile)
modulesZip := filepath.Join(outDir, "modules.zip")
args := []string{"-o", modulesZip, "-j"}
for _, mod := range modules {
args = append(args, "-f", mod)
}
srcs16k := slices.Clone(props.Srcs_16k)
if String(props.Zip.Srcs_16k_cfg_file) != "" {
func() {
dir := filepath.Join(outDir, "unzipped_modules_16k")
must(os.MkdirAll(dir, 0o777))
f := must2(os.Open(*props.Zip.Src))
defer f.Close()
reader := must2(zip.NewReader(f, must2(f.Stat()).Size()))
contents := string(readZipFile(reader, *props.Zip.Srcs_16k_cfg_file))
for _, line := range strings.Split(contents, "\n") {
if line, ok := strings.CutPrefix(line, "modprobe|"); ok {
out := filepath.Join(dir, line)
extractZipFile(reader, filepath.Join("16kb", line), out)
srcs16k = append(srcs16k, out)
}
}
}()
}
if len(srcs16k) > 0 {
args = append(args, "-P", "16k-mode")
for _, mod := range srcs16k {
args = append(args, "-f", mod)
}
}
cmd(*soongZip, args...)
zipsToMerge := []string{modulesZip}
runDepmod(modules, loadFile, String(props.System_dep), &zipsToMerge)
installBlocklistFile(&props, &zipsToMerge)
installOptionsFile(&props, &zipsToMerge)
allInstallsZip := filepath.Join(outDir, "installs.zip")
cmd(*mergeZips, slices.Concat([]string{allInstallsZip}, zipsToMerge)...)
if BoolDefault(props.Strip_debug_symbols, true) {
allInstallsZip = stripDebugSymbols(allInstallsZip)
}
// Move to output locations
must(os.Rename(loadFile, outLoadFile))
must(os.Rename(allInstallsZip, outInstallZip))
}
func runDepmod(modules []string, loadFile string, systemModulesZip string, zipsToMerge *[]string) {
baseDir := filepath.Join(outDir, "depmod_tmp")
fakeVer := "0.0" // depmod requires this
modulesDir := filepath.Join(baseDir, "lib", "modules", fakeVer)
modulesCpDir := modulesDirForAndroidDlkm(modulesDir, false)
must(os.MkdirAll(modulesCpDir, 0o777))
for _, mod := range modules {
cp(mod, filepath.Join(modulesCpDir, filepath.Base(mod)))
}
if systemModulesZip != "" {
modulesDirForSystemDlkm := modulesDirForAndroidDlkm(modulesDir, true)
must(os.MkdirAll(modulesDirForSystemDlkm, 0o777))
cmd(*zipSync, "-d", modulesDirForSystemDlkm, systemModulesZip)
// https://source.corp.google.com/h/googleplex-android/platform/build/+/71d79d0a58e112f76ee2c2dfdebd331971145b4c:core/Makefile;l=480-487;bpv=1;bpt=0;drc=567ee7be9833ea96a65e36fb21d4bd783ff74f1c
// When there is a duplicate module present in both directories, we want modules in PRIVATE_MODULES to take
// precedence. Since depmod does not provide any guarantee about ordering of
// dependency resolution, we achieve this by maually removing any duplicate
// modules with lower priority.
for _, mod := range modules {
name := filepath.Base(mod)
err := os.Remove(filepath.Join(modulesDirForSystemDlkm, name))
if !errors.Is(err, os.ErrNotExist) {
must(err)
}
}
}
modulesLoad := filepath.Join(modulesDir, "modules.load")
cp(loadFile, modulesLoad)
cmd(*depmod, "-b", baseDir, fakeVer)
depFile := filepath.Join(modulesDir, "modules.dep")
// Add a leading slash to paths in modules.dep of android dlkm and vendor ramdisk
switch installPartition {
case "system_dlkm", "vendor_dlkm", "odm_dlkm", "vendor_ramdisk", "vendor_kernel_ramdisk":
r := regexp.MustCompile(`[^:\s]*lib/modules/[^:\s]*`)
contents := must2(os.ReadFile(depFile))
contents = r.ReplaceAll(contents, []byte("/$0"))
must(os.WriteFile(depFile, contents, 0o777))
}
installZip := filepath.Join(baseDir, "depmod_installs.zip")
cmd(*soongZip, "-o", installZip, "-C", modulesDir,
"-f", depFile,
"-f", filepath.Join(modulesDir, "modules.softdep"),
"-f", filepath.Join(modulesDir, "modules.alias"))
*zipsToMerge = append(*zipsToMerge, installZip)
}
// This is the path in soong intermediates where the .ko files will be copied.
// The layout should match the layout on device so that depmod can create meaningful modules.* files.
func modulesDirForAndroidDlkm(modulesDir string, system bool) string {
if installPartition == "system_dlkm" || system {
// The first component can be either system or system_dlkm
// system works because /system/lib/modules is a symlink to /system_dlkm/lib/modules.
// system was chosen to match the contents of the kati built modules.dep
return filepath.Join(modulesDir, "system", "lib", "modules")
} else if installPartition == "vendor_dlkm" {
return filepath.Join(modulesDir, "vendor", "lib", "modules")
} else if installPartition == "odm_dlkm" {
return filepath.Join(modulesDir, "odm", "lib", "modules")
} else if installPartition == "vendor_ramdisk" || installPartition == "vendor_kernel_ramdisk" {
return filepath.Join(modulesDir, "lib", "modules")
} else {
// not an android dlkm module.
return modulesDir
}
}
// Creates the load file at the given location, then returns the list of modules that should be
// installed. (which can be more than just the modules listed in the load file)
func createLoadFile(props *common.PrebuiltKernelModulesPropertiesJSON, loadFile string) []string {
if props.Zip.Src != nil {
if props.Load_by_default != nil {
fmt.Fprintf(os.Stderr, "Load_by_default is incompatible with zip-based prebuilt_kernel_modules\n")
os.Exit(1)
}
if props.Src_filenames_to_load != nil {
fmt.Fprintf(os.Stderr, "Src_filenames_to_load is incompatible with zip-based prebuilt_kernel_modules\n")
os.Exit(1)
}
f := must2(os.Open(*props.Zip.Src))
defer f.Close()
reader := must2(zip.NewReader(f, must2(f.Stat()).Size()))
lf := readZipFile(reader, *props.Zip.Load_file)
dir := filepath.Join(outDir, "unzipped_modules")
must(os.MkdirAll(dir, 0o777))
// Extract the loaded kofiles to a directory
lines := slices.Concat(props.Zip.Extra_loads, strings.Split(string(lf), "\n"))
koFiles := make([]string, 0, len(lines)+len(props.Srcs))
koNames := make([]string, 0, len(lines))
seen := make(map[string]struct{})
for _, line := range lines {
if len(line) == 0 {
continue
}
koFile := filepath.Base(line)
if _, ok := seen[koFile]; ok {
fmt.Fprintf(os.Stderr, "Duplicate ko file: %q\n", koFile)
os.Exit(1)
}
seen[koFile] = struct{}{}
extractedLocation := filepath.Join(dir, koFile)
extractZipFile(reader, koFile, extractedLocation)
koFiles = append(koFiles, extractedLocation)
koNames = append(koNames, koFile)
}
must(os.WriteFile(loadFile, []byte(strings.Join(koNames, "\n")), 0o777))
// Also allow for extra kofiles to be passed outside the zip, via the srcs property.
// This is to support files that have traditionally been outside of TARGET_KERNEL_DIR:
// https://source.corp.google.com/h/googleplex-android/platform/superproject/main/+/main:device/google/common/etm/BoardUserdebugModules.mk;l=23;drc=ee64c48352187c839310d63d69142fa141f39809
for _, src := range props.Srcs {
koFile := filepath.Base(src)
if _, ok := seen[koFile]; ok {
fmt.Fprintf(os.Stderr, "Duplicate ko file: %q\n", koFile)
os.Exit(1)
}
seen[koFile] = struct{}{}
koFiles = append(koFiles, src)
}
return koFiles
} else {
if !BoolDefault(props.Load_by_default, true) {
must(os.WriteFile(loadFile, nil, 0o777))
} else if len(props.Src_filenames_to_load) > 0 {
validNames := make(map[string]bool, len(props.Srcs))
for _, src := range props.Srcs {
validNames[filepath.Base(src)] = true
}
var contents strings.Builder
for _, filenameToLoad := range props.Src_filenames_to_load {
if _, ok := validNames[filenameToLoad]; !ok {
fmt.Printf("%s in src_filenames_to_load not present in srcs\n", filenameToLoad)
os.Exit(1)
}
contents.WriteString(filenameToLoad)
contents.WriteString("\n")
}
must(os.WriteFile(loadFile, []byte(contents.String()), 0o777))
} else {
var contents strings.Builder
for _, filenameToLoad := range props.Srcs {
contents.WriteString(filepath.Base(filenameToLoad))
contents.WriteString("\n")
}
must(os.WriteFile(loadFile, []byte(contents.String()), 0o777))
}
return props.Srcs
}
}
func stripDebugSymbols(modulesZip string) string {
output := filepath.Join(outDir, "stripped.zip")
tmpDir := filepath.Join(outDir, "stripped")
cmd(*zipSync, "-d", tmpDir, modulesZip)
for _, dirEnt := range must2(os.ReadDir(tmpDir)) {
if strings.HasSuffix(dirEnt.Name(), ".ko") {
f := filepath.Join(tmpDir, dirEnt.Name())
cmd(*llvmStrip, "-o", f, "--strip-debug", f)
}
}
cmd(*soongZip, "-o", output, "-C", tmpDir, "-D", tmpDir)
return output
}
// Install the blocklist file, removing empty lines and raising an error if a line is
// not formatted as `blocklist $name.ko`
func installBlocklistFile(props *common.PrebuiltKernelModulesPropertiesJSON, zipsToMerge *[]string) {
var originalContents string
if String(props.Zip.Blocklist_file) != "" {
f := must2(os.Open(*props.Zip.Src))
defer f.Close()
reader := must2(zip.NewReader(f, must2(f.Stat()).Size()))
originalContents = string(readZipFile(reader, *props.Zip.Blocklist_file))
} else if String(props.Blocklist_file) != "" {
originalContents = string(must2(os.ReadFile(*props.Blocklist_file)))
} else {
return
}
lines := strings.Split(originalContents, "\n")
var contents strings.Builder
failed := false
for i, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if !strings.HasPrefix(line, "#") {
fields := strings.Fields(line)
if len(fields) != 2 || fields[0] != "blocklist" {
fmt.Printf("Invalid blocklist line %d: %q\n", i+1, line)
failed = true
}
line = strings.Join(fields, " ")
}
contents.WriteString(line)
contents.WriteString("\n")
}
if failed {
os.Exit(1)
}
out := filepath.Join(outDir, "modules.blocklist")
must(os.WriteFile(out, []byte(contents.String()), 0o777))
installZip := filepath.Join(outDir, "blocklist_install.zip")
cmd(*soongZip, "-o", installZip, "-e", "modules.blocklist", "-f", out)
*zipsToMerge = append(*zipsToMerge, installZip)
}
// Install the optinos file, removing empty lines and raising an error if a line is
// not formatted as `options $name.ko`
func installOptionsFile(props *common.PrebuiltKernelModulesPropertiesJSON, zipsToMerge *[]string) {
if String(props.Options_file) == "" {
return
}
lines := strings.Split(string(must2(os.ReadFile(*props.Options_file))), "\n")
var contents strings.Builder
failed := false
for i, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
continue
}
if !strings.HasPrefix(line, "#") {
fields := strings.Fields(line)
if len(fields) < 2 || fields[0] != "options" {
fmt.Printf("Invalid options line %d: %q\n", i+1, line)
failed = true
}
line = strings.Join(fields, " ")
}
contents.WriteString(line)
contents.WriteString("\n")
}
if failed {
os.Exit(1)
}
out := filepath.Join(outDir, "modules.options")
must(os.WriteFile(out, []byte(contents.String()), 0o777))
installZip := filepath.Join(outDir, "options_install.zip")
cmd(*soongZip, "-o", installZip, "-e", "modules.options", "-f", out)
*zipsToMerge = append(*zipsToMerge, installZip)
}
func readZipFile(zipFile *zip.Reader, pathInZip string) []byte {
in := must2(zipFile.Open(pathInZip))
defer in.Close()
return must2(io.ReadAll(in))
}
func extractZipFile(zipFile *zip.Reader, pathInZip string, outputPath string) {
in := must2(zipFile.Open(pathInZip))
defer in.Close()
out := must2(os.Create(outputPath))
defer out.Close()
must2(io.Copy(out, in))
}
func cp(from string, to string) {
cmd("cp", from, to)
}
func cmd(tool string, args ...string) {
cmd := exec.Command(tool, args...)
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
must(cmd.Run())
}
func must(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
}
func must2[T any](x T, err error) T {
if err != nil {
fmt.Fprintf(os.Stderr, "%s\n", err)
os.Exit(1)
}
return x
}
func String(s *string) string {
if s == nil {
return ""
}
return *s
}
func BoolDefault(b *bool, def bool) bool {
if b == nil {
return def
}
return *b
}