| // Copyright 2015 Google Inc. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package main |
| |
| import ( |
| "flag" |
| "fmt" |
| "io/ioutil" |
| "os" |
| "path/filepath" |
| "strings" |
| "time" |
| |
| "android/soong/bp2build" |
| "android/soong/shared" |
| |
| "github.com/google/blueprint/bootstrap" |
| "github.com/google/blueprint/deptools" |
| |
| "android/soong/android" |
| ) |
| |
| var ( |
| topDir string |
| outDir string |
| availableEnvFile string |
| usedEnvFile string |
| |
| delveListen string |
| delvePath string |
| |
| docFile string |
| bazelQueryViewDir string |
| bp2buildMarker string |
| ) |
| |
| func init() { |
| // Flags that make sense in every mode |
| flag.StringVar(&topDir, "top", "", "Top directory of the Android source tree") |
| flag.StringVar(&outDir, "out", "", "Soong output directory (usually $TOP/out/soong)") |
| flag.StringVar(&availableEnvFile, "available_env", "", "File containing available environment variables") |
| flag.StringVar(&usedEnvFile, "used_env", "", "File containing used environment variables") |
| |
| // Debug flags |
| flag.StringVar(&delveListen, "delve_listen", "", "Delve port to listen on for debugging") |
| flag.StringVar(&delvePath, "delve_path", "", "Path to Delve. Only used if --delve_listen is set") |
| |
| // Flags representing various modes soong_build can run in |
| flag.StringVar(&docFile, "soong_docs", "", "build documentation file to output") |
| flag.StringVar(&bazelQueryViewDir, "bazel_queryview_dir", "", "path to the bazel queryview directory relative to --top") |
| flag.StringVar(&bp2buildMarker, "bp2build_marker", "", "If set, run bp2build, touch the specified marker file then exit") |
| } |
| |
| func newNameResolver(config android.Config) *android.NameResolver { |
| namespacePathsToExport := make(map[string]bool) |
| |
| for _, namespaceName := range config.ExportedNamespaces() { |
| namespacePathsToExport[namespaceName] = true |
| } |
| |
| namespacePathsToExport["."] = true // always export the root namespace |
| |
| exportFilter := func(namespace *android.Namespace) bool { |
| return namespacePathsToExport[namespace.Path] |
| } |
| |
| return android.NewNameResolver(exportFilter) |
| } |
| |
| func newContext(configuration android.Config, prepareBuildActions bool) *android.Context { |
| ctx := android.NewContext(configuration) |
| ctx.Register() |
| if !prepareBuildActions { |
| configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) |
| } |
| ctx.SetNameInterface(newNameResolver(configuration)) |
| ctx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) |
| return ctx |
| } |
| |
| func newConfig(srcDir, outDir string, availableEnv map[string]string) android.Config { |
| configuration, err := android.NewConfig(srcDir, outDir, bootstrap.CmdlineArgs.ModuleListFile, availableEnv) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| return configuration |
| } |
| |
| // Bazel-enabled mode. Soong runs in two passes. |
| // First pass: Analyze the build tree, but only store all bazel commands |
| // needed to correctly evaluate the tree in the second pass. |
| // TODO(cparsons): Don't output any ninja file, as the second pass will overwrite |
| // the incorrect results from the first pass, and file I/O is expensive. |
| func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) { |
| var firstArgs, secondArgs bootstrap.Args |
| |
| firstArgs = bootstrap.CmdlineArgs |
| configuration.SetStopBefore(bootstrap.StopBeforeWriteNinja) |
| bootstrap.RunBlueprint(firstArgs, firstCtx.Context, configuration) |
| |
| // Invoke bazel commands and save results for second pass. |
| if err := configuration.BazelContext.InvokeBazel(); err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| // Second pass: Full analysis, using the bazel command results. Output ninja file. |
| secondConfig, err := android.ConfigForAdditionalRun(configuration) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| secondCtx := newContext(secondConfig, true) |
| secondArgs = bootstrap.CmdlineArgs |
| ninjaDeps := bootstrap.RunBlueprint(secondArgs, secondCtx.Context, secondConfig) |
| ninjaDeps = append(ninjaDeps, extraNinjaDeps...) |
| err = deptools.WriteDepFile(shared.JoinPath(topDir, secondArgs.DepFile), secondArgs.OutFile, ninjaDeps) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", secondArgs.DepFile, err) |
| os.Exit(1) |
| } |
| } |
| |
| // Run the code-generation phase to convert BazelTargetModules to BUILD files. |
| func runQueryView(configuration android.Config, ctx *android.Context) { |
| codegenContext := bp2build.NewCodegenContext(configuration, *ctx, bp2build.QueryView) |
| absoluteQueryViewDir := shared.JoinPath(topDir, bazelQueryViewDir) |
| if err := createBazelQueryView(codegenContext, absoluteQueryViewDir); err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| } |
| |
| func runSoongDocs(configuration android.Config) { |
| ctx := newContext(configuration, false) |
| soongDocsArgs := bootstrap.CmdlineArgs |
| bootstrap.RunBlueprint(soongDocsArgs, ctx.Context, configuration) |
| if err := writeDocs(ctx, configuration, docFile); err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| } |
| |
| func writeMetrics(configuration android.Config) { |
| metricsFile := filepath.Join(configuration.BuildDir(), "soong_build_metrics.pb") |
| err := android.WriteMetrics(configuration, metricsFile) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "error writing soong_build metrics %s: %s", metricsFile, err) |
| os.Exit(1) |
| } |
| } |
| |
| func writeJsonModuleGraph(configuration android.Config, ctx *android.Context, path string, extraNinjaDeps []string) { |
| f, err := os.Create(path) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "%s", err) |
| os.Exit(1) |
| } |
| |
| defer f.Close() |
| ctx.Context.PrintJSONGraph(f) |
| writeFakeNinjaFile(extraNinjaDeps, configuration.BuildDir()) |
| } |
| |
| func doChosenActivity(configuration android.Config, extraNinjaDeps []string) string { |
| bazelConversionRequested := bp2buildMarker != "" |
| mixedModeBuild := configuration.BazelContext.BazelEnabled() |
| generateQueryView := bazelQueryViewDir != "" |
| jsonModuleFile := configuration.Getenv("SOONG_DUMP_JSON_MODULE_GRAPH") |
| |
| blueprintArgs := bootstrap.CmdlineArgs |
| prepareBuildActions := !generateQueryView && jsonModuleFile == "" |
| if bazelConversionRequested { |
| // Run the alternate pipeline of bp2build mutators and singleton to convert |
| // Blueprint to BUILD files before everything else. |
| runBp2Build(configuration, extraNinjaDeps) |
| if bp2buildMarker != "" { |
| return bp2buildMarker |
| } else { |
| return bootstrap.CmdlineArgs.OutFile |
| } |
| } |
| |
| ctx := newContext(configuration, prepareBuildActions) |
| if mixedModeBuild { |
| runMixedModeBuild(configuration, ctx, extraNinjaDeps) |
| } else { |
| ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, ctx.Context, configuration) |
| ninjaDeps = append(ninjaDeps, extraNinjaDeps...) |
| err := deptools.WriteDepFile(shared.JoinPath(topDir, blueprintArgs.DepFile), blueprintArgs.OutFile, ninjaDeps) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "Error writing depfile '%s': %s\n", blueprintArgs.DepFile, err) |
| os.Exit(1) |
| } |
| } |
| |
| // Convert the Soong module graph into Bazel BUILD files. |
| if generateQueryView { |
| runQueryView(configuration, ctx) |
| return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie |
| } |
| |
| if jsonModuleFile != "" { |
| writeJsonModuleGraph(configuration, ctx, jsonModuleFile, extraNinjaDeps) |
| return bootstrap.CmdlineArgs.OutFile // TODO: This is a lie |
| } |
| |
| writeMetrics(configuration) |
| return bootstrap.CmdlineArgs.OutFile |
| } |
| |
| // soong_ui dumps the available environment variables to |
| // soong.environment.available . Then soong_build itself is run with an empty |
| // environment so that the only way environment variables can be accessed is |
| // using Config, which tracks access to them. |
| |
| // At the end of the build, a file called soong.environment.used is written |
| // containing the current value of all used environment variables. The next |
| // time soong_ui is run, it checks whether any environment variables that was |
| // used had changed and if so, it deletes soong.environment.used to cause a |
| // rebuild. |
| // |
| // The dependency of build.ninja on soong.environment.used is declared in |
| // build.ninja.d |
| func parseAvailableEnv() map[string]string { |
| if availableEnvFile == "" { |
| fmt.Fprintf(os.Stderr, "--available_env not set\n") |
| os.Exit(1) |
| } |
| |
| result, err := shared.EnvFromFile(shared.JoinPath(topDir, availableEnvFile)) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "error reading available environment file '%s': %s\n", availableEnvFile, err) |
| os.Exit(1) |
| } |
| |
| return result |
| } |
| |
| func main() { |
| flag.Parse() |
| |
| shared.ReexecWithDelveMaybe(delveListen, delvePath) |
| android.InitSandbox(topDir) |
| |
| availableEnv := parseAvailableEnv() |
| |
| // The top-level Blueprints file is passed as the first argument. |
| srcDir := filepath.Dir(flag.Arg(0)) |
| configuration := newConfig(srcDir, outDir, availableEnv) |
| extraNinjaDeps := []string{ |
| configuration.ProductVariablesFileName, |
| usedEnvFile, |
| } |
| |
| if configuration.Getenv("ALLOW_MISSING_DEPENDENCIES") == "true" { |
| configuration.SetAllowMissingDependencies() |
| } |
| |
| if shared.IsDebugging() { |
| // Add a non-existent file to the dependencies so that soong_build will rerun when the debugger is |
| // enabled even if it completed successfully. |
| extraNinjaDeps = append(extraNinjaDeps, filepath.Join(configuration.BuildDir(), "always_rerun_for_delve")) |
| } |
| |
| if docFile != "" { |
| // We don't write an used variables file when generating documentation |
| // because that is done from within the actual builds as a Ninja action and |
| // thus it would overwrite the actual used variables file so this is |
| // special-cased. |
| // TODO: Fix this by not passing --used_env to the soong_docs invocation |
| runSoongDocs(configuration) |
| return |
| } |
| |
| finalOutputFile := doChosenActivity(configuration, extraNinjaDeps) |
| writeUsedEnvironmentFile(configuration, finalOutputFile) |
| } |
| |
| func writeUsedEnvironmentFile(configuration android.Config, finalOutputFile string) { |
| if usedEnvFile == "" { |
| return |
| } |
| |
| path := shared.JoinPath(topDir, usedEnvFile) |
| data, err := shared.EnvFileContents(configuration.EnvDeps()) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) |
| os.Exit(1) |
| } |
| |
| err = ioutil.WriteFile(path, data, 0666) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "error writing used environment file '%s': %s\n", usedEnvFile, err) |
| os.Exit(1) |
| } |
| |
| // Touch the output file so that it's not older than the file we just |
| // wrote. We can't write the environment file earlier because one an access |
| // new environment variables while writing it. |
| touch(shared.JoinPath(topDir, finalOutputFile)) |
| } |
| |
| // Workarounds to support running bp2build in a clean AOSP checkout with no |
| // prior builds, and exiting early as soon as the BUILD files get generated, |
| // therefore not creating build.ninja files that soong_ui and callers of |
| // soong_build expects. |
| // |
| // These files are: build.ninja and build.ninja.d. Since Kati hasn't been |
| // ran as well, and `nothing` is defined in a .mk file, there isn't a ninja |
| // target called `nothing`, so we manually create it here. |
| func writeFakeNinjaFile(extraNinjaDeps []string, buildDir string) { |
| extraNinjaDepsString := strings.Join(extraNinjaDeps, " \\\n ") |
| |
| ninjaFileName := "build.ninja" |
| ninjaFile := shared.JoinPath(topDir, buildDir, ninjaFileName) |
| ninjaFileD := shared.JoinPath(topDir, buildDir, ninjaFileName) |
| // A workaround to create the 'nothing' ninja target so `m nothing` works, |
| // since bp2build runs without Kati, and the 'nothing' target is declared in |
| // a Makefile. |
| ioutil.WriteFile(ninjaFile, []byte("build nothing: phony\n phony_output = true\n"), 0666) |
| ioutil.WriteFile(ninjaFileD, |
| []byte(fmt.Sprintf("%s: \\\n %s\n", ninjaFileName, extraNinjaDepsString)), |
| 0666) |
| } |
| |
| func touch(path string) { |
| f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0666) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) |
| os.Exit(1) |
| } |
| |
| err = f.Close() |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "Error touching '%s': %s\n", path, err) |
| os.Exit(1) |
| } |
| |
| currentTime := time.Now().Local() |
| err = os.Chtimes(path, currentTime, currentTime) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "error touching '%s': %s\n", path, err) |
| os.Exit(1) |
| } |
| } |
| |
| // Run Soong in the bp2build mode. This creates a standalone context that registers |
| // an alternate pipeline of mutators and singletons specifically for generating |
| // Bazel BUILD files instead of Ninja files. |
| func runBp2Build(configuration android.Config, extraNinjaDeps []string) { |
| // Register an alternate set of singletons and mutators for bazel |
| // conversion for Bazel conversion. |
| bp2buildCtx := android.NewContext(configuration) |
| |
| // Propagate "allow misssing dependencies" bit. This is normally set in |
| // newContext(), but we create bp2buildCtx without calling that method. |
| bp2buildCtx.SetAllowMissingDependencies(configuration.AllowMissingDependencies()) |
| bp2buildCtx.SetNameInterface(newNameResolver(configuration)) |
| bp2buildCtx.RegisterForBazelConversion() |
| |
| // The bp2build process is a purely functional process that only depends on |
| // Android.bp files. It must not depend on the values of per-build product |
| // configurations or variables, since those will generate different BUILD |
| // files based on how the user has configured their tree. |
| bp2buildCtx.SetModuleListFile(bootstrap.CmdlineArgs.ModuleListFile) |
| modulePaths, err := bp2buildCtx.ListModulePaths(configuration.SrcDir()) |
| if err != nil { |
| panic(err) |
| } |
| |
| extraNinjaDeps = append(extraNinjaDeps, modulePaths...) |
| |
| // No need to generate Ninja build rules/statements from Modules and Singletons. |
| configuration.SetStopBefore(bootstrap.StopBeforePrepareBuildActions) |
| |
| // Run the loading and analysis pipeline to prepare the graph of regular |
| // Modules parsed from Android.bp files, and the BazelTargetModules mapped |
| // from the regular Modules. |
| blueprintArgs := bootstrap.CmdlineArgs |
| ninjaDeps := bootstrap.RunBlueprint(blueprintArgs, bp2buildCtx.Context, configuration) |
| ninjaDeps = append(ninjaDeps, extraNinjaDeps...) |
| |
| ninjaDeps = append(ninjaDeps, bootstrap.GlobFileListFiles(configuration)...) |
| |
| // Run the code-generation phase to convert BazelTargetModules to BUILD files |
| // and print conversion metrics to the user. |
| codegenContext := bp2build.NewCodegenContext(configuration, *bp2buildCtx, bp2build.Bp2Build) |
| metrics := bp2build.Codegen(codegenContext) |
| |
| generatedRoot := shared.JoinPath(configuration.BuildDir(), "bp2build") |
| workspaceRoot := shared.JoinPath(configuration.BuildDir(), "workspace") |
| |
| excludes := []string{ |
| "bazel-bin", |
| "bazel-genfiles", |
| "bazel-out", |
| "bazel-testlogs", |
| "bazel-" + filepath.Base(topDir), |
| } |
| |
| if bootstrap.CmdlineArgs.NinjaBuildDir[0] != '/' { |
| excludes = append(excludes, bootstrap.CmdlineArgs.NinjaBuildDir) |
| } |
| |
| symlinkForestDeps := bp2build.PlantSymlinkForest( |
| topDir, workspaceRoot, generatedRoot, configuration.SrcDir(), excludes) |
| |
| // Only report metrics when in bp2build mode. The metrics aren't relevant |
| // for queryview, since that's a total repo-wide conversion and there's a |
| // 1:1 mapping for each module. |
| metrics.Print() |
| |
| ninjaDeps = append(ninjaDeps, codegenContext.AdditionalNinjaDeps()...) |
| ninjaDeps = append(ninjaDeps, symlinkForestDeps...) |
| |
| depFile := bp2buildMarker + ".d" |
| err = deptools.WriteDepFile(shared.JoinPath(topDir, depFile), bp2buildMarker, ninjaDeps) |
| if err != nil { |
| fmt.Fprintf(os.Stderr, "Cannot write depfile '%s': %s\n", depFile, err) |
| os.Exit(1) |
| } |
| |
| if bp2buildMarker != "" { |
| touch(shared.JoinPath(topDir, bp2buildMarker)) |
| } else { |
| writeFakeNinjaFile(extraNinjaDeps, codegenContext.Config().BuildDir()) |
| } |
| } |