Refactor mixed builds to only take one pass
This large refactoring has both immense performance implications and
improves mixed builds complexity / usability. Summary:
1. Queueing calls to Bazel is done in a new mutator instead of a full
soong_build pass. Normal soong_build flow is interrupted (via a
functional hook in blueprint) to invoke bazel and parse its response.
2. Implementing mixed build support for additional modules is as simple
as implementing MixedBuildsBuildable. In this interface, define the
request that must be queued to Bazel, and then subsequently define
how to handle the returned bazel cquery metadata.
3. Mixed builds consists of only a single pass. This greatly
improves mixed build performance.
Result:
A 33% runtime improvement on soong analysis phase with mixed builds.
Caveats:
C++ BazelHandler handling still remains a bit of a mess; I did what
I could within this CL's scope, but this may require additional cleanup.
Test: Treehugger
Test: Verified that aosp_arm ninja file is bit-for-bit identical with or
without this change.
Change-Id: I412d9c94d429105f4ebfafc84100d546069e6621
diff --git a/android/bazel.go b/android/bazel.go
index 67002ec..378eb6f 100644
--- a/android/bazel.go
+++ b/android/bazel.go
@@ -115,6 +115,27 @@
SetBaseModuleType(baseModuleType string)
}
+// MixedBuildBuildable is an interface that module types should implement in order
+// to be "handled by Bazel" in a mixed build.
+type MixedBuildBuildable interface {
+ // IsMixedBuildSupported returns true if and only if this module should be
+ // "handled by Bazel" in a mixed build.
+ // This "escape hatch" allows modules with corner-case scenarios to opt out
+ // of being built with Bazel.
+ IsMixedBuildSupported(ctx BaseModuleContext) bool
+
+ // QueueBazelCall invokes request-queueing functions on the BazelContext
+ // so that these requests are handled when Bazel's cquery is invoked.
+ QueueBazelCall(ctx BaseModuleContext)
+
+ // ProcessBazelQueryResponse uses Bazel information (obtained from the BazelContext)
+ // to set module fields and providers to propagate this module's metadata upstream.
+ // This effectively "bridges the gap" between Bazel and Soong in a mixed build.
+ // Soong modules depending on this module should be oblivious to the fact that
+ // this module was handled by Bazel.
+ ProcessBazelQueryResponse(ctx ModuleContext)
+}
+
// BazelModule is a lightweight wrapper interface around Module for Bazel-convertible modules.
type BazelModule interface {
Module
@@ -342,7 +363,7 @@
// converted or handcrafted Bazel target. As a side effect, calling this
// method will also log whether this module is mixed build enabled for
// metrics reporting.
-func MixedBuildsEnabled(ctx ModuleContext) bool {
+func MixedBuildsEnabled(ctx BaseModuleContext) bool {
mixedBuildEnabled := mixedBuildPossible(ctx)
ctx.Config().LogMixedBuild(ctx, mixedBuildEnabled)
return mixedBuildEnabled
@@ -350,7 +371,7 @@
// mixedBuildPossible returns true if a module is ready to be replaced by a
// converted or handcrafted Bazel target.
-func mixedBuildPossible(ctx ModuleContext) bool {
+func mixedBuildPossible(ctx BaseModuleContext) bool {
if ctx.Os() == Windows {
// Windows toolchains are not currently supported.
return false
diff --git a/android/bazel_handler.go b/android/bazel_handler.go
index c4eb0f3..4d9423a 100644
--- a/android/bazel_handler.go
+++ b/android/bazel_handler.go
@@ -33,6 +33,26 @@
"android/soong/bazel"
)
+func init() {
+ RegisterMixedBuildsMutator(InitRegistrationContext)
+}
+
+func RegisterMixedBuildsMutator(ctx RegistrationContext) {
+ ctx.PostDepsMutators(func(ctx RegisterMutatorsContext) {
+ ctx.BottomUp("mixed_builds_prep", mixedBuildsPrepareMutator).Parallel()
+ })
+}
+
+func mixedBuildsPrepareMutator(ctx BottomUpMutatorContext) {
+ if m := ctx.Module(); m.Enabled() {
+ if mixedBuildMod, ok := m.(MixedBuildBuildable); ok {
+ if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
+ mixedBuildMod.QueueBazelCall(ctx)
+ }
+ }
+ }
+}
+
type cqueryRequest interface {
// Name returns a string name for this request type. Such request type names must be unique,
// and must only consist of alphanumeric characters.
@@ -62,33 +82,32 @@
configKey configKey
}
-// bazelHandler is the interface for a helper object related to deferring to Bazel for
-// processing a module (during Bazel mixed builds). Individual module types should define
-// their own bazel handler if they support deferring to Bazel.
-type BazelHandler interface {
- // Issue query to Bazel to retrieve information about Bazel's view of the current module.
- // If Bazel returns this information, set module properties on the current module to reflect
- // the returned information.
- // Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
- GenerateBazelBuildActions(ctx ModuleContext, label string) bool
-}
-
+// BazelContext is a context object useful for interacting with Bazel during
+// the course of a build. Use of Bazel to evaluate part of the build graph
+// is referred to as a "mixed build". (Some modules are managed by Soong,
+// some are managed by Bazel). To facilitate interop between these build
+// subgraphs, Soong may make requests to Bazel and evaluate their responses
+// so that Soong modules may accurately depend on Bazel targets.
type BazelContext interface {
- // The methods below involve queuing cquery requests to be later invoked
- // by bazel. If any of these methods return (_, false), then the request
- // has been queued to be run later.
+ // Add a cquery request to the bazel request queue. All queued requests
+ // will be sent to Bazel on a subsequent invocation of InvokeBazel.
+ QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey)
+
+ // ** Cquery Results Retrieval Functions
+ // The below functions pertain to retrieving cquery results from a prior
+ // InvokeBazel function call and parsing the results.
// Returns result files built by building the given bazel target label.
- GetOutputFiles(label string, cfgKey configKey) ([]string, bool)
+ GetOutputFiles(label string, cfgKey configKey) ([]string, error)
- // TODO(cparsons): Other cquery-related methods should be added here.
// Returns the results of GetOutputFiles and GetCcObjectFiles in a single query (in that order).
- GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error)
+ GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error)
// Returns the executable binary resultant from building together the python sources
- GetPythonBinary(label string, cfgKey configKey) (string, bool)
+ // TODO(b/232976601): Remove.
+ GetPythonBinary(label string, cfgKey configKey) (string, error)
- // ** End cquery methods
+ // ** end Cquery Results Retrieval Functions
// Issues commands to Bazel to receive results for all cquery requests
// queued in the BazelContext.
@@ -153,19 +172,23 @@
LabelToPythonBinary map[string]string
}
-func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
- result, ok := m.LabelToOutputFiles[label]
- return result, ok
+func (m MockBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
+ panic("unimplemented")
}
-func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
- result, ok := m.LabelToCcInfo[label]
- return result, ok, nil
+func (m MockBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
+ result, _ := m.LabelToOutputFiles[label]
+ return result, nil
}
-func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
- result, ok := m.LabelToPythonBinary[label]
- return result, ok
+func (m MockBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
+ result, _ := m.LabelToCcInfo[label]
+ return result, nil
+}
+
+func (m MockBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
+ result, _ := m.LabelToPythonBinary[label]
+ return result, nil
}
func (m MockBazelContext) InvokeBazel() error {
@@ -188,46 +211,53 @@
var _ BazelContext = MockBazelContext{}
-func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
- rawString, ok := bazelCtx.cquery(label, cquery.GetOutputFiles, cfgKey)
- var ret []string
- if ok {
+func (bazelCtx *bazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
+ key := cqueryKey{label, requestType, cfgKey}
+ bazelCtx.requestMutex.Lock()
+ defer bazelCtx.requestMutex.Unlock()
+ bazelCtx.requests[key] = true
+}
+
+func (bazelCtx *bazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
+ key := cqueryKey{label, cquery.GetOutputFiles, cfgKey}
+ if rawString, ok := bazelCtx.results[key]; ok {
bazelOutput := strings.TrimSpace(rawString)
- ret = cquery.GetOutputFiles.ParseResult(bazelOutput)
+ return cquery.GetOutputFiles.ParseResult(bazelOutput), nil
}
- return ret, ok
+ return nil, fmt.Errorf("no bazel response found for %v", key)
}
-func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
- result, ok := bazelCtx.cquery(label, cquery.GetCcInfo, cfgKey)
- if !ok {
- return cquery.CcInfo{}, ok, nil
- }
-
- bazelOutput := strings.TrimSpace(result)
- ret, err := cquery.GetCcInfo.ParseResult(bazelOutput)
- return ret, ok, err
-}
-
-func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
- rawString, ok := bazelCtx.cquery(label, cquery.GetPythonBinary, cfgKey)
- var ret string
- if ok {
+func (bazelCtx *bazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
+ key := cqueryKey{label, cquery.GetCcInfo, cfgKey}
+ if rawString, ok := bazelCtx.results[key]; ok {
bazelOutput := strings.TrimSpace(rawString)
- ret = cquery.GetPythonBinary.ParseResult(bazelOutput)
+ return cquery.GetCcInfo.ParseResult(bazelOutput)
}
- return ret, ok
+ return cquery.CcInfo{}, fmt.Errorf("no bazel response found for %v", key)
}
-func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, bool) {
+func (bazelCtx *bazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
+ key := cqueryKey{label, cquery.GetPythonBinary, cfgKey}
+ if rawString, ok := bazelCtx.results[key]; ok {
+ bazelOutput := strings.TrimSpace(rawString)
+ return cquery.GetPythonBinary.ParseResult(bazelOutput), nil
+ }
+ return "", fmt.Errorf("no bazel response found for %v", key)
+}
+
+func (n noopBazelContext) QueueBazelRequest(label string, requestType cqueryRequest, cfgKey configKey) {
panic("unimplemented")
}
-func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, bool, error) {
+func (n noopBazelContext) GetOutputFiles(label string, cfgKey configKey) ([]string, error) {
panic("unimplemented")
}
-func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, bool) {
+func (n noopBazelContext) GetCcInfo(label string, cfgKey configKey) (cquery.CcInfo, error) {
+ panic("unimplemented")
+}
+
+func (n noopBazelContext) GetPythonBinary(label string, cfgKey configKey) (string, error) {
panic("unimplemented")
}
@@ -314,24 +344,6 @@
return true
}
-// Adds a cquery request to the Bazel request queue, to be later invoked, or
-// returns the result of the given request if the request was already made.
-// If the given request was already made (and the results are available), then
-// returns (result, true). If the request is queued but no results are available,
-// then returns ("", false).
-func (context *bazelContext) cquery(label string, requestType cqueryRequest,
- cfgKey configKey) (string, bool) {
- key := cqueryKey{label, requestType, cfgKey}
- if result, ok := context.results[key]; ok {
- return result, true
- } else {
- context.requestMutex.Lock()
- defer context.requestMutex.Unlock()
- context.requests[key] = true
- return "", false
- }
-}
-
func pwdPrefix() string {
// Darwin doesn't have /proc
if runtime.GOOS != "darwin" {
@@ -916,7 +928,7 @@
return arch + "|" + os
}
-func GetConfigKey(ctx ModuleContext) configKey {
+func GetConfigKey(ctx BaseModuleContext) configKey {
return configKey{
// use string because Arch is not a valid key in go
arch: ctx.Arch().String(),
diff --git a/android/bazel_handler_test.go b/android/bazel_handler_test.go
index e5cff90..cfdccd7 100644
--- a/android/bazel_handler_test.go
+++ b/android/bazel_handler_test.go
@@ -5,6 +5,8 @@
"path/filepath"
"reflect"
"testing"
+
+ "android/soong/bazel/cquery"
)
func TestRequestResultsAfterInvokeBazel(t *testing.T) {
@@ -13,17 +15,14 @@
bazelContext, _ := testBazelContext(t, map[bazelCommand]string{
bazelCommand{command: "cquery", expression: "deps(@soong_injection//mixed_builds:buildroot, 2)"}: `//foo:bar|arm64_armv8-a|android>>out/foo/bar.txt`,
})
- g, ok := bazelContext.GetOutputFiles(label, cfg)
- if ok {
- t.Errorf("Did not expect cquery results prior to running InvokeBazel(), but got %s", g)
- }
+ bazelContext.QueueBazelRequest(label, cquery.GetOutputFiles, cfg)
err := bazelContext.InvokeBazel()
if err != nil {
t.Fatalf("Did not expect error invoking Bazel, but got %s", err)
}
- g, ok = bazelContext.GetOutputFiles(label, cfg)
- if !ok {
- t.Errorf("Expected cquery results after running InvokeBazel(), but got none")
+ g, err := bazelContext.GetOutputFiles(label, cfg)
+ if err != nil {
+ t.Errorf("Expected cquery results after running InvokeBazel(), but got err %v", err)
} else if w := []string{"out/foo/bar.txt"}; !reflect.DeepEqual(w, g) {
t.Errorf("Expected output %s, got %s", w, g)
}
diff --git a/android/config.go b/android/config.go
index d695217..9e96c6a 100644
--- a/android/config.go
+++ b/android/config.go
@@ -2047,7 +2047,7 @@
return Bool(c.productVariables.HostMusl)
}
-func (c *config) LogMixedBuild(ctx ModuleContext, useBazel bool) {
+func (c *config) LogMixedBuild(ctx BaseModuleContext, useBazel bool) {
moduleName := ctx.Module().Name()
c.mixedBuildsLock.Lock()
defer c.mixedBuildsLock.Unlock()
diff --git a/android/filegroup.go b/android/filegroup.go
index 1bf5e07..485e0b9 100644
--- a/android/filegroup.go
+++ b/android/filegroup.go
@@ -18,6 +18,7 @@
"strings"
"android/soong/bazel"
+ "android/soong/bazel/cquery"
"github.com/google/blueprint"
)
@@ -101,6 +102,7 @@
srcs Paths
}
+var _ MixedBuildBuildable = (*fileGroup)(nil)
var _ SourceFileProducer = (*fileGroup)(nil)
// filegroup contains a list of files that are referenced by other modules
@@ -114,42 +116,11 @@
return module
}
-func (fg *fileGroup) maybeGenerateBazelBuildActions(ctx ModuleContext) {
- if !MixedBuildsEnabled(ctx) {
- return
- }
-
- archVariant := ctx.Arch().String()
- osVariant := ctx.Os()
- if len(fg.Srcs()) == 1 && fg.Srcs()[0].Base() == fg.Name() {
- // This will be a regular file target, not filegroup, in Bazel.
- // See FilegroupBp2Build for more information.
- archVariant = Common.String()
- osVariant = CommonOS
- }
-
- bazelCtx := ctx.Config().BazelContext
- filePaths, ok := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{archVariant, osVariant})
- if !ok {
- return
- }
-
- bazelOuts := make(Paths, 0, len(filePaths))
- for _, p := range filePaths {
- src := PathForBazelOut(ctx, p)
- bazelOuts = append(bazelOuts, src)
- }
-
- fg.srcs = bazelOuts
-}
-
func (fg *fileGroup) GenerateAndroidBuildActions(ctx ModuleContext) {
fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
if fg.properties.Path != nil {
fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
}
-
- fg.maybeGenerateBazelBuildActions(ctx)
}
func (fg *fileGroup) Srcs() Paths {
@@ -161,3 +132,38 @@
ctx.StrictRaw(makeVar, strings.Join(fg.srcs.Strings(), " "))
}
}
+
+func (fg *fileGroup) QueueBazelCall(ctx BaseModuleContext) {
+ bazelCtx := ctx.Config().BazelContext
+
+ bazelCtx.QueueBazelRequest(
+ fg.GetBazelLabel(ctx, fg),
+ cquery.GetOutputFiles,
+ configKey{Common.String(), CommonOS})
+}
+
+func (fg *fileGroup) IsMixedBuildSupported(ctx BaseModuleContext) bool {
+ return true
+}
+
+func (fg *fileGroup) ProcessBazelQueryResponse(ctx ModuleContext) {
+ fg.srcs = PathsForModuleSrcExcludes(ctx, fg.properties.Srcs, fg.properties.Exclude_srcs)
+ if fg.properties.Path != nil {
+ fg.srcs = PathsWithModuleSrcSubDir(ctx, fg.srcs, String(fg.properties.Path))
+ }
+
+ bazelCtx := ctx.Config().BazelContext
+ filePaths, err := bazelCtx.GetOutputFiles(fg.GetBazelLabel(ctx, fg), configKey{Common.String(), CommonOS})
+ if err != nil {
+ ctx.ModuleErrorf(err.Error())
+ return
+ }
+
+ bazelOuts := make(Paths, 0, len(filePaths))
+ for _, p := range filePaths {
+ src := PathForBazelOut(ctx, p)
+ bazelOuts = append(bazelOuts, src)
+ }
+
+ fg.srcs = bazelOuts
+}
diff --git a/android/module.go b/android/module.go
index ab68e24..97a46dd 100644
--- a/android/module.go
+++ b/android/module.go
@@ -2275,7 +2275,11 @@
return
}
- m.module.GenerateAndroidBuildActions(ctx)
+ if mixedBuildMod, handled := m.isHandledByBazel(ctx); handled {
+ mixedBuildMod.ProcessBazelQueryResponse(ctx)
+ } else {
+ m.module.GenerateAndroidBuildActions(ctx)
+ }
if ctx.Failed() {
return
}
@@ -2331,6 +2335,18 @@
m.variables = ctx.variables
}
+func (m *ModuleBase) isHandledByBazel(ctx ModuleContext) (MixedBuildBuildable, bool) {
+ if !ctx.Config().BazelContext.BazelEnabled() {
+ return nil, false
+ }
+ if mixedBuildMod, ok := m.module.(MixedBuildBuildable); ok {
+ if mixedBuildMod.IsMixedBuildSupported(ctx) && MixedBuildsEnabled(ctx) {
+ return mixedBuildMod, true
+ }
+ }
+ return nil, false
+}
+
// Check the supplied dist structure to make sure that it is valid.
//
// property - the base property, e.g. dist or dists[1], which is combined with the
diff --git a/cc/binary.go b/cc/binary.go
index c5017c1..7b5591a 100644
--- a/cc/binary.go
+++ b/cc/binary.go
@@ -17,6 +17,7 @@
import (
"path/filepath"
+ "android/soong/bazel/cquery"
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
@@ -562,25 +563,32 @@
}
type ccBinaryBazelHandler struct {
- android.BazelHandler
+ BazelHandler
module *Module
}
-func (handler *ccBinaryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (handler *ccBinaryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
- if ok {
- if len(filePaths) != 1 {
- ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths)
- return false
- }
- outputFilePath := android.PathForBazelOut(ctx, filePaths[0])
- handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
- // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped.
- handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath
+ bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx))
+}
+
+func (handler *ccBinaryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
+ if err != nil {
+ ctx.ModuleErrorf(err.Error())
+ return
}
- return ok
+
+ if len(filePaths) != 1 {
+ ctx.ModuleErrorf("expected exactly one output file for '%s', but got %s", label, filePaths)
+ return
+ }
+ outputFilePath := android.PathForBazelOut(ctx, filePaths[0])
+ handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
+ // TODO(b/220164721): We need to decide if we should return the stripped as the unstripped.
+ handler.module.linker.(*binaryDecorator).unstrippedOutputFile = outputFilePath
}
func binaryBp2build(ctx android.TopDownMutatorContext, m *Module, typ string) {
diff --git a/cc/cc.go b/cc/cc.go
index 8606920..3d21f63 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -772,6 +772,19 @@
return ok && ccDepTag == testPerSrcDepTag
}
+// bazelHandler is the interface for a helper object related to deferring to Bazel for
+// processing a cc module (during Bazel mixed builds). Individual module types should define
+// their own bazel handler if they support being handled by Bazel.
+type BazelHandler interface {
+ // QueueBazelCall invokes request-queueing functions on the BazelContext
+ //so that these requests are handled when Bazel's cquery is invoked.
+ QueueBazelCall(ctx android.BaseModuleContext, label string)
+
+ // ProcessBazelQueryResponse uses information retrieved from Bazel to set properties
+ // on the current module with given label.
+ ProcessBazelQueryResponse(ctx android.ModuleContext, label string)
+}
+
// Module contains the properties and members used by all C/C++ module types, and implements
// the blueprint.Module interface. It delegates to compiler, linker, and installer interfaces
// to construct the output file. Behavior can be customized with a Customizer, or "decorator",
@@ -811,7 +824,7 @@
compiler compiler
linker linker
installer installer
- bazelHandler android.BazelHandler
+ bazelHandler BazelHandler
features []feature
stl *stl
@@ -1773,31 +1786,51 @@
return subName
}
-// Returns true if Bazel was successfully used for the analysis of this module.
-func (c *Module) maybeGenerateBazelActions(actx android.ModuleContext) bool {
- var bazelModuleLabel string
+var _ android.MixedBuildBuildable = (*Module)(nil)
+
+func (c *Module) getBazelModuleLabel(ctx android.BaseModuleContext) string {
if c.typ() == fullLibrary && c.static() {
// cc_library is a special case in bp2build; two targets are generated -- one for each
// of the shared and static variants. The shared variant keeps the module name, but the
// static variant uses a different suffixed name.
- bazelModuleLabel = bazelLabelForStaticModule(actx, c)
- } else {
- bazelModuleLabel = c.GetBazelLabel(actx, c)
+ return bazelLabelForStaticModule(ctx, c)
+ }
+ return c.GetBazelLabel(ctx, c)
+}
+
+func (c *Module) QueueBazelCall(ctx android.BaseModuleContext) {
+ c.bazelHandler.QueueBazelCall(ctx, c.getBazelModuleLabel(ctx))
+}
+
+func (c *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
+ return c.bazelHandler != nil
+}
+
+func (c *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
+ bazelModuleLabel := c.getBazelModuleLabel(ctx)
+
+ c.bazelHandler.ProcessBazelQueryResponse(ctx, bazelModuleLabel)
+
+ c.Properties.SubName = GetSubnameProperty(ctx, c)
+ apexInfo := ctx.Provider(android.ApexInfoProvider).(android.ApexInfo)
+ if !apexInfo.IsForPlatform() {
+ c.hideApexVariantFromMake = true
}
- bazelActionsUsed := false
- // Mixed builds mode is disabled for modules outside of device OS.
- // TODO(b/200841190): Support non-device OS in mixed builds.
- if android.MixedBuildsEnabled(actx) && c.bazelHandler != nil {
- bazelActionsUsed = c.bazelHandler.GenerateBazelBuildActions(actx, bazelModuleLabel)
+ c.makeLinkType = GetMakeLinkType(ctx, c)
+
+ mctx := &moduleContext{
+ ModuleContext: ctx,
+ moduleContextImpl: moduleContextImpl{
+ mod: c,
+ },
}
- return bazelActionsUsed
+ mctx.ctx = mctx
+
+ c.maybeInstall(mctx, apexInfo)
}
func (c *Module) GenerateAndroidBuildActions(actx android.ModuleContext) {
- // TODO(cparsons): Any logic in this method occurring prior to querying Bazel should be
- // requested from Bazel instead.
-
// Handle the case of a test module split by `test_per_src` mutator.
//
// The `test_per_src` mutator adds an extra variation named "", depending on all the other
@@ -1824,11 +1857,6 @@
}
ctx.ctx = ctx
- if c.maybeGenerateBazelActions(actx) {
- c.maybeInstall(ctx, apexInfo)
- return
- }
-
deps := c.depsToPaths(ctx)
if ctx.Failed() {
return
diff --git a/cc/library.go b/cc/library.go
index fdbbccb..5746529 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -642,18 +642,18 @@
}
type ccLibraryBazelHandler struct {
- android.BazelHandler
+ BazelHandler
module *Module
}
// generateStaticBazelBuildActions constructs the StaticLibraryInfo Soong
// provider from a Bazel shared library's CcInfo provider.
-func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool {
+func (handler *ccLibraryBazelHandler) generateStaticBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) {
rootStaticArchives := ccInfo.RootStaticArchives
if len(rootStaticArchives) != 1 {
ctx.ModuleErrorf("expected exactly one root archive file for '%s', but got %s", label, rootStaticArchives)
- return false
+ return
}
outputFilePath := android.PathForBazelOut(ctx, rootStaticArchives[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
@@ -679,17 +679,17 @@
Build(),
})
- return true
+ return
}
// generateSharedBazelBuildActions constructs the SharedLibraryInfo Soong
// provider from a Bazel shared library's CcInfo provider.
-func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) bool {
+func (handler *ccLibraryBazelHandler) generateSharedBazelBuildActions(ctx android.ModuleContext, label string, ccInfo cquery.CcInfo) {
rootDynamicLibraries := ccInfo.RootDynamicLibraries
if len(rootDynamicLibraries) != 1 {
ctx.ModuleErrorf("expected exactly one root dynamic library file for '%s', but got %s", label, rootDynamicLibraries)
- return false
+ return
}
outputFilePath := android.PathForBazelOut(ctx, rootDynamicLibraries[0])
handler.module.outputFile = android.OptionalPathForPath(outputFilePath)
@@ -709,30 +709,27 @@
// TODO(b/190524881): Include transitive static libraries in this provider to support
// static libraries with deps. The provider key for this is TransitiveStaticLibrariesForOrdering.
})
- return true
}
-func (handler *ccLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (handler *ccLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
+ bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
+}
+
+func (handler *ccLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
- return false
- }
- if !ok {
- return ok
+ return
}
if handler.module.static() {
- if ok := handler.generateStaticBazelBuildActions(ctx, label, ccInfo); !ok {
- return false
- }
+ handler.generateStaticBazelBuildActions(ctx, label, ccInfo)
} else if handler.module.Shared() {
- if ok := handler.generateSharedBazelBuildActions(ctx, label, ccInfo); !ok {
- return false
- }
+ handler.generateSharedBazelBuildActions(ctx, label, ccInfo)
} else {
- return false
+ ctx.ModuleErrorf("Unhandled bazel case for %s (neither shared nor static!)", ctx.ModuleName())
}
handler.module.linker.(*libraryDecorator).setFlagExporterInfoFromCcInfo(ctx, ccInfo)
@@ -746,7 +743,6 @@
// implementation.
i.(*libraryDecorator).collectedSnapshotHeaders = android.Paths{}
}
- return ok
}
func (library *libraryDecorator) setFlagExporterInfoFromCcInfo(ctx android.ModuleContext, ccInfo cquery.CcInfo) {
diff --git a/cc/library_headers.go b/cc/library_headers.go
index 41ebcc7..6fd9568 100644
--- a/cc/library_headers.go
+++ b/cc/library_headers.go
@@ -17,6 +17,7 @@
import (
"android/soong/android"
"android/soong/bazel"
+ "android/soong/bazel/cquery"
)
func init() {
@@ -47,28 +48,30 @@
ctx.RegisterModuleType("cc_prebuilt_library_headers", prebuiltLibraryHeaderFactory)
}
-type libraryHeaderBazelHander struct {
- android.BazelHandler
+type libraryHeaderBazelHandler struct {
+ BazelHandler
module *Module
library *libraryDecorator
}
-func (h *libraryHeaderBazelHander) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (handler *libraryHeaderBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
+ bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
+}
+
+func (h *libraryHeaderBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
- ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
- return false
- }
- if !ok {
- return false
+ ctx.ModuleErrorf(err.Error())
+ return
}
outputPaths := ccInfo.OutputFiles
if len(outputPaths) != 1 {
ctx.ModuleErrorf("expected exactly one output file for %q, but got %q", label, outputPaths)
- return false
+ return
}
outputPath := android.PathForBazelOut(ctx, outputPaths[0])
@@ -83,8 +86,6 @@
// validation will fail. For now, set this to an empty list.
// TODO(cparsons): More closely mirror the collectHeadersForSnapshot implementation.
h.library.collectedSnapshotHeaders = android.Paths{}
-
- return true
}
// cc_library_headers contains a set of c/c++ headers which are imported by
@@ -96,7 +97,7 @@
library.HeaderOnly()
module.sdkMemberTypes = []android.SdkMemberType{headersLibrarySdkMemberType}
module.bazelable = true
- module.bazelHandler = &libraryHeaderBazelHander{module: module, library: library}
+ module.bazelHandler = &libraryHeaderBazelHandler{module: module, library: library}
return module.Init()
}
diff --git a/cc/object.go b/cc/object.go
index bd5bd45..a3be6b1 100644
--- a/cc/object.go
+++ b/cc/object.go
@@ -19,6 +19,7 @@
"android/soong/android"
"android/soong/bazel"
+ "android/soong/bazel/cquery"
)
//
@@ -46,23 +47,30 @@
}
type objectBazelHandler struct {
- android.BazelHandler
+ BazelHandler
module *Module
}
-func (handler *objectBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (handler *objectBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- objPaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
- if ok {
- if len(objPaths) != 1 {
- ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths)
- return false
- }
+ bazelCtx.QueueBazelRequest(label, cquery.GetOutputFiles, android.GetConfigKey(ctx))
+}
- handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0]))
+func (handler *objectBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ objPaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
+ if err != nil {
+ ctx.ModuleErrorf(err.Error())
+ return
}
- return ok
+
+ if len(objPaths) != 1 {
+ ctx.ModuleErrorf("expected exactly one object file for '%s', but got %s", label, objPaths)
+ return
+ }
+
+ handler.module.outputFile = android.OptionalPathForPath(android.PathForBazelOut(ctx, objPaths[0]))
}
type ObjectLinkerProperties struct {
diff --git a/cc/prebuilt.go b/cc/prebuilt.go
index f54c6f8..e8c60ba 100644
--- a/cc/prebuilt.go
+++ b/cc/prebuilt.go
@@ -20,6 +20,7 @@
"android/soong/android"
"android/soong/bazel"
+ "android/soong/bazel/cquery"
)
func init() {
@@ -406,25 +407,28 @@
}
type prebuiltStaticLibraryBazelHandler struct {
- android.BazelHandler
+ BazelHandler
module *Module
library *libraryDecorator
}
-func (h *prebuiltStaticLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (h *prebuiltStaticLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
+ bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
+}
+
+func (h *prebuiltStaticLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
- ctx.ModuleErrorf("Error getting Bazel CcInfo: %s", err)
- }
- if !ok {
- return false
+ ctx.ModuleErrorf(err.Error())
+ return
}
staticLibs := ccInfo.CcStaticLibraryFiles
if len(staticLibs) > 1 {
ctx.ModuleErrorf("expected 1 static library from bazel target %q, got %s", label, staticLibs)
- return false
+ return
}
// TODO(b/184543518): cc_prebuilt_library_static may have properties for re-exporting flags
@@ -439,7 +443,7 @@
if len(staticLibs) == 0 {
h.module.outputFile = android.OptionalPath{}
- return true
+ return
}
out := android.PathForBazelOut(ctx, staticLibs[0])
@@ -451,30 +455,31 @@
TransitiveStaticLibrariesForOrdering: depSet,
})
-
- return true
}
type prebuiltSharedLibraryBazelHandler struct {
- android.BazelHandler
+ BazelHandler
module *Module
library *libraryDecorator
}
-func (h *prebuiltSharedLibraryBazelHandler) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (h *prebuiltSharedLibraryBazelHandler) QueueBazelCall(ctx android.BaseModuleContext, label string) {
bazelCtx := ctx.Config().BazelContext
- ccInfo, ok, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
+ bazelCtx.QueueBazelRequest(label, cquery.GetCcInfo, android.GetConfigKey(ctx))
+}
+
+func (h *prebuiltSharedLibraryBazelHandler) ProcessBazelQueryResponse(ctx android.ModuleContext, label string) {
+ bazelCtx := ctx.Config().BazelContext
+ ccInfo, err := bazelCtx.GetCcInfo(label, android.GetConfigKey(ctx))
if err != nil {
- ctx.ModuleErrorf("Error getting Bazel CcInfo for %s: %s", label, err)
- }
- if !ok {
- return false
+ ctx.ModuleErrorf(err.Error())
+ return
}
sharedLibs := ccInfo.CcSharedLibraryFiles
if len(sharedLibs) != 1 {
ctx.ModuleErrorf("expected 1 shared library from bazel target %s, got %q", label, sharedLibs)
- return false
+ return
}
// TODO(b/184543518): cc_prebuilt_library_shared may have properties for re-exporting flags
@@ -489,7 +494,7 @@
if len(sharedLibs) == 0 {
h.module.outputFile = android.OptionalPath{}
- return true
+ return
}
out := android.PathForBazelOut(ctx, sharedLibs[0])
@@ -514,8 +519,6 @@
h.library.setFlagExporterInfoFromCcInfo(ctx, ccInfo)
h.module.maybeUnhideFromMake()
-
- return true
}
func (p *prebuiltObjectLinker) prebuilt() *android.Prebuilt {
diff --git a/cmd/soong_build/main.go b/cmd/soong_build/main.go
index 4b3161b..ad379d5 100644
--- a/cmd/soong_build/main.go
+++ b/cmd/soong_build/main.go
@@ -129,44 +129,26 @@
return configuration
}
-// Bazel-enabled mode. Soong runs in two passes.
-// First pass: Analyze the build tree, but only store all bazel commands
-// needed to correctly evaluate the tree in the second pass.
-// TODO(cparsons): Don't output any ninja file, as the second pass will overwrite
-// the incorrect results from the first pass, and file I/O is expensive.
-func runMixedModeBuild(configuration android.Config, firstCtx *android.Context, extraNinjaDeps []string) {
- firstCtx.EventHandler.Begin("mixed_build")
- defer firstCtx.EventHandler.End("mixed_build")
+// Bazel-enabled mode. Attaches a mutator to queue Bazel requests, adds a
+// BeforePrepareBuildActionsHook to invoke Bazel, and then uses Bazel metadata
+// for modules that should be handled by Bazel.
+func runMixedModeBuild(configuration android.Config, ctx *android.Context, extraNinjaDeps []string) {
+ ctx.EventHandler.Begin("mixed_build")
+ defer ctx.EventHandler.End("mixed_build")
- firstCtx.EventHandler.Begin("prepare")
- bootstrap.RunBlueprint(cmdlineArgs, bootstrap.StopBeforeWriteNinja, firstCtx.Context, configuration)
- firstCtx.EventHandler.End("prepare")
-
- firstCtx.EventHandler.Begin("bazel")
- // Invoke bazel commands and save results for second pass.
- if err := configuration.BazelContext.InvokeBazel(); err != nil {
- fmt.Fprintf(os.Stderr, "%s", err)
- os.Exit(1)
+ bazelHook := func() error {
+ ctx.EventHandler.Begin("bazel")
+ defer ctx.EventHandler.End("bazel")
+ return configuration.BazelContext.InvokeBazel()
}
- // Second pass: Full analysis, using the bazel command results. Output ninja file.
- secondConfig, err := android.ConfigForAdditionalRun(configuration)
- if err != nil {
- fmt.Fprintf(os.Stderr, "%s", err)
- os.Exit(1)
- }
- firstCtx.EventHandler.End("bazel")
+ ctx.SetBeforePrepareBuildActionsHook(bazelHook)
- secondCtx := newContext(secondConfig)
- secondCtx.EventHandler = firstCtx.EventHandler
- secondCtx.EventHandler.Begin("analyze")
- ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, secondCtx.Context, secondConfig)
- ninjaDeps = append(ninjaDeps, extraNinjaDeps...)
- secondCtx.EventHandler.End("analyze")
+ ninjaDeps := bootstrap.RunBlueprint(cmdlineArgs, bootstrap.DoEverything, ctx.Context, configuration)
- globListFiles := writeBuildGlobsNinjaFile(secondCtx, configuration.SoongOutDir(), configuration)
+ globListFiles := writeBuildGlobsNinjaFile(ctx, configuration.SoongOutDir(), configuration)
ninjaDeps = append(ninjaDeps, globListFiles...)
- writeDepFile(cmdlineArgs.OutFile, *secondCtx.EventHandler, ninjaDeps)
+ writeDepFile(cmdlineArgs.OutFile, *ctx.EventHandler, ninjaDeps)
}
// Run the code-generation phase to convert BazelTargetModules to BUILD files.
diff --git a/genrule/genrule.go b/genrule/genrule.go
index 3531ee6..8649b15 100644
--- a/genrule/genrule.go
+++ b/genrule/genrule.go
@@ -25,6 +25,7 @@
"strconv"
"strings"
+ "android/soong/bazel/cquery"
"github.com/google/blueprint"
"github.com/google/blueprint/bootstrap"
"github.com/google/blueprint/proptools"
@@ -189,6 +190,8 @@
modulePaths []string
}
+var _ android.MixedBuildBuildable = (*Module)(nil)
+
type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
type generateTask struct {
@@ -249,27 +252,36 @@
}
}
-// Returns true if information was available from Bazel, false if bazel invocation still needs to occur.
-func (c *Module) GenerateBazelBuildActions(ctx android.ModuleContext, label string) bool {
+func (g *Module) ProcessBazelQueryResponse(ctx android.ModuleContext) {
+ g.generateCommonBuildActions(ctx)
+
+ label := g.GetBazelLabel(ctx, g)
bazelCtx := ctx.Config().BazelContext
- filePaths, ok := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
- if ok {
- var bazelOutputFiles android.Paths
- exportIncludeDirs := map[string]bool{}
- for _, bazelOutputFile := range filePaths {
- bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile))
- exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
- }
- c.outputFiles = bazelOutputFiles
- c.outputDeps = bazelOutputFiles
- for includePath, _ := range exportIncludeDirs {
- c.exportedIncludeDirs = append(c.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
- }
+ filePaths, err := bazelCtx.GetOutputFiles(label, android.GetConfigKey(ctx))
+ if err != nil {
+ ctx.ModuleErrorf(err.Error())
+ return
}
- return ok
+
+ var bazelOutputFiles android.Paths
+ exportIncludeDirs := map[string]bool{}
+ for _, bazelOutputFile := range filePaths {
+ bazelOutputFiles = append(bazelOutputFiles, android.PathForBazelOut(ctx, bazelOutputFile))
+ exportIncludeDirs[filepath.Dir(bazelOutputFile)] = true
+ }
+ g.outputFiles = bazelOutputFiles
+ g.outputDeps = bazelOutputFiles
+ for includePath, _ := range exportIncludeDirs {
+ g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForBazelOut(ctx, includePath))
+ }
}
-func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+// generateCommonBuildActions contains build action generation logic
+// common to both the mixed build case and the legacy case of genrule processing.
+// To fully support genrule in mixed builds, the contents of this function should
+// approach zero; there should be no genrule action registration done directly
+// by Soong logic in the mixed-build case.
+func (g *Module) generateCommonBuildActions(ctx android.ModuleContext) {
g.subName = ctx.ModuleSubDir()
// Collect the module directory for IDE info in java/jdeps.go.
@@ -575,29 +587,35 @@
}
g.outputFiles = outputFiles.Paths()
+}
- bazelModuleLabel := g.GetBazelLabel(ctx, g)
- bazelActionsUsed := false
- if android.MixedBuildsEnabled(ctx) {
- bazelActionsUsed = g.GenerateBazelBuildActions(ctx, bazelModuleLabel)
+func (g *Module) GenerateAndroidBuildActions(ctx android.ModuleContext) {
+ g.generateCommonBuildActions(ctx)
+
+ // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
+ // the genrules on AOSP. That will make things simpler to look at the graph in the common
+ // case. For larger sets of outputs, inject a phony target in between to limit ninja file
+ // growth.
+ if len(g.outputFiles) <= 6 {
+ g.outputDeps = g.outputFiles
+ } else {
+ phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
+ ctx.Build(pctx, android.BuildParams{
+ Rule: blueprint.Phony,
+ Output: phonyFile,
+ Inputs: g.outputFiles,
+ })
+ g.outputDeps = android.Paths{phonyFile}
}
- if !bazelActionsUsed {
- // For <= 6 outputs, just embed those directly in the users. Right now, that covers >90% of
- // the genrules on AOSP. That will make things simpler to look at the graph in the common
- // case. For larger sets of outputs, inject a phony target in between to limit ninja file
- // growth.
- if len(g.outputFiles) <= 6 {
- g.outputDeps = g.outputFiles
- } else {
- phonyFile := android.PathForModuleGen(ctx, "genrule-phony")
- ctx.Build(pctx, android.BuildParams{
- Rule: blueprint.Phony,
- Output: phonyFile,
- Inputs: g.outputFiles,
- })
- g.outputDeps = android.Paths{phonyFile}
- }
- }
+}
+
+func (g *Module) QueueBazelCall(ctx android.BaseModuleContext) {
+ bazelCtx := ctx.Config().BazelContext
+ bazelCtx.QueueBazelRequest(g.GetBazelLabel(ctx, g), cquery.GetOutputFiles, android.GetConfigKey(ctx))
+}
+
+func (g *Module) IsMixedBuildSupported(ctx android.BaseModuleContext) bool {
+ return true
}
// Collect information for opening IDE project files in java/jdeps.go.
