Snap for 8782634 from e76d4122ee550cf1f7da8e64bcd6f77fc5542e00 to sdk-release
Change-Id: I665dc6e68887dcbd5054fb49be77f164173aee59
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 8c045b4..3342bbe 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -113,6 +113,13 @@
return ninjaDeps
}
+ if ctx.BeforePrepareBuildActionsHook != nil {
+ err := ctx.BeforePrepareBuildActionsHook()
+ if err != nil {
+ fatalErrors([]error{err})
+ }
+ }
+
extraDeps, errs = ctx.PrepareBuildActions(config)
if len(errs) > 0 {
fatalErrors(errs)
diff --git a/bpmodify/bpmodify.go b/bpmodify/bpmodify.go
index 431eb83..aecda73 100644
--- a/bpmodify/bpmodify.go
+++ b/bpmodify/bpmodify.go
@@ -15,6 +15,7 @@
"os/exec"
"path/filepath"
"strings"
+ "syscall"
"unicode"
"github.com/google/blueprint/parser"
@@ -68,6 +69,11 @@
return err
}
defer f.Close()
+
+ if *write {
+ syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
+ }
+
in = f
}
diff --git a/context.go b/context.go
index 47b31ae..5448f1e 100644
--- a/context.go
+++ b/context.go
@@ -75,6 +75,8 @@
// Used for metrics-related event logging.
EventHandler *metrics.EventHandler
+ BeforePrepareBuildActionsHook func() error
+
moduleFactories map[string]ModuleFactory
nameInterface NameInterface
moduleGroups []*moduleGroup
@@ -270,6 +272,11 @@
// set during each runMutator
splitModules modulesOrAliases
+ // Used by TransitionMutator implementations
+ transitionVariations []string
+ currentTransitionMutator string
+ requiredVariationsLock sync.Mutex
+
// set during PrepareBuildActions
actionDefs localBuildActions
@@ -617,6 +624,230 @@
return info
}
+type IncomingTransitionContext interface {
+ // Module returns the target of the dependency edge for which the transition
+ // is being computed
+ Module() Module
+
+ // Config returns the config object that was passed to
+ // Context.PrepareBuildActions.
+ Config() interface{}
+}
+
+type OutgoingTransitionContext interface {
+ // Module returns the target of the dependency edge for which the transition
+ // is being computed
+ Module() Module
+
+ // DepTag() Returns the dependency tag through which this dependency is
+ // reached
+ DepTag() DependencyTag
+}
+
+// Transition mutators implement a top-down mechanism where a module tells its
+// direct dependencies what variation they should be built in but the dependency
+// has the final say.
+//
+// When implementing a transition mutator, one needs to implement four methods:
+// - Split() that tells what variations a module has by itself
+// - OutgoingTransition() where a module tells what it wants from its
+// dependency
+// - IncomingTransition() where a module has the final say about its own
+// variation
+// - Mutate() that changes the state of a module depending on its variation
+//
+// That the effective variation of module B when depended on by module A is the
+// composition the outgoing transition of module A and the incoming transition
+// of module B.
+//
+// the outgoing transition should not take the properties of the dependency into
+// account, only those of the module that depends on it. For this reason, the
+// dependency is not even passed into it as an argument. Likewise, the incoming
+// transition should not take the properties of the depending module into
+// account and is thus not informed about it. This makes for a nice
+// decomposition of the decision logic.
+//
+// A given transition mutator only affects its own variation; other variations
+// stay unchanged along the dependency edges.
+//
+// Soong makes sure that all modules are created in the desired variations and
+// that dependency edges are set up correctly. This ensures that "missing
+// variation" errors do not happen and allows for more flexible changes in the
+// value of the variation among dependency edges (as oppposed to bottom-up
+// mutators where if module A in variation X depends on module B and module B
+// has that variation X, A must depend on variation X of B)
+//
+// The limited power of the context objects passed to individual mutators
+// methods also makes it more difficult to shoot oneself in the foot. Complete
+// safety is not guaranteed because no one prevents individual transition
+// mutators from mutating modules in illegal ways and for e.g. Split() or
+// Mutate() to run their own visitations of the transitive dependency of the
+// module and both of these are bad ideas, but it's better than no guardrails at
+// all.
+//
+// This model is pretty close to Bazel's configuration transitions. The mapping
+// between concepts in Soong and Bazel is as follows:
+// - Module == configured target
+// - Variant == configuration
+// - Variation name == configuration flag
+// - Variation == configuration flag value
+// - Outgoing transition == attribute transition
+// - Incoming transition == rule transition
+//
+// The Split() method does not have a Bazel equivalent and Bazel split
+// transitions do not have a Soong equivalent.
+//
+// Mutate() does not make sense in Bazel due to the different models of the
+// two systems: when creating new variations, Soong clones the old module and
+// thus some way is needed to change it state whereas Bazel creates each
+// configuration of a given configured target anew.
+type TransitionMutator interface {
+ // Returns the set of variations that should be created for a module no matter
+ // who depends on it. Used when Make depends on a particular variation or when
+ // the module knows its variations just based on information given to it in
+ // the Blueprint file. This method should not mutate the module it is called
+ // on.
+ Split(ctx BaseModuleContext) []string
+
+ // Called on a module to determine which variation it wants from its direct
+ // dependencies. The dependency itself can override this decision. This method
+ // should not mutate the module itself.
+ OutgoingTransition(ctx OutgoingTransitionContext, sourceVariation string) string
+
+ // Called on a module to determine which variation it should be in based on
+ // the variation modules that depend on it want. This gives the module a final
+ // say about its own variations. This method should not mutate the module
+ // itself.
+ IncomingTransition(ctx IncomingTransitionContext, incomingVariation string) string
+
+ // Called after a module was split into multiple variations on each variation.
+ // It should not split the module any further but adding new dependencies is
+ // fine. Unlike all the other methods on TransitionMutator, this method is
+ // allowed to mutate the module.
+ Mutate(ctx BottomUpMutatorContext, variation string)
+}
+
+type transitionMutatorImpl struct {
+ name string
+ mutator TransitionMutator
+}
+
+// Adds each argument in items to l if it's not already there.
+func addToStringListIfNotPresent(l []string, items ...string) []string {
+OUTER:
+ for _, i := range items {
+ for _, existing := range l {
+ if existing == i {
+ continue OUTER
+ }
+ }
+
+ l = append(l, i)
+ }
+
+ return l
+}
+
+func (t *transitionMutatorImpl) addRequiredVariation(m *moduleInfo, variation string) {
+ m.requiredVariationsLock.Lock()
+ defer m.requiredVariationsLock.Unlock()
+
+ // This is only a consistency check. Leaking the variations of a transition
+ // mutator to another one could well lead to issues that are difficult to
+ // track down.
+ if m.currentTransitionMutator != "" && m.currentTransitionMutator != t.name {
+ panic(fmt.Errorf("transition mutator is %s in mutator %s", m.currentTransitionMutator, t.name))
+ }
+
+ m.currentTransitionMutator = t.name
+ m.transitionVariations = addToStringListIfNotPresent(m.transitionVariations, variation)
+}
+
+func (t *transitionMutatorImpl) topDownMutator(mctx TopDownMutatorContext) {
+ module := mctx.(*mutatorContext).module
+ mutatorSplits := t.mutator.Split(mctx)
+ if mutatorSplits == nil || len(mutatorSplits) == 0 {
+ panic(fmt.Errorf("transition mutator %s returned no splits for module %s", t.name, mctx.ModuleName()))
+ }
+
+ // transitionVariations for given a module can be mutated by the module itself
+ // and modules that directly depend on it. Since this is a top-down mutator,
+ // all modules that directly depend on this module have already been processed
+ // so no locking is necessary.
+ module.transitionVariations = addToStringListIfNotPresent(module.transitionVariations, mutatorSplits...)
+ sort.Strings(module.transitionVariations)
+
+ for _, srcVariation := range module.transitionVariations {
+ for _, dep := range module.directDeps {
+ finalVariation := t.transition(mctx)(mctx.Module(), srcVariation, dep.module.logicModule, dep.tag)
+ t.addRequiredVariation(dep.module, finalVariation)
+ }
+ }
+}
+
+type transitionContextImpl struct {
+ module Module
+ depTag DependencyTag
+ config interface{}
+}
+
+func (c *transitionContextImpl) Module() Module {
+ return c.module
+}
+
+func (c *transitionContextImpl) DepTag() DependencyTag {
+ return c.depTag
+}
+
+func (c *transitionContextImpl) Config() interface{} {
+ return c.config
+}
+
+func (t *transitionMutatorImpl) transition(mctx BaseMutatorContext) Transition {
+ return func(source Module, sourceVariation string, dep Module, depTag DependencyTag) string {
+ tc := &transitionContextImpl{module: dep, depTag: depTag, config: mctx.Config()}
+ outgoingVariation := t.mutator.OutgoingTransition(tc, sourceVariation)
+ finalVariation := t.mutator.IncomingTransition(tc, outgoingVariation)
+ return finalVariation
+ }
+}
+
+func (t *transitionMutatorImpl) bottomUpMutator(mctx BottomUpMutatorContext) {
+ mc := mctx.(*mutatorContext)
+ // Fetch and clean up transition mutator state. No locking needed since the
+ // only time interaction between multiple modules is required is during the
+ // computation of the variations required by a given module.
+ variations := mc.module.transitionVariations
+ mc.module.transitionVariations = nil
+ mc.module.currentTransitionMutator = ""
+
+ if len(variations) < 1 {
+ panic(fmt.Errorf("no variations found for module %s by mutator %s",
+ mctx.ModuleName(), t.name))
+ }
+
+ if len(variations) == 1 && variations[0] == "" {
+ // Module is not split, just apply the transition
+ mc.applyTransition(t.transition(mctx))
+ } else {
+ mc.createVariationsWithTransition(t.transition(mctx), variations...)
+ }
+}
+
+func (t *transitionMutatorImpl) mutateMutator(mctx BottomUpMutatorContext) {
+ module := mctx.(*mutatorContext).module
+ currentVariation := module.variant.variations[t.name]
+ t.mutator.Mutate(mctx, currentVariation)
+}
+
+func (c *Context) RegisterTransitionMutator(name string, mutator TransitionMutator) {
+ impl := &transitionMutatorImpl{name: name, mutator: mutator}
+
+ c.RegisterTopDownMutator(name+"_deps", impl.topDownMutator).Parallel()
+ c.RegisterBottomUpMutator(name, impl.bottomUpMutator).Parallel()
+ c.RegisterBottomUpMutator(name+"_mutate", impl.mutateMutator).Parallel()
+}
+
type MutatorHandle interface {
// Set the mutator to visit modules in parallel while maintaining ordering. Calling any
// method on the mutator context is thread-safe, but the mutator must handle synchronization
@@ -1301,7 +1532,7 @@
}
func (c *Context) createVariations(origModule *moduleInfo, mutatorName string,
- defaultVariationName *string, variationNames []string, local bool) (modulesOrAliases, []error) {
+ depChooser depChooser, variationNames []string, local bool) (modulesOrAliases, []error) {
if len(variationNames) == 0 {
panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q",
@@ -1337,7 +1568,7 @@
newModules = append(newModules, newModule)
- newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName, defaultVariationName)
+ newErrs := c.convertDepsToVariation(newModule, depChooser)
if len(newErrs) > 0 {
errs = append(errs, newErrs...)
}
@@ -1353,31 +1584,79 @@
return newModules, errs
}
-func (c *Context) convertDepsToVariation(module *moduleInfo,
- mutatorName, variationName string, defaultVariationName *string) (errs []error) {
+type depChooser func(source *moduleInfo, dep depInfo) (*moduleInfo, string)
+// This function is called for every dependency edge to determine which
+// variation of the dependency is needed. Its inputs are the depending module,
+// its variation, the dependency and the dependency tag.
+type Transition func(source Module, sourceVariation string, dep Module, depTag DependencyTag) string
+
+func chooseDepByTransition(mutatorName string, transition Transition) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ sourceVariation := source.variant.variations[mutatorName]
+ depLogicModule := dep.module.logicModule
+ if depLogicModule == nil {
+ // This is really a lie because the original dependency before the split
+ // went away when it was split. We choose an arbitrary split module
+ // instead and hope that whatever information the transition wants from it
+ // is the same as in the original one
+ // TODO(lberki): this can be fixed by calling transition() once and saving
+ // its results somewhere
+ depLogicModule = dep.module.splitModules[0].moduleOrAliasTarget().logicModule
+ }
+
+ desiredVariation := transition(source.logicModule, sourceVariation, depLogicModule, dep.tag)
+ for _, m := range dep.module.splitModules {
+ if m.moduleOrAliasVariant().variations[mutatorName] == desiredVariation {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+
+ return nil, desiredVariation
+ }
+}
+
+func chooseDep(candidates modulesOrAliases, mutatorName, variationName string, defaultVariationName *string) (*moduleInfo, string) {
+ for _, m := range candidates {
+ if m.moduleOrAliasVariant().variations[mutatorName] == variationName {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+
+ if defaultVariationName != nil {
+ // give it a second chance; match with defaultVariationName
+ for _, m := range candidates {
+ if m.moduleOrAliasVariant().variations[mutatorName] == *defaultVariationName {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+ }
+
+ return nil, variationName
+}
+
+func chooseDepExplicit(mutatorName string,
+ variationName string, defaultVariationName *string) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ return chooseDep(dep.module.splitModules, mutatorName, variationName, defaultVariationName)
+ }
+}
+
+func chooseDepInherit(mutatorName string, defaultVariationName *string) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ sourceVariation := source.variant.variations[mutatorName]
+ return chooseDep(dep.module.splitModules, mutatorName, sourceVariation, defaultVariationName)
+ }
+}
+
+func (c *Context) convertDepsToVariation(module *moduleInfo, depChooser depChooser) (errs []error) {
for i, dep := range module.directDeps {
if dep.module.logicModule == nil {
- var newDep *moduleInfo
- for _, m := range dep.module.splitModules {
- if m.moduleOrAliasVariant().variations[mutatorName] == variationName {
- newDep = m.moduleOrAliasTarget()
- break
- }
- }
- if newDep == nil && defaultVariationName != nil {
- // give it a second chance; match with defaultVariationName
- for _, m := range dep.module.splitModules {
- if m.moduleOrAliasVariant().variations[mutatorName] == *defaultVariationName {
- newDep = m.moduleOrAliasTarget()
- break
- }
- }
- }
+ newDep, missingVariation := depChooser(module, dep)
if newDep == nil {
errs = append(errs, &BlueprintError{
Err: fmt.Errorf("failed to find variation %q for module %q needed by %q",
- variationName, dep.module.Name(), module.Name()),
+ missingVariation, dep.module.Name(), module.Name()),
Pos: module.pos,
})
continue
@@ -2253,6 +2532,7 @@
Deps []jsonDep
Type string
Blueprint string
+ CreatedBy *string
Module map[string]interface{}
}
@@ -2282,6 +2562,18 @@
AddJSONData(d *map[string]interface{})
}
+// JSONAction contains the action-related info we expose to json module graph
+type JSONAction struct {
+ Inputs []string
+ Outputs []string
+}
+
+// JSONActionSupplier allows JSON representation of additional actions that are not registered in
+// Ninja
+type JSONActionSupplier interface {
+ JSONActions() []JSONAction
+}
+
func jsonModuleFromModuleInfo(m *moduleInfo) *JsonModule {
result := &JsonModule{
jsonModuleName: *jsonModuleNameFromModuleInfo(m),
@@ -2290,6 +2582,10 @@
Blueprint: m.relBlueprintsFile,
Module: make(map[string]interface{}),
}
+ if m.createdBy != nil {
+ n := m.createdBy.Name()
+ result.CreatedBy = &n
+ }
if j, ok := m.logicModule.(JSONDataSupplier); ok {
j.AddJSONData(&result.Module)
}
@@ -2311,17 +2607,27 @@
Blueprint: m.relBlueprintsFile,
Module: make(map[string]interface{}),
}
- var actions []map[string]interface{}
+ var actions []JSONAction
for _, bDef := range m.actionDefs.buildDefs {
- actions = append(actions, map[string]interface{}{
- "Inputs": append(
+ actions = append(actions, JSONAction{
+ Inputs: append(
getNinjaStringsWithNilPkgNames(bDef.Inputs),
getNinjaStringsWithNilPkgNames(bDef.Implicits)...),
- "Outputs": append(
+ Outputs: append(
getNinjaStringsWithNilPkgNames(bDef.Outputs),
getNinjaStringsWithNilPkgNames(bDef.ImplicitOutputs)...),
})
}
+
+ if j, ok := m.logicModule.(JSONActionSupplier); ok {
+ actions = append(actions, j.JSONActions()...)
+ }
+ for _, p := range m.providers {
+ if j, ok := p.(JSONActionSupplier); ok {
+ actions = append(actions, j.JSONActions()...)
+ }
+ }
+
result.Module["Actions"] = actions
return result
}
@@ -3619,7 +3925,7 @@
return ""
}
-// WriteBuildFile writes the Ninja manifeset text for the generated build
+// WriteBuildFile writes the Ninja manifest text for the generated build
// actions to w. If this is called before PrepareBuildActions successfully
// completes then ErrBuildActionsNotReady is returned.
func (c *Context) WriteBuildFile(w io.StringWriter) error {
@@ -4095,6 +4401,10 @@
c.EventHandler.End(name)
}
+func (c *Context) SetBeforePrepareBuildActionsHook(hookFn func() error) {
+ c.BeforePrepareBuildActionsHook = hookFn
+}
+
func (c *Context) writeLocalBuildActions(nw *ninjaWriter,
defs *localBuildActions) error {
diff --git a/metrics/event_handler.go b/metrics/event_handler.go
index c19d039..f6bcaf1 100644
--- a/metrics/event_handler.go
+++ b/metrics/event_handler.go
@@ -62,6 +62,13 @@
h.scopeStartTimes = append(h.scopeStartTimes, _now())
}
+// Do wraps a function with calls to Begin() and End().
+func (h *EventHandler) Do(name string, f func()) {
+ h.Begin(name)
+ defer h.End(name)
+ f()
+}
+
// End logs the end of an event. All events nested within this event must have
// themselves been marked completed.
func (h *EventHandler) End(name string) {
diff --git a/module_ctx.go b/module_ctx.go
index b7246c1..f2a04c8 100644
--- a/module_ctx.go
+++ b/module_ctx.go
@@ -991,11 +991,17 @@
}
func (mctx *mutatorContext) CreateVariations(variationNames ...string) []Module {
- return mctx.createVariations(variationNames, false)
+ depChooser := chooseDepInherit(mctx.name, mctx.defaultVariation)
+ return mctx.createVariations(variationNames, depChooser, false)
+}
+
+func (mctx *mutatorContext) createVariationsWithTransition(transition Transition, variationNames ...string) []Module {
+ return mctx.createVariations(variationNames, chooseDepByTransition(mctx.name, transition), false)
}
func (mctx *mutatorContext) CreateLocalVariations(variationNames ...string) []Module {
- return mctx.createVariations(variationNames, true)
+ depChooser := chooseDepInherit(mctx.name, mctx.defaultVariation)
+ return mctx.createVariations(variationNames, depChooser, true)
}
func (mctx *mutatorContext) SetVariationProvider(module Module, provider ProviderKey, value interface{}) {
@@ -1008,9 +1014,9 @@
panic(fmt.Errorf("module %q is not a newly created variant of %q", module, mctx.module))
}
-func (mctx *mutatorContext) createVariations(variationNames []string, local bool) []Module {
+func (mctx *mutatorContext) createVariations(variationNames []string, depChooser depChooser, local bool) []Module {
var ret []Module
- modules, errs := mctx.context.createVariations(mctx.module, mctx.name, mctx.defaultVariation, variationNames, local)
+ modules, errs := mctx.context.createVariations(mctx.module, mctx.name, depChooser, variationNames, local)
if len(errs) > 0 {
mctx.errs = append(mctx.errs, errs...)
}
@@ -1091,8 +1097,13 @@
panic(fmt.Errorf("no %q variation in module variations %q", targetVariationName, foundVariations))
}
+func (mctx *mutatorContext) applyTransition(transition Transition) {
+ mctx.context.convertDepsToVariation(mctx.module, chooseDepByTransition(mctx.name, transition))
+}
+
func (mctx *mutatorContext) SetDependencyVariation(variationName string) {
- mctx.context.convertDepsToVariation(mctx.module, mctx.name, variationName, nil)
+ mctx.context.convertDepsToVariation(mctx.module, chooseDepExplicit(
+ mctx.name, variationName, nil))
}
func (mctx *mutatorContext) SetDefaultDependencyVariation(variationName *string) {
diff --git a/proptools/escape.go b/proptools/escape.go
index 4ef10f0..9443a95 100644
--- a/proptools/escape.go
+++ b/proptools/escape.go
@@ -122,4 +122,8 @@
return ShellEscape(NinjaEscape(s))
}
+func NinjaAndShellEscapeIncludingSpaces(s string) string {
+ return ShellEscapeIncludingSpaces(NinjaEscape(s))
+}
+
var singleQuoteReplacer = strings.NewReplacer(`'`, `'\''`)
diff --git a/singleton_ctx.go b/singleton_ctx.go
index 455f6fc..d579b8e 100644
--- a/singleton_ctx.go
+++ b/singleton_ctx.go
@@ -157,6 +157,10 @@
// Fs returns a pathtools.Filesystem that can be used to interact with files. Using the Filesystem interface allows
// the singleton to be used in build system tests that run against a mock filesystem.
Fs() pathtools.FileSystem
+
+ // ModuleVariantsFromName returns the list of module variants named `name` in the same namespace as `referer`.
+ // Allows generating build actions for `referer` based on the metadata for `name` deferred until the singleton context.
+ ModuleVariantsFromName(referer Module, name string) []Module
}
var _ SingletonContext = (*singletonContext)(nil)
@@ -369,3 +373,26 @@
func (s *singletonContext) Fs() pathtools.FileSystem {
return s.context.fs
}
+
+func (s *singletonContext) ModuleVariantsFromName(referer Module, name string) []Module {
+ c := s.context
+
+ refererInfo := c.moduleInfo[referer]
+ if refererInfo == nil {
+ s.ModuleErrorf(referer, "could not find module %q", referer.Name())
+ return nil
+ }
+
+ moduleGroup, exists := c.nameInterface.ModuleFromName(name, refererInfo.namespace())
+ if !exists {
+ return nil
+ }
+ result := make([]Module, 0, len(moduleGroup.modules))
+ for _, module := range moduleGroup.modules {
+ moduleInfo := module.module()
+ if moduleInfo != nil {
+ result = append(result, moduleInfo.logicModule)
+ }
+ }
+ return result
+}
