Snap for 10453563 from bbbced136fa7dc2b3bc0df7c3b08f6206cb72356 to mainline-permission-release
Change-Id: I48c2530f330fc2fd318628d4f226cfc10a5c60f7
diff --git a/Android.bp b/Android.bp
index c84d04a..20fa495 100644
--- a/Android.bp
+++ b/Android.bp
@@ -39,6 +39,7 @@
pkgPath: "github.com/google/blueprint",
srcs: [
"context.go",
+ "levenshtein.go",
"glob.go",
"live_tracker.go",
"mangle.go",
@@ -54,6 +55,7 @@
],
testSrcs: [
"context_test.go",
+ "levenshtein_test.go",
"glob_test.go",
"module_ctx_test.go",
"ninja_strings_test.go",
@@ -184,8 +186,12 @@
blueprint_go_binary {
name: "bpmodify",
- deps: ["blueprint-parser"],
+ deps: [
+ "blueprint-parser",
+ "blueprint-proptools",
+ ],
srcs: ["bpmodify/bpmodify.go"],
+ testSrcs: ["bpmodify/bpmodify_test.go"],
}
blueprint_go_binary {
diff --git a/README.md b/README.md
index 961bc64..22525bf 100644
--- a/README.md
+++ b/README.md
@@ -1,25 +1,6 @@
Blueprint Build System
======================
-Blueprint is being archived on 2021 May 3.
+Blueprint is part of Soong.
-On 2021 May 3, we will be archiving the Blueprint project. This means it will
-not be possible to file new issues or open new pull requests for this GitHub
-project. As the project is being archived, patches -- including security
-patches -- will not be applied after May 3. The source tree will remain
-available, but changes to Blueprint in AOSP will not be merged here and
-Blueprint's source tree in AOSP will eventually stop being usable outside of
-Android.
-
-Whereas there are no meta-build systems one can use as a drop-in replacement for
-Blueprint, there are a number of build systems that can be used:
-
-* [Bazel](https://bazel.build), Google's multi-language build tool to build and
- test software of any size, quickly and reliably
-* [Soong](https://source.android.com/setup/build), for building the Android
- operating system itself
-* [CMake](https://cmake.org), an open-source, cross-platform family of tools
- designed to build, test and package software
-* [Buck](https://buck.build), a fast build system that encourages the creation
- of small, reusable modules over a variety of platforms and languages
-* The venerable [GNU Make](https://www.gnu.org/software/make/)
+For more information, see `build/soong/README.md` .
diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go
index ceeee19..bf12cd3 100644
--- a/bootstrap/bootstrap.go
+++ b/bootstrap/bootstrap.go
@@ -16,7 +16,6 @@
import (
"fmt"
- "go/build"
"path/filepath"
"runtime"
"strings"
@@ -33,21 +32,15 @@
pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join("$ToolDir", "loadplugins"))
parallelCompile = pctx.StaticVariable("parallelCompile", func() string {
- // Parallel compilation is only supported on >= go1.9
- for _, r := range build.Default.ReleaseTags {
- if r == "go1.9" {
- numCpu := runtime.NumCPU()
- // This will cause us to recompile all go programs if the
- // number of cpus changes. We don't get a lot of benefit from
- // higher values, so cap this to make it cheaper to move trees
- // between machines.
- if numCpu > 8 {
- numCpu = 8
- }
- return fmt.Sprintf("-c %d", numCpu)
- }
+ numCpu := runtime.NumCPU()
+ // This will cause us to recompile all go programs if the
+ // number of cpus changes. We don't get a lot of benefit from
+ // higher values, so cap this to make it cheaper to move trees
+ // between machines.
+ if numCpu > 8 {
+ numCpu = 8
}
- return ""
+ return fmt.Sprintf("-c %d", numCpu)
}())
compile = pctx.StaticRule("compile",
@@ -149,7 +142,7 @@
},
"depfile")
- _ = pctx.VariableFunc("ToolDir", func(config interface{}) (string, error) {
+ _ = pctx.VariableFunc("ToolDir", func(ctx blueprint.VariableFuncContext, config interface{}) (string, error) {
return config.(BootstrapConfig).HostToolDir(), nil
})
)
@@ -693,9 +686,10 @@
// Build the main build.ninja
ctx.Build(pctx, blueprint.BuildParams{
- Rule: generateBuildNinja,
- Outputs: i.Outputs,
- Inputs: i.Inputs,
+ Rule: generateBuildNinja,
+ Outputs: i.Outputs,
+ Inputs: i.Inputs,
+ OrderOnly: i.OrderOnlyInputs,
Args: map[string]string{
"builder": primaryBuilderFile,
"env": envAssignments,
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 8c045b4..3c41cce 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -18,7 +18,6 @@
"bufio"
"fmt"
"io"
- "io/ioutil"
"os"
"path/filepath"
"runtime"
@@ -41,9 +40,10 @@
TraceFile string
}
-// Returns the list of dependencies the emitted Ninja files has. These can be
-// written to the .d file for the output so that it is correctly rebuilt when
-// needed in case Blueprint is itself invoked from Ninja
+// RunBlueprint emits `args.OutFile` (a Ninja file) and returns the list of
+// its dependencies. These can be written to a `${args.OutFile}.d` file
+// so that it is correctly rebuilt when needed in case Blueprint is itself
+// invoked from Ninja
func RunBlueprint(args Args, stopBefore StopBefore, ctx *blueprint.Context, config interface{}) []string {
runtime.GOMAXPROCS(runtime.NumCPU())
@@ -71,53 +71,58 @@
defer trace.Stop()
}
- srcDir := "."
-
- ninjaDeps := make([]string, 0)
-
- if args.ModuleListFile != "" {
- ctx.SetModuleListFile(args.ModuleListFile)
- ninjaDeps = append(ninjaDeps, args.ModuleListFile)
- } else {
+ if args.ModuleListFile == "" {
fatalf("-l <moduleListFile> is required and must be nonempty")
}
+ ctx.SetModuleListFile(args.ModuleListFile)
+
+ var ninjaDeps []string
+ ninjaDeps = append(ninjaDeps, args.ModuleListFile)
+
ctx.BeginEvent("list_modules")
- filesToParse, err := ctx.ListModulePaths(srcDir)
- ctx.EndEvent("list_modules")
- if err != nil {
+ var filesToParse []string
+ if f, err := ctx.ListModulePaths("."); err != nil {
fatalf("could not enumerate files: %v\n", err.Error())
+ } else {
+ filesToParse = f
}
+ ctx.EndEvent("list_modules")
ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory())
ctx.RegisterModuleType("blueprint_go_binary", newGoBinaryModuleFactory())
ctx.RegisterSingletonType("bootstrap", newSingletonFactory())
+ blueprint.RegisterPackageIncludesModuleType(ctx)
ctx.BeginEvent("parse_bp")
- blueprintFiles, errs := ctx.ParseFileList(".", filesToParse, config)
- if len(errs) > 0 {
+ if blueprintFiles, errs := ctx.ParseFileList(".", filesToParse, config); len(errs) > 0 {
fatalErrors(errs)
+ } else {
+ ctx.EndEvent("parse_bp")
+ ninjaDeps = append(ninjaDeps, blueprintFiles...)
}
- ctx.EndEvent("parse_bp")
- // Add extra ninja file dependencies
- ninjaDeps = append(ninjaDeps, blueprintFiles...)
-
- extraDeps, errs := ctx.ResolveDependencies(config)
- if len(errs) > 0 {
+ if resolvedDeps, errs := ctx.ResolveDependencies(config); len(errs) > 0 {
fatalErrors(errs)
+ } else {
+ ninjaDeps = append(ninjaDeps, resolvedDeps...)
}
- ninjaDeps = append(ninjaDeps, extraDeps...)
if stopBefore == StopBeforePrepareBuildActions {
return ninjaDeps
}
- extraDeps, errs = ctx.PrepareBuildActions(config)
- if len(errs) > 0 {
- fatalErrors(errs)
+ if ctx.BeforePrepareBuildActionsHook != nil {
+ if err := ctx.BeforePrepareBuildActionsHook(); err != nil {
+ fatalErrors([]error{err})
+ }
}
- ninjaDeps = append(ninjaDeps, extraDeps...)
+
+ if buildActionsDeps, errs := ctx.PrepareBuildActions(config); len(errs) > 0 {
+ fatalErrors(errs)
+ } else {
+ ninjaDeps = append(ninjaDeps, buildActionsDeps...)
+ }
if stopBefore == StopBeforeWriteNinja {
return ninjaDeps
@@ -131,37 +136,32 @@
ctx.BeginEvent("write_files")
defer ctx.EndEvent("write_files")
if args.EmptyNinjaFile {
- if err := ioutil.WriteFile(joinPath(ctx.SrcDir(), args.OutFile), []byte(nil), outFilePermissions); err != nil {
+ if err := os.WriteFile(joinPath(ctx.SrcDir(), args.OutFile), []byte(nil), outFilePermissions); err != nil {
fatalf("error writing empty Ninja file: %s", err)
}
- }
-
- if !args.EmptyNinjaFile {
- f, err = os.OpenFile(joinPath(ctx.SrcDir(), args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
+ out = io.Discard.(io.StringWriter)
+ } else {
+ f, err := os.OpenFile(joinPath(ctx.SrcDir(), args.OutFile), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, outFilePermissions)
if err != nil {
fatalf("error opening Ninja file: %s", err)
}
+ defer f.Close()
buf = bufio.NewWriterSize(f, 16*1024*1024)
out = buf
- } else {
- out = ioutil.Discard.(io.StringWriter)
}
- err = ctx.WriteBuildFile(out)
- if err != nil {
+ if err := ctx.WriteBuildFile(out); err != nil {
fatalf("error writing Ninja file contents: %s", err)
}
if buf != nil {
- err = buf.Flush()
- if err != nil {
+ if err := buf.Flush(); err != nil {
fatalf("error flushing Ninja file contents: %s", err)
}
}
if f != nil {
- err = f.Close()
- if err != nil {
+ if err := f.Close(); err != nil {
fatalf("error closing Ninja file: %s", err)
}
}
diff --git a/bootstrap/config.go b/bootstrap/config.go
index 9972b5d..1d256ba 100644
--- a/bootstrap/config.go
+++ b/bootstrap/config.go
@@ -25,7 +25,7 @@
)
func bootstrapVariable(name string, value func(BootstrapConfig) string) blueprint.Variable {
- return pctx.VariableFunc(name, func(config interface{}) (string, error) {
+ return pctx.VariableFunc(name, func(ctx blueprint.VariableFuncContext, config interface{}) (string, error) {
c, ok := config.(BootstrapConfig)
if !ok {
panic(fmt.Sprintf("Bootstrap rules were passed a configuration that does not include theirs, config=%q",
@@ -105,10 +105,11 @@
)
type PrimaryBuilderInvocation struct {
- Inputs []string
- Outputs []string
- Args []string
- Console bool
- Description string
- Env map[string]string
+ Inputs []string
+ OrderOnlyInputs []string
+ Outputs []string
+ Args []string
+ Console bool
+ Description string
+ Env map[string]string
}
diff --git a/bootstrap/glob.go b/bootstrap/glob.go
index 70495dc..a766676 100644
--- a/bootstrap/glob.go
+++ b/bootstrap/glob.go
@@ -44,7 +44,7 @@
// in a build failure with a "missing and no known rule to make it" error.
var (
- _ = pctx.VariableFunc("globCmd", func(config interface{}) (string, error) {
+ _ = pctx.VariableFunc("globCmd", func(ctx blueprint.VariableFuncContext, config interface{}) (string, error) {
return filepath.Join(config.(BootstrapConfig).SoongOutDir(), "bpglob"), nil
})
@@ -237,6 +237,8 @@
return nil, errs
}
+ // PrepareBuildActions() will write $OUTDIR/soong/globs/$m/$i files
+ // where $m=bp2build|build and $i=0..numGlobBuckets
extraDeps, errs = ctx.PrepareBuildActions(config)
if len(extraDeps) > 0 {
return nil, []error{fmt.Errorf("shouldn't have extra deps")}
diff --git a/bpmodify/bpmodify.go b/bpmodify/bpmodify.go
index 431eb83..1df808e 100644
--- a/bpmodify/bpmodify.go
+++ b/bpmodify/bpmodify.go
@@ -2,7 +2,6 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
-
package main
import (
@@ -15,6 +14,7 @@
"os/exec"
"path/filepath"
"strings"
+ "syscall"
"unicode"
"github.com/google/blueprint/parser"
@@ -22,27 +22,34 @@
var (
// main operation modes
- list = flag.Bool("l", false, "list files that would be modified by bpmodify")
- write = flag.Bool("w", false, "write result to (source) file instead of stdout")
- doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
- sortLists = flag.Bool("s", false, "sort touched lists, even if they were unsorted")
- targetedModules = new(identSet)
- targetedProperty = new(qualifiedProperty)
- addIdents = new(identSet)
- removeIdents = new(identSet)
- removeProperty = flag.Bool("remove-property", false, "remove the property")
- setString *string
- addLiteral *string
+ list = flag.Bool("l", false, "list files that would be modified by bpmodify")
+ write = flag.Bool("w", false, "write result to (source) file instead of stdout")
+ doDiff = flag.Bool("d", false, "display diffs instead of rewriting files")
+ sortLists = flag.Bool("s", false, "sort touched lists, even if they were unsorted")
+ targetedModules = new(identSet)
+ targetedProperties = new(qualifiedProperties)
+ addIdents = new(identSet)
+ removeIdents = new(identSet)
+ removeProperty = flag.Bool("remove-property", false, "remove the property")
+ moveProperty = flag.Bool("move-property", false, "moves contents of property into newLocation")
+ newLocation string
+ setString *string
+ addLiteral *string
+ setBool *string
+ replaceProperty = new(replacements)
)
func init() {
flag.Var(targetedModules, "m", "comma or whitespace separated list of modules on which to operate")
- flag.Var(targetedProperty, "parameter", "alias to -property=`name`")
- flag.Var(targetedProperty, "property", "fully qualified `name` of property to modify (default \"deps\")")
+ flag.Var(targetedProperties, "parameter", "alias to -property=`name1[,name2[,... […]")
+ flag.StringVar(&newLocation, "new-location", "", " use with moveProperty to move contents of -property into a property with name -new-location ")
+ flag.Var(targetedProperties, "property", "comma-separated list of fully qualified `name`s of properties to modify (default \"deps\")")
flag.Var(addIdents, "a", "comma or whitespace separated list of identifiers to add")
- flag.Var(stringPtrFlag{&addLiteral}, "add-literal", "a literal to add")
+ flag.Var(stringPtrFlag{&addLiteral}, "add-literal", "a literal to add to a list")
flag.Var(removeIdents, "r", "comma or whitespace separated list of identifiers to remove")
flag.Var(stringPtrFlag{&setString}, "str", "set a string property")
+ flag.Var(replaceProperty, "replace-property", "property names to be replaced, in the form of oldName1=newName1,oldName2=newName2")
+ flag.Var(stringPtrFlag{&setBool}, "set-bool", "a boolean value to set a property with (not a list)")
flag.Usage = usage
}
@@ -68,16 +75,16 @@
return err
}
defer f.Close()
+ if *write {
+ syscall.Flock(int(f.Fd()), syscall.LOCK_EX)
+ }
in = f
}
-
src, err := ioutil.ReadAll(in)
if err != nil {
return err
}
-
r := bytes.NewBuffer(src)
-
file, errs := parser.Parse(filename, r, parser.NewScope(nil))
if len(errs) > 0 {
for _, err := range errs {
@@ -85,7 +92,6 @@
}
return fmt.Errorf("%d parsing errors", len(errs))
}
-
modified, errs := findModules(file)
if len(errs) > 0 {
for _, err := range errs {
@@ -93,13 +99,11 @@
}
fmt.Fprintln(os.Stderr, "continuing...")
}
-
if modified {
res, err := parser.Print(file)
if err != nil {
return err
}
-
if *list {
fmt.Fprintln(out, filename)
}
@@ -117,23 +121,19 @@
fmt.Printf("diff %s bpfmt/%s\n", filename, filename)
out.Write(data)
}
-
if !*list && !*write && !*doDiff {
_, err = out.Write(res)
}
}
-
return err
}
-
func findModules(file *parser.File) (modified bool, errs []error) {
-
for _, def := range file.Defs {
if module, ok := def.(*parser.Module); ok {
for _, prop := range module.Properties {
- if prop.Name == "name" && prop.Value.Type() == parser.StringType {
- if targetedModule(prop.Value.Eval().(*parser.String).Value) {
- m, newErrs := processModule(module, prop.Name, file)
+ if prop.Name == "name" && prop.Value.Type() == parser.StringType && targetedModule(prop.Value.Eval().(*parser.String).Value) {
+ for _, p := range targetedProperties.properties {
+ m, newErrs := processModuleProperty(module, prop.Name, file, p)
errs = append(errs, newErrs...)
modified = modified || m
}
@@ -141,23 +141,25 @@
}
}
}
-
return modified, errs
}
-func processModule(module *parser.Module, moduleName string,
- file *parser.File) (modified bool, errs []error) {
- prop, parent, err := getRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes())
+func processModuleProperty(module *parser.Module, moduleName string,
+ file *parser.File, property qualifiedProperty) (modified bool, errs []error) {
+ prop, parent, err := getRecursiveProperty(module, property.name(), property.prefixes())
if err != nil {
return false, []error{err}
}
if prop == nil {
if len(addIdents.idents) > 0 || addLiteral != nil {
// We are adding something to a non-existing list prop, so we need to create it first.
- prop, modified, err = createRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes(), &parser.List{})
+ prop, modified, err = createRecursiveProperty(module, property.name(), property.prefixes(), &parser.List{})
} else if setString != nil {
// We setting a non-existent string property, so we need to create it first.
- prop, modified, err = createRecursiveProperty(module, targetedProperty.name(), targetedProperty.prefixes(), &parser.String{})
+ prop, modified, err = createRecursiveProperty(module, property.name(), property.prefixes(), &parser.String{})
+ } else if setBool != nil {
+ // We are setting a non-existent property, so we need to create it first.
+ prop, modified, err = createRecursiveProperty(module, property.name(), property.prefixes(), &parser.Bool{})
} else {
// We cannot find an existing prop, and we aren't adding anything to the prop,
// which means we must be removing something from a non-existing prop,
@@ -171,23 +173,22 @@
} else if *removeProperty {
// remove-property is used solely, so return here.
return parent.RemoveProperty(prop.Name), nil
+ } else if *moveProperty {
+ return parent.MovePropertyContents(prop.Name, newLocation), nil
}
- m, errs := processParameter(prop.Value, targetedProperty.String(), moduleName, file)
+ m, errs := processParameter(prop.Value, property.String(), moduleName, file)
modified = modified || m
return modified, errs
}
-
func getRecursiveProperty(module *parser.Module, name string, prefixes []string) (prop *parser.Property, parent *parser.Map, err error) {
prop, parent, _, err = getOrCreateRecursiveProperty(module, name, prefixes, nil)
return prop, parent, err
}
-
func createRecursiveProperty(module *parser.Module, name string, prefixes []string,
empty parser.Expression) (prop *parser.Property, modified bool, err error) {
prop, _, modified, err = getOrCreateRecursiveProperty(module, name, prefixes, empty)
return prop, modified, err
}
-
func getOrCreateRecursiveProperty(module *parser.Module, name string, prefixes []string,
empty parser.Expression) (prop *parser.Property, parent *parser.Map, modified bool, err error) {
m := &module.Map
@@ -223,38 +224,48 @@
return nil, nil, false, nil
}
}
-
func processParameter(value parser.Expression, paramName, moduleName string,
file *parser.File) (modified bool, errs []error) {
if _, ok := value.(*parser.Variable); ok {
return false, []error{fmt.Errorf("parameter %s in module %s is a variable, unsupported",
paramName, moduleName)}
}
-
if _, ok := value.(*parser.Operator); ok {
return false, []error{fmt.Errorf("parameter %s in module %s is an expression, unsupported",
paramName, moduleName)}
}
+ if (*replaceProperty).size() != 0 {
+ if list, ok := value.Eval().(*parser.List); ok {
+ return parser.ReplaceStringsInList(list, (*replaceProperty).oldNameToNewName), nil
+ } else if str, ok := value.Eval().(*parser.String); ok {
+ oldVal := str.Value
+ replacementValue := (*replaceProperty).oldNameToNewName[oldVal]
+ if replacementValue != "" {
+ str.Value = replacementValue
+ return true, nil
+ } else {
+ return false, nil
+ }
+ }
+ return false, []error{fmt.Errorf("expected parameter %s in module %s to be a list or string, found %s",
+ paramName, moduleName, value.Type().String())}
+ }
if len(addIdents.idents) > 0 || len(removeIdents.idents) > 0 {
list, ok := value.(*parser.List)
if !ok {
return false, []error{fmt.Errorf("expected parameter %s in module %s to be list, found %s",
- paramName, moduleName, value.Type().String())}
+ paramName, moduleName, value.Type())}
}
-
wasSorted := parser.ListIsSorted(list)
-
for _, a := range addIdents.idents {
m := parser.AddStringToList(list, a)
modified = modified || m
}
-
for _, r := range removeIdents.idents {
m := parser.RemoveStringFromList(list, r)
modified = modified || m
}
-
if (wasSorted || *sortLists) && modified {
parser.SortList(file, list)
}
@@ -273,20 +284,32 @@
}
list.Values = append(list.Values, value)
modified = true
+ } else if setBool != nil {
+ res, ok := value.(*parser.Bool)
+ if !ok {
+ return false, []error{fmt.Errorf("expected parameter %s in module %s to be bool, found %s",
+ paramName, moduleName, value.Type().String())}
+ }
+ if *setBool == "true" {
+ res.Value = true
+ } else if *setBool == "false" {
+ res.Value = false
+ } else {
+ return false, []error{fmt.Errorf("expected parameter %s to be true or false, found %s",
+ paramName, *setBool)}
+ }
+ modified = true
} else if setString != nil {
str, ok := value.(*parser.String)
if !ok {
return false, []error{fmt.Errorf("expected parameter %s in module %s to be string, found %s",
paramName, moduleName, value.Type().String())}
}
-
str.Value = *setString
modified = true
}
-
return modified, nil
}
-
func targetedModule(name string) bool {
if targetedModules.all {
return true
@@ -296,12 +319,11 @@
return true
}
}
-
return false
}
-
func visitFile(path string, f os.FileInfo, err error) error {
- if err == nil && f.Name() == "Blueprints" {
+ //TODO(dacek): figure out a better way to target intended .bp files without parsing errors
+ if err == nil && (f.Name() == "Blueprints" || strings.HasSuffix(f.Name(), ".bp")) {
err = processFile(path, nil, os.Stdout)
}
if err != nil {
@@ -309,11 +331,9 @@
}
return nil
}
-
func walkDir(path string) {
filepath.Walk(path, visitFile)
}
-
func main() {
defer func() {
if err := recover(); err != nil {
@@ -321,13 +341,16 @@
}
os.Exit(exitCode)
}()
-
flag.Parse()
- if len(targetedProperty.parts) == 0 {
- targetedProperty.Set("deps")
+ if len(targetedProperties.properties) == 0 && *moveProperty {
+ report(fmt.Errorf("-move-property must specify property"))
+ return
}
+ if len(targetedProperties.properties) == 0 {
+ targetedProperties.Set("deps")
+ }
if flag.NArg() == 0 {
if *write {
report(fmt.Errorf("error: cannot use -w with standard input"))
@@ -338,22 +361,27 @@
}
return
}
-
if len(targetedModules.idents) == 0 {
report(fmt.Errorf("-m parameter is required"))
return
}
- if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 && setString == nil && addLiteral == nil && !*removeProperty {
- report(fmt.Errorf("-a, -add-literal, -r, -remove-property or -str parameter is required"))
+ if len(addIdents.idents) == 0 && len(removeIdents.idents) == 0 && setString == nil && addLiteral == nil && !*removeProperty && !*moveProperty && (*replaceProperty).size() == 0 && setBool == nil {
+ report(fmt.Errorf("-a, -add-literal, -r, -remove-property, -move-property, replace-property or -str parameter is required"))
return
}
-
- if *removeProperty && (len(addIdents.idents) > 0 || len(removeIdents.idents) > 0 || setString != nil || addLiteral != nil) {
+ if *removeProperty && (len(addIdents.idents) > 0 || len(removeIdents.idents) > 0 || setString != nil || addLiteral != nil || (*replaceProperty).size() > 0) {
report(fmt.Errorf("-remove-property cannot be used with other parameter(s)"))
return
}
-
+ if *moveProperty && (len(addIdents.idents) > 0 || len(removeIdents.idents) > 0 || setString != nil || addLiteral != nil || (*replaceProperty).size() > 0) {
+ report(fmt.Errorf("-move-property cannot be used with other parameter(s)"))
+ return
+ }
+ if *moveProperty && newLocation == "" {
+ report(fmt.Errorf("-move-property must specify -new-location"))
+ return
+ }
for i := 0; i < flag.NArg(); i++ {
path := flag.Arg(i)
switch dir, err := os.Stat(path); {
@@ -376,17 +404,14 @@
}
defer os.Remove(f1.Name())
defer f1.Close()
-
f2, err := ioutil.TempFile("", "bpfmt")
if err != nil {
return
}
defer os.Remove(f2.Name())
defer f2.Close()
-
f1.Write(b1)
f2.Write(b2)
-
data, err = exec.Command("diff", "-uw", f1.Name(), f2.Name()).CombinedOutput()
if len(data) > 0 {
// diff exits with a non-zero status when the files don't match.
@@ -394,7 +419,6 @@
err = nil
}
return
-
}
type stringPtrFlag struct {
@@ -405,7 +429,6 @@
*f.s = &s
return nil
}
-
func (f stringPtrFlag) String() string {
if f.s == nil || *f.s == nil {
return ""
@@ -413,6 +436,59 @@
return **f.s
}
+type replacements struct {
+ oldNameToNewName map[string]string
+}
+
+func (m *replacements) String() string {
+ ret := ""
+ sep := ""
+ for k, v := range m.oldNameToNewName {
+ ret += sep
+ ret += k
+ ret += ":"
+ ret += v
+ sep = ","
+ }
+ return ret
+}
+
+func (m *replacements) Set(s string) error {
+ usedNames := make(map[string]struct{})
+
+ pairs := strings.Split(s, ",")
+ length := len(pairs)
+ m.oldNameToNewName = make(map[string]string)
+ for i := 0; i < length; i++ {
+
+ pair := strings.SplitN(pairs[i], "=", 2)
+ if len(pair) != 2 {
+ return fmt.Errorf("Invalid replacement pair %s", pairs[i])
+ }
+ oldName := pair[0]
+ newName := pair[1]
+ if _, seen := usedNames[oldName]; seen {
+ return fmt.Errorf("Duplicated replacement name %s", oldName)
+ }
+ if _, seen := usedNames[newName]; seen {
+ return fmt.Errorf("Duplicated replacement name %s", newName)
+ }
+ usedNames[oldName] = struct{}{}
+ usedNames[newName] = struct{}{}
+ m.oldNameToNewName[oldName] = newName
+ }
+ return nil
+}
+
+func (m *replacements) Get() interface{} {
+ //TODO(dacek): Remove Get() method from interface as it seems unused.
+ return m.oldNameToNewName
+}
+
+func (m *replacements) size() (length int) {
+ return len(m.oldNameToNewName)
+}
+
type identSet struct {
idents []string
all bool
@@ -421,7 +497,6 @@
func (m *identSet) String() string {
return strings.Join(m.idents, ",")
}
-
func (m *identSet) Set(s string) error {
m.idents = strings.FieldsFunc(s, func(c rune) bool {
return unicode.IsSpace(c) || c == ','
@@ -431,42 +506,70 @@
}
return nil
}
-
func (m *identSet) Get() interface{} {
return m.idents
}
+type qualifiedProperties struct {
+ properties []qualifiedProperty
+}
+
type qualifiedProperty struct {
parts []string
}
-var _ flag.Getter = (*qualifiedProperty)(nil)
+var _ flag.Getter = (*qualifiedProperties)(nil)
func (p *qualifiedProperty) name() string {
return p.parts[len(p.parts)-1]
}
-
func (p *qualifiedProperty) prefixes() []string {
return p.parts[:len(p.parts)-1]
}
-
func (p *qualifiedProperty) String() string {
return strings.Join(p.parts, ".")
}
-func (p *qualifiedProperty) Set(s string) error {
- p.parts = strings.Split(s, ".")
- if len(p.parts) == 0 {
+func parseQualifiedProperty(s string) (*qualifiedProperty, error) {
+ parts := strings.Split(s, ".")
+ if len(parts) == 0 {
+ return nil, fmt.Errorf("%q is not a valid property name", s)
+ }
+ for _, part := range parts {
+ if part == "" {
+ return nil, fmt.Errorf("%q is not a valid property name", s)
+ }
+ }
+ prop := qualifiedProperty{parts}
+ return &prop, nil
+
+}
+
+func (p *qualifiedProperties) Set(s string) error {
+ properties := strings.Split(s, ",")
+ if len(properties) == 0 {
return fmt.Errorf("%q is not a valid property name", s)
}
- for _, part := range p.parts {
- if part == "" {
- return fmt.Errorf("%q is not a valid property name", s)
+
+ p.properties = make([]qualifiedProperty, len(properties))
+ for i := 0; i < len(properties); i++ {
+ tmp, err := parseQualifiedProperty(properties[i])
+ if err != nil {
+ return err
}
+ p.properties[i] = *tmp
}
return nil
}
-func (p *qualifiedProperty) Get() interface{} {
- return p.parts
+func (p *qualifiedProperties) String() string {
+ arrayLength := len(p.properties)
+ props := make([]string, arrayLength)
+ for i := 0; i < len(p.properties); i++ {
+ props[i] = p.properties[i].String()
+ }
+ return strings.Join(props, ",")
+}
+func (p *qualifiedProperties) Get() interface{} {
+ return p.properties
}
diff --git a/bpmodify/bpmodify_test.go b/bpmodify/bpmodify_test.go
index 4340edb..7bd8b57 100644
--- a/bpmodify/bpmodify_test.go
+++ b/bpmodify/bpmodify_test.go
@@ -4,14 +4,13 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
-
package main
import (
@@ -23,15 +22,19 @@
)
var testCases = []struct {
- name string
- input string
- output string
- property string
- addSet string
- removeSet string
- addLiteral *string
- setString *string
- removeProperty bool
+ name string
+ input string
+ output string
+ property string
+ addSet string
+ removeSet string
+ addLiteral *string
+ setString *string
+ setBool *string
+ removeProperty bool
+ replaceProperty string
+ moveProperty bool
+ newLocation string
}{
{
name: "add",
@@ -306,6 +309,39 @@
setString: proptools.StringPtr("bar"),
},
{
+ name: "set bool",
+ input: `
+ cc_foo {
+ name: "foo",
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ foo: true,
+ }
+ `,
+ property: "foo",
+ setBool: proptools.StringPtr("true"),
+ },
+ {
+ name: "set existing bool",
+ input: `
+ cc_foo {
+ name: "foo",
+ foo: true,
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ foo: false,
+ }
+ `,
+ property: "foo",
+ setBool: proptools.StringPtr("false"),
+ },
+ {
name: "remove existing property",
input: `
cc_foo {
@@ -354,6 +390,147 @@
`,
property: "bar",
removeProperty: true,
+ }, {
+ name: "replace property",
+ property: "deps",
+ input: `
+ cc_foo {
+ name: "foo",
+ deps: ["baz", "unchanged"],
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ deps: [
+ "baz_lib",
+ "unchanged",
+ ],
+ }
+ `,
+ replaceProperty: "baz=baz_lib,foobar=foobar_lib",
+ }, {
+ name: "replace property multiple modules",
+ property: "deps,required",
+ input: `
+ cc_foo {
+ name: "foo",
+ deps: ["baz", "unchanged"],
+ unchanged: ["baz"],
+ required: ["foobar"],
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ deps: [
+ "baz_lib",
+ "unchanged",
+ ],
+ unchanged: ["baz"],
+ required: ["foobar_lib"],
+ }
+ `,
+ replaceProperty: "baz=baz_lib,foobar=foobar_lib",
+ }, {
+ name: "replace property string value",
+ property: "name",
+ input: `
+ cc_foo {
+ name: "foo",
+ deps: ["baz"],
+ unchanged: ["baz"],
+ required: ["foobar"],
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo_lib",
+ deps: ["baz"],
+ unchanged: ["baz"],
+ required: ["foobar"],
+ }
+ `,
+ replaceProperty: "foo=foo_lib",
+ }, {
+ name: "replace property string and list values",
+ property: "name,deps",
+ input: `
+ cc_foo {
+ name: "foo",
+ deps: ["baz"],
+ unchanged: ["baz"],
+ required: ["foobar"],
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo_lib",
+ deps: ["baz_lib"],
+ unchanged: ["baz"],
+ required: ["foobar"],
+ }
+ `,
+ replaceProperty: "foo=foo_lib,baz=baz_lib",
+ }, {
+ name: "move contents of property into non-existing property",
+ input: `
+ cc_foo {
+ name: "foo",
+ bar: ["barContents"],
+ }
+`,
+ output: `
+ cc_foo {
+ name: "foo",
+ baz: ["barContents"],
+ }
+ `,
+ property: "bar",
+ moveProperty: true,
+ newLocation: "baz",
+ }, {
+ name: "move contents of property into existing property",
+ input: `
+ cc_foo {
+ name: "foo",
+ baz: ["bazContents"],
+ bar: ["barContents"],
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ baz: [
+ "bazContents",
+ "barContents",
+ ],
+
+ }
+ `,
+ property: "bar",
+ moveProperty: true,
+ newLocation: "baz",
+ }, {
+ name: "replace nested",
+ input: `
+ cc_foo {
+ name: "foo",
+ foo: {
+ bar: "baz",
+ },
+ }
+ `,
+ output: `
+ cc_foo {
+ name: "foo",
+ foo: {
+ bar: "baz2",
+ },
+ }
+ `,
+ property: "foo.bar",
+ replaceProperty: "baz=baz2",
},
}
@@ -364,16 +541,19 @@
}
return result
}
-
func TestProcessModule(t *testing.T) {
for i, testCase := range testCases {
t.Run(testCase.name, func(t *testing.T) {
- targetedProperty.Set(testCase.property)
+ targetedProperties.Set(testCase.property)
addIdents.Set(testCase.addSet)
removeIdents.Set(testCase.removeSet)
removeProperty = &testCase.removeProperty
+ moveProperty = &testCase.moveProperty
+ newLocation = testCase.newLocation
setString = testCase.setString
+ setBool = testCase.setBool
addLiteral = testCase.addLiteral
+ replaceProperty.Set(testCase.replaceProperty)
inAst, errs := parser.ParseAndEval("", strings.NewReader(testCase.input), parser.NewScope(nil))
if len(errs) > 0 {
@@ -384,16 +564,18 @@
t.Errorf("%+v", testCase)
t.FailNow()
}
-
if inModule, ok := inAst.Defs[0].(*parser.Module); !ok {
t.Fatalf(" input must only contain a single module definition: %s", testCase.input)
} else {
- _, errs := processModule(inModule, "", inAst)
- if len(errs) > 0 {
- t.Errorf("test case %d:", i)
- for _, err := range errs {
- t.Errorf(" %s", err)
+ for _, p := range targetedProperties.properties {
+ _, errs := processModuleProperty(inModule, "", inAst, p)
+ if len(errs) > 0 {
+ t.Errorf("test case %d:", i)
+ for _, err := range errs {
+ t.Errorf(" %s", err)
+ }
}
+
}
inModuleText, _ := parser.Print(inAst)
inModuleString := string(inModuleText)
@@ -407,5 +589,46 @@
}
})
}
+}
+func TestReplacementsCycleError(t *testing.T) {
+ cycleString := "old1=new1,new1=old1"
+ err := replaceProperty.Set(cycleString)
+
+ if err.Error() != "Duplicated replacement name new1" {
+ t.Errorf("Error message did not match")
+ t.Errorf("Expected ")
+ t.Errorf(" Duplicated replacement name new1")
+ t.Errorf("actual error:")
+ t.Errorf(" %s", err.Error())
+ t.FailNow()
+ }
+}
+
+func TestReplacementsDuplicatedError(t *testing.T) {
+ cycleString := "a=b,a=c"
+ err := replaceProperty.Set(cycleString)
+
+ if err.Error() != "Duplicated replacement name a" {
+ t.Errorf("Error message did not match")
+ t.Errorf("Expected ")
+ t.Errorf(" Duplicated replacement name a")
+ t.Errorf("actual error:")
+ t.Errorf(" %s", err.Error())
+ t.FailNow()
+ }
+}
+
+func TestReplacementsMultipleReplacedToSame(t *testing.T) {
+ cycleString := "a=c,d=c"
+ err := replaceProperty.Set(cycleString)
+
+ if err.Error() != "Duplicated replacement name c" {
+ t.Errorf("Error message did not match")
+ t.Errorf("Expected ")
+ t.Errorf(" Duplicated replacement name c")
+ t.Errorf("actual error:")
+ t.Errorf(" %s", err.Error())
+ t.FailNow()
+ }
}
diff --git a/bpmodify/rename_module_and_deps.py b/bpmodify/rename_module_and_deps.py
new file mode 100755
index 0000000..b1e6db9
--- /dev/null
+++ b/bpmodify/rename_module_and_deps.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2022 The Android Open Source Project
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""A tool to run bpmodify for a given module in order to rename it and all references to it"""
+import os
+import subprocess
+import sys
+
+
+def main():
+ if len(sys.argv) < 2:
+ print("Usage: rename_module_and_deps <pathToModule1,pathToModule2,...>")
+ return
+
+ modulePaths = sys.argv[1].split(",")
+ replacementsList = []
+ colonReplacementsList = []
+
+ for modulePath in modulePaths:
+ moduleName = modulePath.split("/")[-1]
+ replacementsList.append(moduleName + "=" + moduleName + "_lib")
+ # add in the colon replacement
+ colonReplaceString = ":" + moduleName + "=" + ":" + moduleName + "_lib"
+ replacementsList.append(colonReplaceString)
+ colonReplacementsList.append(colonReplaceString)
+
+ replacementsString = ",".join(replacementsList)
+ colonReplacementsString = ",".join(colonReplacementsList)
+ buildTop = os.getenv("ANDROID_BUILD_TOP")
+
+ if not buildTop:
+ raise Exception(
+ "$ANDROID_BUILD_TOP not found in environment. Have you run lunch?")
+
+ rename_deps_cmd = f"{buildTop}/prebuilts/go/linux-x86/bin/go run bpmodify.go -w -m=* -property=static_libs,deps,required,test_suites,name,host,libs,data_bins,data_native_bins,tools,shared_libs,file_contexts,target.not_windows.required,target.android.required,target.platform.required -replace-property={replacementsString} {buildTop}"
+ print(rename_deps_cmd)
+ subprocess.check_output(rename_deps_cmd, shell=True)
+
+ # Some properties (for example, data ), refer to files. Such properties may also refer to a filegroup module by prefixing it with a colon. Replacing these module references must thus be done separately.
+ colon_rename_deps_cmd = f"{buildTop}/prebuilts/go/linux-x86/bin/go run bpmodify.go -w -m=* -property=data -replace-property={colonReplacementsString} {buildTop}"
+ print(colon_rename_deps_cmd)
+ subprocess.check_output(colon_rename_deps_cmd, shell=True)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/context.go b/context.go
index 6496948..17daa8a 100644
--- a/context.go
+++ b/context.go
@@ -17,6 +17,8 @@
import (
"bytes"
"context"
+ "crypto/sha256"
+ "encoding/base64"
"encoding/json"
"errors"
"fmt"
@@ -50,15 +52,15 @@
// through a series of four phases. Each phase corresponds with a some methods
// on the Context object
//
-// Phase Methods
-// ------------ -------------------------------------------
-// 1. Registration RegisterModuleType, RegisterSingletonType
+// Phase Methods
+// ------------ -------------------------------------------
+// 1. Registration RegisterModuleType, RegisterSingletonType
//
-// 2. Parse ParseBlueprintsFiles, Parse
+// 2. Parse ParseBlueprintsFiles, Parse
//
-// 3. Generate ResolveDependencies, PrepareBuildActions
+// 3. Generate ResolveDependencies, PrepareBuildActions
//
-// 4. Write WriteBuildFile
+// 4. Write WriteBuildFile
//
// The registration phase prepares the context to process Blueprints files
// containing various types of modules. The parse phase reads in one or more
@@ -75,6 +77,8 @@
// Used for metrics-related event logging.
EventHandler *metrics.EventHandler
+ BeforePrepareBuildActionsHook func() error
+
moduleFactories map[string]ModuleFactory
nameInterface NameInterface
moduleGroups []*moduleGroup
@@ -135,6 +139,70 @@
// Can be set by tests to avoid invalidating Module values after mutators.
skipCloneModulesAfterMutators bool
+
+ // String values that can be used to gate build graph traversal
+ includeTags *IncludeTags
+
+ sourceRootDirs *SourceRootDirs
+}
+
+// A container for String keys. The keys can be used to gate build graph traversal
+type SourceRootDirs struct {
+ dirs []string
+}
+
+func (dirs *SourceRootDirs) Add(names ...string) {
+ dirs.dirs = append(dirs.dirs, names...)
+}
+
+func (dirs *SourceRootDirs) SourceRootDirAllowed(path string) (bool, string) {
+ sort.Slice(dirs.dirs, func(i, j int) bool {
+ return len(dirs.dirs[i]) < len(dirs.dirs[j])
+ })
+ last := len(dirs.dirs)
+ for i := range dirs.dirs {
+ // iterate from longest paths (most specific)
+ prefix := dirs.dirs[last-i-1]
+ disallowedPrefix := false
+ if len(prefix) >= 1 && prefix[0] == '-' {
+ prefix = prefix[1:]
+ disallowedPrefix = true
+ }
+ if strings.HasPrefix(path, prefix) {
+ if disallowedPrefix {
+ return false, prefix
+ } else {
+ return true, prefix
+ }
+ }
+ }
+ return true, ""
+}
+
+func (c *Context) AddSourceRootDirs(dirs ...string) {
+ c.sourceRootDirs.Add(dirs...)
+}
+
+// A container for String keys. The keys can be used to gate build graph traversal
+type IncludeTags map[string]bool
+
+func (tags *IncludeTags) Add(names ...string) {
+ for _, name := range names {
+ (*tags)[name] = true
+ }
+}
+
+func (tags *IncludeTags) Contains(tag string) bool {
+ _, exists := (*tags)[tag]
+ return exists
+}
+
+func (c *Context) AddIncludeTags(names ...string) {
+ c.includeTags.Add(names...)
+}
+
+func (c *Context) ContainsIncludeTag(name string) bool {
+ return c.includeTags.Contains(name)
}
// An Error describes a problem that was encountered that is related to a
@@ -270,6 +338,11 @@
// set during each runMutator
splitModules modulesOrAliases
+ // Used by TransitionMutator implementations
+ transitionVariations []string
+ currentTransitionMutator string
+ requiredVariationsLock sync.Mutex
+
// set during PrepareBuildActions
actionDefs localBuildActions
@@ -392,6 +465,8 @@
globs: make(map[globKey]pathtools.GlobResult),
fs: pathtools.OsFs,
finishedMutators: make(map[*mutatorInfo]bool),
+ includeTags: &IncludeTags{},
+ sourceRootDirs: &SourceRootDirs{},
outDir: nil,
requiredNinjaMajor: 1,
requiredNinjaMinor: 7,
@@ -448,38 +523,38 @@
//
// As an example, the follow code:
//
-// type myModule struct {
-// properties struct {
-// Foo string
-// Bar []string
-// }
-// }
+// type myModule struct {
+// properties struct {
+// Foo string
+// Bar []string
+// }
+// }
//
-// func NewMyModule() (blueprint.Module, []interface{}) {
-// module := new(myModule)
-// properties := &module.properties
-// return module, []interface{}{properties}
-// }
+// func NewMyModule() (blueprint.Module, []interface{}) {
+// module := new(myModule)
+// properties := &module.properties
+// return module, []interface{}{properties}
+// }
//
-// func main() {
-// ctx := blueprint.NewContext()
-// ctx.RegisterModuleType("my_module", NewMyModule)
-// // ...
-// }
+// func main() {
+// ctx := blueprint.NewContext()
+// ctx.RegisterModuleType("my_module", NewMyModule)
+// // ...
+// }
//
// would support parsing a module defined in a Blueprints file as follows:
//
-// my_module {
-// name: "myName",
-// foo: "my foo string",
-// bar: ["my", "bar", "strings"],
-// }
+// my_module {
+// name: "myName",
+// foo: "my foo string",
+// bar: ["my", "bar", "strings"],
+// }
//
// The factory function may be called from multiple goroutines. Any accesses
// to global variables must be synchronized.
func (c *Context) RegisterModuleType(name string, factory ModuleFactory) {
if _, present := c.moduleFactories[name]; present {
- panic(errors.New("module type name is already registered"))
+ panic(fmt.Errorf("module type %q is already registered", name))
}
c.moduleFactories[name] = factory
}
@@ -500,7 +575,7 @@
func (c *Context) RegisterSingletonType(name string, factory SingletonFactory) {
for _, s := range c.singletonInfo {
if s.name == name {
- panic(errors.New("singleton name is already registered"))
+ panic(fmt.Errorf("singleton %q is already registered", name))
}
}
@@ -522,7 +597,7 @@
func (c *Context) RegisterPreSingletonType(name string, factory SingletonFactory) {
for _, s := range c.preSingletonInfo {
if s.name == name {
- panic(errors.New("presingleton name is already registered"))
+ panic(fmt.Errorf("presingleton %q is already registered", name))
}
}
@@ -575,7 +650,7 @@
func (c *Context) RegisterTopDownMutator(name string, mutator TopDownMutator) MutatorHandle {
for _, m := range c.mutatorInfo {
if m.name == name && m.topDownMutator != nil {
- panic(fmt.Errorf("mutator name %s is already registered", name))
+ panic(fmt.Errorf("mutator %q is already registered", name))
}
}
@@ -602,7 +677,7 @@
func (c *Context) RegisterBottomUpMutator(name string, mutator BottomUpMutator) MutatorHandle {
for _, m := range c.variantMutatorNames {
if m == name {
- panic(fmt.Errorf("mutator name %s is already registered", name))
+ panic(fmt.Errorf("mutator %q is already registered", name))
}
}
@@ -617,6 +692,230 @@
return info
}
+type IncomingTransitionContext interface {
+ // Module returns the target of the dependency edge for which the transition
+ // is being computed
+ Module() Module
+
+ // Config returns the config object that was passed to
+ // Context.PrepareBuildActions.
+ Config() interface{}
+}
+
+type OutgoingTransitionContext interface {
+ // Module returns the target of the dependency edge for which the transition
+ // is being computed
+ Module() Module
+
+ // DepTag() Returns the dependency tag through which this dependency is
+ // reached
+ DepTag() DependencyTag
+}
+
+// Transition mutators implement a top-down mechanism where a module tells its
+// direct dependencies what variation they should be built in but the dependency
+// has the final say.
+//
+// When implementing a transition mutator, one needs to implement four methods:
+// - Split() that tells what variations a module has by itself
+// - OutgoingTransition() where a module tells what it wants from its
+// dependency
+// - IncomingTransition() where a module has the final say about its own
+// variation
+// - Mutate() that changes the state of a module depending on its variation
+//
+// That the effective variation of module B when depended on by module A is the
+// composition the outgoing transition of module A and the incoming transition
+// of module B.
+//
+// the outgoing transition should not take the properties of the dependency into
+// account, only those of the module that depends on it. For this reason, the
+// dependency is not even passed into it as an argument. Likewise, the incoming
+// transition should not take the properties of the depending module into
+// account and is thus not informed about it. This makes for a nice
+// decomposition of the decision logic.
+//
+// A given transition mutator only affects its own variation; other variations
+// stay unchanged along the dependency edges.
+//
+// Soong makes sure that all modules are created in the desired variations and
+// that dependency edges are set up correctly. This ensures that "missing
+// variation" errors do not happen and allows for more flexible changes in the
+// value of the variation among dependency edges (as oppposed to bottom-up
+// mutators where if module A in variation X depends on module B and module B
+// has that variation X, A must depend on variation X of B)
+//
+// The limited power of the context objects passed to individual mutators
+// methods also makes it more difficult to shoot oneself in the foot. Complete
+// safety is not guaranteed because no one prevents individual transition
+// mutators from mutating modules in illegal ways and for e.g. Split() or
+// Mutate() to run their own visitations of the transitive dependency of the
+// module and both of these are bad ideas, but it's better than no guardrails at
+// all.
+//
+// This model is pretty close to Bazel's configuration transitions. The mapping
+// between concepts in Soong and Bazel is as follows:
+// - Module == configured target
+// - Variant == configuration
+// - Variation name == configuration flag
+// - Variation == configuration flag value
+// - Outgoing transition == attribute transition
+// - Incoming transition == rule transition
+//
+// The Split() method does not have a Bazel equivalent and Bazel split
+// transitions do not have a Soong equivalent.
+//
+// Mutate() does not make sense in Bazel due to the different models of the
+// two systems: when creating new variations, Soong clones the old module and
+// thus some way is needed to change it state whereas Bazel creates each
+// configuration of a given configured target anew.
+type TransitionMutator interface {
+ // Returns the set of variations that should be created for a module no matter
+ // who depends on it. Used when Make depends on a particular variation or when
+ // the module knows its variations just based on information given to it in
+ // the Blueprint file. This method should not mutate the module it is called
+ // on.
+ Split(ctx BaseModuleContext) []string
+
+ // Called on a module to determine which variation it wants from its direct
+ // dependencies. The dependency itself can override this decision. This method
+ // should not mutate the module itself.
+ OutgoingTransition(ctx OutgoingTransitionContext, sourceVariation string) string
+
+ // Called on a module to determine which variation it should be in based on
+ // the variation modules that depend on it want. This gives the module a final
+ // say about its own variations. This method should not mutate the module
+ // itself.
+ IncomingTransition(ctx IncomingTransitionContext, incomingVariation string) string
+
+ // Called after a module was split into multiple variations on each variation.
+ // It should not split the module any further but adding new dependencies is
+ // fine. Unlike all the other methods on TransitionMutator, this method is
+ // allowed to mutate the module.
+ Mutate(ctx BottomUpMutatorContext, variation string)
+}
+
+type transitionMutatorImpl struct {
+ name string
+ mutator TransitionMutator
+}
+
+// Adds each argument in items to l if it's not already there.
+func addToStringListIfNotPresent(l []string, items ...string) []string {
+OUTER:
+ for _, i := range items {
+ for _, existing := range l {
+ if existing == i {
+ continue OUTER
+ }
+ }
+
+ l = append(l, i)
+ }
+
+ return l
+}
+
+func (t *transitionMutatorImpl) addRequiredVariation(m *moduleInfo, variation string) {
+ m.requiredVariationsLock.Lock()
+ defer m.requiredVariationsLock.Unlock()
+
+ // This is only a consistency check. Leaking the variations of a transition
+ // mutator to another one could well lead to issues that are difficult to
+ // track down.
+ if m.currentTransitionMutator != "" && m.currentTransitionMutator != t.name {
+ panic(fmt.Errorf("transition mutator is %s in mutator %s", m.currentTransitionMutator, t.name))
+ }
+
+ m.currentTransitionMutator = t.name
+ m.transitionVariations = addToStringListIfNotPresent(m.transitionVariations, variation)
+}
+
+func (t *transitionMutatorImpl) topDownMutator(mctx TopDownMutatorContext) {
+ module := mctx.(*mutatorContext).module
+ mutatorSplits := t.mutator.Split(mctx)
+ if mutatorSplits == nil || len(mutatorSplits) == 0 {
+ panic(fmt.Errorf("transition mutator %s returned no splits for module %s", t.name, mctx.ModuleName()))
+ }
+
+ // transitionVariations for given a module can be mutated by the module itself
+ // and modules that directly depend on it. Since this is a top-down mutator,
+ // all modules that directly depend on this module have already been processed
+ // so no locking is necessary.
+ module.transitionVariations = addToStringListIfNotPresent(module.transitionVariations, mutatorSplits...)
+ sort.Strings(module.transitionVariations)
+
+ for _, srcVariation := range module.transitionVariations {
+ for _, dep := range module.directDeps {
+ finalVariation := t.transition(mctx)(mctx.Module(), srcVariation, dep.module.logicModule, dep.tag)
+ t.addRequiredVariation(dep.module, finalVariation)
+ }
+ }
+}
+
+type transitionContextImpl struct {
+ module Module
+ depTag DependencyTag
+ config interface{}
+}
+
+func (c *transitionContextImpl) Module() Module {
+ return c.module
+}
+
+func (c *transitionContextImpl) DepTag() DependencyTag {
+ return c.depTag
+}
+
+func (c *transitionContextImpl) Config() interface{} {
+ return c.config
+}
+
+func (t *transitionMutatorImpl) transition(mctx BaseMutatorContext) Transition {
+ return func(source Module, sourceVariation string, dep Module, depTag DependencyTag) string {
+ tc := &transitionContextImpl{module: dep, depTag: depTag, config: mctx.Config()}
+ outgoingVariation := t.mutator.OutgoingTransition(tc, sourceVariation)
+ finalVariation := t.mutator.IncomingTransition(tc, outgoingVariation)
+ return finalVariation
+ }
+}
+
+func (t *transitionMutatorImpl) bottomUpMutator(mctx BottomUpMutatorContext) {
+ mc := mctx.(*mutatorContext)
+ // Fetch and clean up transition mutator state. No locking needed since the
+ // only time interaction between multiple modules is required is during the
+ // computation of the variations required by a given module.
+ variations := mc.module.transitionVariations
+ mc.module.transitionVariations = nil
+ mc.module.currentTransitionMutator = ""
+
+ if len(variations) < 1 {
+ panic(fmt.Errorf("no variations found for module %s by mutator %s",
+ mctx.ModuleName(), t.name))
+ }
+
+ if len(variations) == 1 && variations[0] == "" {
+ // Module is not split, just apply the transition
+ mc.applyTransition(t.transition(mctx))
+ } else {
+ mc.createVariationsWithTransition(t.transition(mctx), variations...)
+ }
+}
+
+func (t *transitionMutatorImpl) mutateMutator(mctx BottomUpMutatorContext) {
+ module := mctx.(*mutatorContext).module
+ currentVariation := module.variant.variations[t.name]
+ t.mutator.Mutate(mctx, currentVariation)
+}
+
+func (c *Context) RegisterTransitionMutator(name string, mutator TransitionMutator) {
+ impl := &transitionMutatorImpl{name: name, mutator: mutator}
+
+ c.RegisterTopDownMutator(name+"_deps", impl.topDownMutator).Parallel()
+ c.RegisterBottomUpMutator(name, impl.bottomUpMutator).Parallel()
+ c.RegisterBottomUpMutator(name+"_mutate", impl.mutateMutator).Parallel()
+}
+
type MutatorHandle interface {
// Set the mutator to visit modules in parallel while maintaining ordering. Calling any
// method on the mutator context is thread-safe, but the mutator must handle synchronization
@@ -709,6 +1008,71 @@
return c.ParseFileList(baseDir, pathsToParse, config)
}
+type shouldVisitFileInfo struct {
+ shouldVisitFile bool
+ skippedModules []string
+ reasonForSkip string
+ errs []error
+}
+
+// Returns a boolean for whether this file should be analyzed
+// Evaluates to true if the file either
+// 1. does not contain a blueprint_package_includes
+// 2. contains a blueprint_package_includes and all requested tags are set
+// This should be processed before adding any modules to the build graph
+func shouldVisitFile(c *Context, file *parser.File) shouldVisitFileInfo {
+ skippedModules := []string{}
+ var blueprintPackageIncludes *PackageIncludes
+ for _, def := range file.Defs {
+ switch def := def.(type) {
+ case *parser.Module:
+ skippedModules = append(skippedModules, def.Name())
+ if def.Type != "blueprint_package_includes" {
+ continue
+ }
+ module, errs := processModuleDef(def, file.Name, c.moduleFactories, nil, c.ignoreUnknownModuleTypes)
+ if len(errs) > 0 {
+ // This file contains errors in blueprint_package_includes
+ // Visit anyways so that we can report errors on other modules in the file
+ return shouldVisitFileInfo{
+ shouldVisitFile: true,
+ errs: errs,
+ }
+ }
+ logicModule, _ := c.cloneLogicModule(module)
+ blueprintPackageIncludes = logicModule.(*PackageIncludes)
+ }
+ }
+
+ if blueprintPackageIncludes != nil {
+ packageMatches := blueprintPackageIncludes.MatchesIncludeTags(c)
+ if !packageMatches {
+ return shouldVisitFileInfo{
+ shouldVisitFile: false,
+ skippedModules: skippedModules,
+ reasonForSkip: fmt.Sprintf(
+ "module is defined in %q which contains a blueprint_package_includes module with unsatisfied tags",
+ file.Name,
+ ),
+ }
+ }
+ }
+
+ shouldVisit, invalidatingPrefix := c.sourceRootDirs.SourceRootDirAllowed(file.Name)
+ if !shouldVisit {
+ return shouldVisitFileInfo{
+ shouldVisitFile: shouldVisit,
+ skippedModules: skippedModules,
+ reasonForSkip: fmt.Sprintf(
+ "%q is a descendant of %q, and that path prefix was not included in PRODUCT_SOURCE_ROOT_DIRS",
+ file.Name,
+ invalidatingPrefix,
+ ),
+ }
+ }
+ return shouldVisitFileInfo{shouldVisitFile: true}
+}
+
func (c *Context) ParseFileList(rootDir string, filePaths []string,
config interface{}) (deps []string, errs []error) {
@@ -724,9 +1088,15 @@
added chan<- struct{}
}
+ type newSkipInfo struct {
+ shouldVisitFileInfo
+ file string
+ }
+
moduleCh := make(chan newModuleInfo)
errsCh := make(chan []error)
doneCh := make(chan struct{})
+ skipCh := make(chan newSkipInfo)
var numErrs uint32
var numGoroutines int32
@@ -761,6 +1131,20 @@
}
return nil
}
+ shouldVisitInfo := shouldVisitFile(c, file)
+ errs := shouldVisitInfo.errs
+ if len(errs) > 0 {
+ atomic.AddUint32(&numErrs, uint32(len(errs)))
+ errsCh <- errs
+ }
+ if !shouldVisitInfo.shouldVisitFile {
+ skipCh <- newSkipInfo{
+ file: file.Name,
+ shouldVisitFileInfo: shouldVisitInfo,
+ }
+ // TODO: Write a file that lists the skipped bp files
+ return
+ }
for _, def := range file.Defs {
switch def := def.(type) {
@@ -814,6 +1198,14 @@
if n == 0 {
break loop
}
+ case skipped := <-skipCh:
+ nctx := newNamespaceContextFromFilename(skipped.file)
+ for _, name := range skipped.skippedModules {
+ c.nameInterface.NewSkippedModule(nctx, name, SkippedModuleInfo{
+ filename: skipped.file,
+ reason: skipped.reasonForSkip,
+ })
+ }
}
}
@@ -1301,7 +1693,7 @@
}
func (c *Context) createVariations(origModule *moduleInfo, mutatorName string,
- defaultVariationName *string, variationNames []string, local bool) (modulesOrAliases, []error) {
+ depChooser depChooser, variationNames []string, local bool) (modulesOrAliases, []error) {
if len(variationNames) == 0 {
panic(fmt.Errorf("mutator %q passed zero-length variation list for module %q",
@@ -1337,7 +1729,7 @@
newModules = append(newModules, newModule)
- newErrs := c.convertDepsToVariation(newModule, mutatorName, variationName, defaultVariationName)
+ newErrs := c.convertDepsToVariation(newModule, depChooser)
if len(newErrs) > 0 {
errs = append(errs, newErrs...)
}
@@ -1353,31 +1745,79 @@
return newModules, errs
}
-func (c *Context) convertDepsToVariation(module *moduleInfo,
- mutatorName, variationName string, defaultVariationName *string) (errs []error) {
+type depChooser func(source *moduleInfo, dep depInfo) (*moduleInfo, string)
+// This function is called for every dependency edge to determine which
+// variation of the dependency is needed. Its inputs are the depending module,
+// its variation, the dependency and the dependency tag.
+type Transition func(source Module, sourceVariation string, dep Module, depTag DependencyTag) string
+
+func chooseDepByTransition(mutatorName string, transition Transition) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ sourceVariation := source.variant.variations[mutatorName]
+ depLogicModule := dep.module.logicModule
+ if depLogicModule == nil {
+ // This is really a lie because the original dependency before the split
+ // went away when it was split. We choose an arbitrary split module
+ // instead and hope that whatever information the transition wants from it
+ // is the same as in the original one
+ // TODO(lberki): this can be fixed by calling transition() once and saving
+ // its results somewhere
+ depLogicModule = dep.module.splitModules[0].moduleOrAliasTarget().logicModule
+ }
+
+ desiredVariation := transition(source.logicModule, sourceVariation, depLogicModule, dep.tag)
+ for _, m := range dep.module.splitModules {
+ if m.moduleOrAliasVariant().variations[mutatorName] == desiredVariation {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+
+ return nil, desiredVariation
+ }
+}
+
+func chooseDep(candidates modulesOrAliases, mutatorName, variationName string, defaultVariationName *string) (*moduleInfo, string) {
+ for _, m := range candidates {
+ if m.moduleOrAliasVariant().variations[mutatorName] == variationName {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+
+ if defaultVariationName != nil {
+ // give it a second chance; match with defaultVariationName
+ for _, m := range candidates {
+ if m.moduleOrAliasVariant().variations[mutatorName] == *defaultVariationName {
+ return m.moduleOrAliasTarget(), ""
+ }
+ }
+ }
+
+ return nil, variationName
+}
+
+func chooseDepExplicit(mutatorName string,
+ variationName string, defaultVariationName *string) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ return chooseDep(dep.module.splitModules, mutatorName, variationName, defaultVariationName)
+ }
+}
+
+func chooseDepInherit(mutatorName string, defaultVariationName *string) depChooser {
+ return func(source *moduleInfo, dep depInfo) (*moduleInfo, string) {
+ sourceVariation := source.variant.variations[mutatorName]
+ return chooseDep(dep.module.splitModules, mutatorName, sourceVariation, defaultVariationName)
+ }
+}
+
+func (c *Context) convertDepsToVariation(module *moduleInfo, depChooser depChooser) (errs []error) {
for i, dep := range module.directDeps {
if dep.module.logicModule == nil {
- var newDep *moduleInfo
- for _, m := range dep.module.splitModules {
- if m.moduleOrAliasVariant().variations[mutatorName] == variationName {
- newDep = m.moduleOrAliasTarget()
- break
- }
- }
- if newDep == nil && defaultVariationName != nil {
- // give it a second chance; match with defaultVariationName
- for _, m := range dep.module.splitModules {
- if m.moduleOrAliasVariant().variations[mutatorName] == *defaultVariationName {
- newDep = m.moduleOrAliasTarget()
- break
- }
- }
- }
+ newDep, missingVariation := depChooser(module, dep)
if newDep == nil {
errs = append(errs, &BlueprintError{
Err: fmt.Errorf("failed to find variation %q for module %q needed by %q",
- variationName, dep.module.Name(), module.Name()),
+ missingVariation, dep.module.Name(), module.Name()),
Pos: module.pos,
})
continue
@@ -1519,7 +1959,7 @@
pprof.Do(ctx, pprof.Labels("blueprint", "ResolveDependencies"), func(ctx context.Context) {
c.initProviders()
- c.liveGlobals = newLiveTracker(config)
+ c.liveGlobals = newLiveTracker(c, config)
deps, errs = c.generateSingletonBuildActions(config, c.preSingletonInfo, c.liveGlobals)
if len(errs) > 0 {
@@ -2235,12 +2675,13 @@
return
}
-type jsonVariationMap []Variation
+type jsonVariations []Variation
type jsonModuleName struct {
Name string
- Variations jsonVariationMap
- DependencyVariations jsonVariationMap
+ Variant string
+ Variations jsonVariations
+ DependencyVariations jsonVariations
}
type jsonDep struct {
@@ -2253,11 +2694,12 @@
Deps []jsonDep
Type string
Blueprint string
+ CreatedBy *string
Module map[string]interface{}
}
-func toJsonVariationMap(vm variationMap) jsonVariationMap {
- m := make(jsonVariationMap, 0, len(vm))
+func toJsonVariationMap(vm variationMap) jsonVariations {
+ m := make(jsonVariations, 0, len(vm))
for k, v := range vm {
m = append(m, Variation{k, v})
}
@@ -2273,6 +2715,7 @@
func jsonModuleNameFromModuleInfo(m *moduleInfo) *jsonModuleName {
return &jsonModuleName{
Name: m.Name(),
+ Variant: m.variant.name,
Variations: toJsonVariationMap(m.variant.variations),
DependencyVariations: toJsonVariationMap(m.variant.dependencyVariations),
}
@@ -2282,6 +2725,18 @@
AddJSONData(d *map[string]interface{})
}
+// JSONAction contains the action-related info we expose to json module graph
+type JSONAction struct {
+ Inputs []string
+ Outputs []string
+}
+
+// JSONActionSupplier allows JSON representation of additional actions that are not registered in
+// Ninja
+type JSONActionSupplier interface {
+ JSONActions() []JSONAction
+}
+
func jsonModuleFromModuleInfo(m *moduleInfo) *JsonModule {
result := &JsonModule{
jsonModuleName: *jsonModuleNameFromModuleInfo(m),
@@ -2290,6 +2745,10 @@
Blueprint: m.relBlueprintsFile,
Module: make(map[string]interface{}),
}
+ if m.createdBy != nil {
+ n := m.createdBy.Name()
+ result.CreatedBy = &n
+ }
if j, ok := m.logicModule.(JSONDataSupplier); ok {
j.AddJSONData(&result.Module)
}
@@ -2304,24 +2763,35 @@
func jsonModuleWithActionsFromModuleInfo(m *moduleInfo) *JsonModule {
result := &JsonModule{
jsonModuleName: jsonModuleName{
- Name: m.Name(),
+ Name: m.Name(),
+ Variant: m.variant.name,
},
Deps: make([]jsonDep, 0),
Type: m.typeName,
Blueprint: m.relBlueprintsFile,
Module: make(map[string]interface{}),
}
- var actions []map[string]interface{}
+ var actions []JSONAction
for _, bDef := range m.actionDefs.buildDefs {
- actions = append(actions, map[string]interface{}{
- "Inputs": append(
+ actions = append(actions, JSONAction{
+ Inputs: append(
getNinjaStringsWithNilPkgNames(bDef.Inputs),
getNinjaStringsWithNilPkgNames(bDef.Implicits)...),
- "Outputs": append(
+ Outputs: append(
getNinjaStringsWithNilPkgNames(bDef.Outputs),
getNinjaStringsWithNilPkgNames(bDef.ImplicitOutputs)...),
})
}
+
+ if j, ok := m.logicModule.(JSONActionSupplier); ok {
+ actions = append(actions, j.JSONActions()...)
+ }
+ for _, p := range m.providers {
+ if j, ok := p.(JSONActionSupplier); ok {
+ actions = append(actions, j.JSONActions()...)
+ }
+ }
+
result.Module["Actions"] = actions
return result
}
@@ -2336,6 +2806,30 @@
return strs
}
+func (c *Context) GetOutputsFromModuleNames(moduleNames []string) map[string][]string {
+ modulesToOutputs := make(map[string][]string)
+ for _, m := range c.modulesSorted {
+ if inList(m.Name(), moduleNames) {
+ jmWithActions := jsonModuleWithActionsFromModuleInfo(m)
+ for _, a := range jmWithActions.Module["Actions"].([]JSONAction) {
+ modulesToOutputs[m.Name()] = append(modulesToOutputs[m.Name()], a.Outputs...)
+ }
+ // There could be several modules with the same name, so keep looping
+ }
+ }
+
+ return modulesToOutputs
+}
+
+func inList(s string, l []string) bool {
+ for _, element := range l {
+ if s == element {
+ return true
+ }
+ }
+ return false
+}
+
// PrintJSONGraph prints info of modules in a JSON file.
func (c *Context) PrintJSONGraphAndActions(wGraph io.Writer, wActions io.Writer) {
modulesToGraph := make([]*JsonModule, 0)
@@ -2453,6 +2947,8 @@
pprof.Do(ctx, pprof.Labels("blueprint", "runMutators"), func(ctx context.Context) {
for _, mutator := range c.mutatorInfo {
pprof.Do(ctx, pprof.Labels("mutator", mutator.name), func(context.Context) {
+ c.BeginEvent(mutator.name)
+ defer c.EndEvent(mutator.name)
var newDeps []string
if mutator.topDownMutator != nil {
newDeps, errs = c.runMutator(config, mutator, topDownMutator)
@@ -3119,8 +3615,8 @@
}
func (c *Context) missingDependencyError(module *moduleInfo, depName string) (errs error) {
- err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName)
-
+ guess := namesLike(depName, module.Name(), c.moduleGroups)
+ err := c.nameInterface.MissingDependencyError(module.Name(), module.namespace(), depName, guess)
return &BlueprintError{
Err: err,
Pos: module.pos,
@@ -3388,32 +3884,30 @@
}
targets := map[string]string{}
-
- // Collect all the module build targets.
- for _, module := range c.moduleInfo {
- for _, buildDef := range module.actionDefs.buildDefs {
+ var collectTargets = func(actionDefs localBuildActions) error {
+ for _, buildDef := range actionDefs.buildDefs {
ruleName := buildDef.Rule.fullName(c.pkgNames)
for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) {
outputValue, err := output.Eval(c.globalVariables)
if err != nil {
- return nil, err
+ return err
}
targets[outputValue] = ruleName
}
}
+ return nil
+ }
+ // Collect all the module build targets.
+ for _, module := range c.moduleInfo {
+ if err := collectTargets(module.actionDefs); err != nil {
+ return nil, err
+ }
}
// Collect all the singleton build targets.
for _, info := range c.singletonInfo {
- for _, buildDef := range info.actionDefs.buildDefs {
- ruleName := buildDef.Rule.fullName(c.pkgNames)
- for _, output := range append(buildDef.Outputs, buildDef.ImplicitOutputs...) {
- outputValue, err := output.Eval(c.globalVariables)
- if err != nil {
- return nil, err
- }
- targets[outputValue] = ruleName
- }
+ if err := collectTargets(info.actionDefs); err != nil {
+ return nil, err
}
}
@@ -3619,7 +4113,7 @@
return ""
}
-// WriteBuildFile writes the Ninja manifeset text for the generated build
+// WriteBuildFile writes the Ninja manifest text for the generated build
// actions to w. If this is called before PrepareBuildActions successfully
// completes then ErrBuildActionsNotReady is returned.
func (c *Context) WriteBuildFile(w io.StringWriter) error {
@@ -3632,59 +4126,46 @@
nw := newNinjaWriter(w)
- err = c.writeBuildFileHeader(nw)
- if err != nil {
+ if err = c.writeBuildFileHeader(nw); err != nil {
return
}
- err = c.writeNinjaRequiredVersion(nw)
- if err != nil {
+ if err = c.writeNinjaRequiredVersion(nw); err != nil {
return
}
- err = c.writeSubninjas(nw)
- if err != nil {
+ if err = c.writeSubninjas(nw); err != nil {
return
}
// TODO: Group the globals by package.
- err = c.writeGlobalVariables(nw)
- if err != nil {
+ if err = c.writeGlobalVariables(nw); err != nil {
return
}
- err = c.writeGlobalPools(nw)
- if err != nil {
+ if err = c.writeGlobalPools(nw); err != nil {
return
}
- err = c.writeBuildDir(nw)
- if err != nil {
+ if err = c.writeBuildDir(nw); err != nil {
return
}
- err = c.writeGlobalRules(nw)
- if err != nil {
+ if err = c.writeGlobalRules(nw); err != nil {
return
}
- err = c.writeAllModuleActions(nw)
- if err != nil {
+ if err = c.writeAllModuleActions(nw); err != nil {
return
}
- err = c.writeAllSingletonActions(nw)
- if err != nil {
+ if err = c.writeAllSingletonActions(nw); err != nil {
return
}
})
- if err != nil {
- return err
- }
-
- return nil
+ return err
}
type pkgAssociation struct {
@@ -3965,9 +4446,10 @@
}
func (c *Context) writeAllModuleActions(nw *ninjaWriter) error {
+ c.BeginEvent("modules")
+ defer c.EndEvent("modules")
headerTemplate := template.New("moduleHeader")
- _, err := headerTemplate.Parse(moduleHeaderTemplate)
- if err != nil {
+ if _, err := headerTemplate.Parse(moduleHeaderTemplate); err != nil {
// This is a programming error.
panic(err)
}
@@ -3978,6 +4460,11 @@
}
sort.Sort(moduleSorter{modules, c.nameInterface})
+ phonys := c.deduplicateOrderOnlyDeps(modules)
+ if err := c.writeLocalBuildActions(nw, phonys); err != nil {
+ return err
+ }
+
buf := bytes.NewBuffer(nil)
for _, module := range modules {
@@ -4004,28 +4491,23 @@
"pos": relPos,
"variant": module.variant.name,
}
- err = headerTemplate.Execute(buf, infoMap)
- if err != nil {
+ if err := headerTemplate.Execute(buf, infoMap); err != nil {
return err
}
- err = nw.Comment(buf.String())
- if err != nil {
+ if err := nw.Comment(buf.String()); err != nil {
return err
}
- err = nw.BlankLine()
- if err != nil {
+ if err := nw.BlankLine(); err != nil {
return err
}
- err = c.writeLocalBuildActions(nw, &module.actionDefs)
- if err != nil {
+ if err := c.writeLocalBuildActions(nw, &module.actionDefs); err != nil {
return err
}
- err = nw.BlankLine()
- if err != nil {
+ if err := nw.BlankLine(); err != nil {
return err
}
}
@@ -4034,6 +4516,8 @@
}
func (c *Context) writeAllSingletonActions(nw *ninjaWriter) error {
+ c.BeginEvent("singletons")
+ defer c.EndEvent("singletons")
headerTemplate := template.New("singletonHeader")
_, err := headerTemplate.Parse(singletonHeaderTemplate)
if err != nil {
@@ -4087,6 +4571,10 @@
return nil
}
+func (c *Context) GetEventHandler() *metrics.EventHandler {
+ return c.EventHandler
+}
+
func (c *Context) BeginEvent(name string) {
c.EventHandler.Begin(name)
}
@@ -4095,6 +4583,106 @@
c.EventHandler.End(name)
}
+func (c *Context) SetBeforePrepareBuildActionsHook(hookFn func() error) {
+ c.BeforePrepareBuildActionsHook = hookFn
+}
+
+// phonyCandidate represents the state of a set of deps that decides its eligibility
+// to be extracted as a phony output
+type phonyCandidate struct {
+ sync.Once
+ phony *buildDef // the phony buildDef that wraps the set
+ first *buildDef // the first buildDef that uses this set
+}
+
+// keyForPhonyCandidate gives a unique identifier for a set of deps.
+// If any of the deps use a variable, we return an empty string to signal
+// that this set of deps is ineligible for extraction.
+func keyForPhonyCandidate(deps []ninjaString) string {
+ hasher := sha256.New()
+ for _, d := range deps {
+ if len(d.Variables()) != 0 {
+ return ""
+ }
+ io.WriteString(hasher, d.Value(nil))
+ }
+ return base64.RawURLEncoding.EncodeToString(hasher.Sum(nil))
+}
+
+// scanBuildDef is called for every known buildDef `b` that has a non-empty `b.OrderOnly`.
+// If `b.OrderOnly` is not present in `candidates`, it gets stored.
+// But if `b.OrderOnly` already exists in `candidates`, then `b.OrderOnly`
+// (and phonyCandidate#first.OrderOnly) will be replaced with phonyCandidate#phony.Outputs
+func scanBuildDef(wg *sync.WaitGroup, candidates *sync.Map, phonyCount *atomic.Uint32, b *buildDef) {
+ defer wg.Done()
+ key := keyForPhonyCandidate(b.OrderOnly)
+ if key == "" {
+ return
+ }
+ if v, loaded := candidates.LoadOrStore(key, &phonyCandidate{
+ first: b,
+ }); loaded {
+ m := v.(*phonyCandidate)
+ m.Do(func() {
+ // this is the second occurrence and hence it makes sense to
+ // extract it as a phony output
+ phonyCount.Add(1)
+ m.phony = &buildDef{
+ Rule: Phony,
+ Outputs: []ninjaString{simpleNinjaString("dedup-" + key)},
+ Inputs: m.first.OrderOnly, //we could also use b.OrderOnly
+ Optional: true,
+ }
+ // the previously recorded build-def, which first had these deps as its
+ // order-only deps, should now use this phony output instead
+ m.first.OrderOnly = m.phony.Outputs
+ m.first = nil
+ })
+ b.OrderOnly = m.phony.Outputs
+ }
+}
+
+// deduplicateOrderOnlyDeps searches for common sets of order-only dependencies across all
+// buildDef instances in the provided moduleInfo instances. Each such
+// common set forms a new buildDef representing a phony output that then becomes
+// the sole order-only dependency of those buildDef instances
+func (c *Context) deduplicateOrderOnlyDeps(infos []*moduleInfo) *localBuildActions {
+ c.BeginEvent("deduplicate_order_only_deps")
+ defer c.EndEvent("deduplicate_order_only_deps")
+
+ candidates := sync.Map{} //used as map[key]*candidate
+ phonyCount := atomic.Uint32{}
+ wg := sync.WaitGroup{}
+ for _, info := range infos {
+ for _, b := range info.actionDefs.buildDefs {
+ if len(b.OrderOnly) > 0 {
+ wg.Add(1)
+ go scanBuildDef(&wg, &candidates, &phonyCount, b)
+ }
+ }
+ }
+ wg.Wait()
+
+ // now collect all created phonys to return
+ phonys := make([]*buildDef, 0, phonyCount.Load())
+ candidates.Range(func(_ any, v any) bool {
+ candidate := v.(*phonyCandidate)
+ if candidate.phony != nil {
+ phonys = append(phonys, candidate.phony)
+ }
+ return true
+ })
+
+ c.EventHandler.Do("sort_phony_builddefs", func() {
+ // sorting for determinism, the phony output names are stable
+ sort.Slice(phonys, func(i int, j int) bool {
+ return phonys[i].Outputs[0].Value(nil) < phonys[j].Outputs[0].Value(nil)
+ })
+ })
+
+ return &localBuildActions{buildDefs: phonys}
+}
+
func (c *Context) writeLocalBuildActions(nw *ninjaWriter,
defs *localBuildActions) error {
@@ -4103,7 +4691,7 @@
// A localVariable doesn't need the package names or config to
// determine its name or value.
name := v.fullName(nil)
- value, err := v.value(nil)
+ value, err := v.value(nil, nil)
if err != nil {
panic(err)
}
@@ -4218,7 +4806,7 @@
`
-var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+var moduleHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
Module: {{.name}}
Variant: {{.variant}}
Type: {{.typeName}}
@@ -4226,7 +4814,53 @@
Defined: {{.pos}}
`
-var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+var singletonHeaderTemplate = `# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
Singleton: {{.name}}
Factory: {{.goFactory}}
`
+
+// Blueprint module type that can be used to gate blueprint files beneath this directory
+type PackageIncludes struct {
+ properties struct {
+ // Package will be included if all include tags in this list are set
+ Match_all []string
+ }
+ name *string `blueprint:"mutated"`
+}
+
+func (pi *PackageIncludes) Name() string {
+ return proptools.String(pi.name)
+}
+
+// This module type does not have any build actions
+func (pi *PackageIncludes) GenerateBuildActions(ctx ModuleContext) {
+}
+
+func newPackageIncludesFactory() (Module, []interface{}) {
+ module := &PackageIncludes{}
+ AddLoadHook(module, func(ctx LoadHookContext) {
+ module.name = proptools.StringPtr(ctx.ModuleDir() + "_includes") // Generate a synthetic name
+ })
+ return module, []interface{}{&module.properties}
+}
+
+func RegisterPackageIncludesModuleType(ctx *Context) {
+ ctx.RegisterModuleType("blueprint_package_includes", newPackageIncludesFactory)
+}
+
+func (pi *PackageIncludes) MatchAll() []string {
+ return pi.properties.Match_all
+}
+
+// Returns true if all requested include tags are set in the Context object
+func (pi *PackageIncludes) MatchesIncludeTags(ctx *Context) bool {
+ if len(pi.MatchAll()) == 0 {
+ ctx.ModuleErrorf(pi, "Match_all must be a non-empty list")
+ }
+ for _, includeTag := range pi.MatchAll() {
+ if !ctx.ContainsIncludeTag(includeTag) {
+ return false
+ }
+ }
+ return true
+}
diff --git a/context_test.go b/context_test.go
index 6308ba9..1a1fb0d 100644
--- a/context_test.go
+++ b/context_test.go
@@ -18,6 +18,7 @@
"bytes"
"errors"
"fmt"
+ "path/filepath"
"reflect"
"strings"
"sync"
@@ -173,11 +174,11 @@
}
}
-// |===B---D - represents a non-walkable edge
-// A = represents a walkable edge
-// |===C===E---G
-// | | A should not be visited because it's the root node.
-// |===F===| B, D and E should not be walked.
+// > |===B---D - represents a non-walkable edge
+// > A = represents a walkable edge
+// > |===C===E---G
+// > | | A should not be visited because it's the root node.
+// > |===F===| B, D and E should not be walked.
func TestWalkDeps(t *testing.T) {
ctx := NewContext()
ctx.MockFileSystem(map[string][]byte{
@@ -186,31 +187,31 @@
name: "A",
deps: ["B", "C"],
}
-
+
bar_module {
name: "B",
deps: ["D"],
}
-
+
foo_module {
name: "C",
deps: ["E", "F"],
}
-
+
foo_module {
name: "D",
}
-
+
bar_module {
name: "E",
deps: ["G"],
}
-
+
foo_module {
name: "F",
deps: ["G"],
}
-
+
foo_module {
name: "G",
}
@@ -248,12 +249,12 @@
}
}
-// |===B---D - represents a non-walkable edge
-// A = represents a walkable edge
-// |===C===E===\ A should not be visited because it's the root node.
-// | | B, D should not be walked.
-// |===F===G===H G should be visited multiple times
-// \===/ H should only be visited once
+// > |===B---D - represents a non-walkable edge
+// > A = represents a walkable edge
+// > |===C===E===\ A should not be visited because it's the root node.
+// > | | B, D should not be walked.
+// > |===F===G===H G should be visited multiple times
+// > \===/ H should only be visited once
func TestWalkDepsDuplicates(t *testing.T) {
ctx := NewContext()
ctx.MockFileSystem(map[string][]byte{
@@ -329,11 +330,11 @@
}
}
-// - represents a non-walkable edge
-// A = represents a walkable edge
-// |===B-------\ A should not be visited because it's the root node.
-// | | B -> D should not be walked.
-// |===C===D===E B -> C -> D -> E should be walked
+// > - represents a non-walkable edge
+// > A = represents a walkable edge
+// > |===B-------\ A should not be visited because it's the root node.
+// > | | B -> D should not be walked.
+// > |===C===D===E B -> C -> D -> E should be walked
func TestWalkDepsDuplicates_IgnoreFirstPath(t *testing.T) {
ctx := NewContext()
ctx.MockFileSystem(map[string][]byte{
@@ -461,17 +462,17 @@
Deps []string
}
- ctx.CreateModule(newBarModule, &props{
+ ctx.CreateModule(newBarModule, "new_bar", &props{
Name: "B",
Deps: []string{"D"},
})
- ctx.CreateModule(newBarModule, &props{
+ ctx.CreateModule(newBarModule, "new_bar", &props{
Name: "C",
Deps: []string{"D"},
})
- ctx.CreateModule(newFooModule, &props{
+ ctx.CreateModule(newFooModule, "new_foo", &props{
Name: "D",
})
}
@@ -588,7 +589,7 @@
foo_module {
name: "A",
}
-
+
bar_module {
deps: ["A"],
}
@@ -1084,3 +1085,480 @@
}
})
}
+
+func TestPackageIncludes(t *testing.T) {
+ dir1_foo_bp := `
+ blueprint_package_includes {
+ match_all: ["use_dir1"],
+ }
+ foo_module {
+ name: "foo",
+ }
+ `
+ dir2_foo_bp := `
+ blueprint_package_includes {
+ match_all: ["use_dir2"],
+ }
+ foo_module {
+ name: "foo",
+ }
+ `
+ mockFs := map[string][]byte{
+ "dir1/Android.bp": []byte(dir1_foo_bp),
+ "dir2/Android.bp": []byte(dir2_foo_bp),
+ }
+ testCases := []struct {
+ desc string
+ includeTags []string
+ expectedDir string
+ expectedErr string
+ }{
+ {
+ desc: "use_dir1 is set, use dir1 foo",
+ includeTags: []string{"use_dir1"},
+ expectedDir: "dir1",
+ },
+ {
+ desc: "use_dir2 is set, use dir2 foo",
+ includeTags: []string{"use_dir2"},
+ expectedDir: "dir2",
+ },
+ {
+ desc: "duplicate module error if both use_dir1 and use_dir2 are set",
+ includeTags: []string{"use_dir1", "use_dir2"},
+ expectedDir: "",
+ expectedErr: `module "foo" already defined`,
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(tc.desc, func(t *testing.T) {
+ ctx := NewContext()
+ // Register mock FS
+ ctx.MockFileSystem(mockFs)
+ // Register module types
+ ctx.RegisterModuleType("foo_module", newFooModule)
+ RegisterPackageIncludesModuleType(ctx)
+ // Add include tags for test case
+ ctx.AddIncludeTags(tc.includeTags...)
+ // Run test
+ _, actualErrs := ctx.ParseFileList(".", []string{"dir1/Android.bp", "dir2/Android.bp"}, nil)
+ // Evaluate
+ if !strings.Contains(fmt.Sprintf("%s", actualErrs), fmt.Sprintf("%s", tc.expectedErr)) {
+ t.Errorf("Expected errors: %s, got errors: %s\n", tc.expectedErr, actualErrs)
+ }
+ if tc.expectedErr != "" {
+ return // expectedDir check not necessary
+ }
+ actualBpFile := ctx.moduleGroupFromName("foo", nil).modules.firstModule().relBlueprintsFile
+ if tc.expectedDir != filepath.Dir(actualBpFile) {
+ t.Errorf("Expected foo from %s, got %s\n", tc.expectedDir, filepath.Dir(actualBpFile))
+ }
+ })
+ }
+
+}
+
+func TestDeduplicateOrderOnlyDeps(t *testing.T) {
+ outputs := func(names ...string) []ninjaString {
+ r := make([]ninjaString, len(names))
+ for i, name := range names {
+ r[i] = literalNinjaString(name)
+ }
+ return r
+ }
+ b := func(output string, inputs []string, orderOnlyDeps []string) *buildDef {
+ return &buildDef{
+ Outputs: outputs(output),
+ Inputs: outputs(inputs...),
+ OrderOnly: outputs(orderOnlyDeps...),
+ }
+ }
+ m := func(bs ...*buildDef) *moduleInfo {
+ return &moduleInfo{actionDefs: localBuildActions{buildDefs: bs}}
+ }
+ type testcase struct {
+ modules []*moduleInfo
+ expectedPhonys []*buildDef
+ conversions map[string][]ninjaString
+ }
+ testCases := []testcase{{
+ modules: []*moduleInfo{
+ m(b("A", nil, []string{"d"})),
+ m(b("B", nil, []string{"d"})),
+ },
+ expectedPhonys: []*buildDef{
+ b("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ", []string{"d"}, nil),
+ },
+ conversions: map[string][]ninjaString{
+ "A": outputs("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ"),
+ "B": outputs("dedup-GKw-c0PwFokMUQ6T-TUmEWnZ4_VlQ2Qpgw-vCTT0-OQ"),
+ },
+ }, {
+ modules: []*moduleInfo{
+ m(b("A", nil, []string{"a"})),
+ m(b("B", nil, []string{"b"})),
+ },
+ }, {
+ modules: []*moduleInfo{
+ m(b("A", nil, []string{"a"})),
+ m(b("B", nil, []string{"b"})),
+ m(b("C", nil, []string{"a"})),
+ },
+ expectedPhonys: []*buildDef{b("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs", []string{"a"}, nil)},
+ conversions: map[string][]ninjaString{
+ "A": outputs("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs"),
+ "B": outputs("b"),
+ "C": outputs("dedup-ypeBEsobvcr6wjGzmiPcTaeG7_gUfE5yuYB3ha_uSLs"),
+ },
+ }, {
+ modules: []*moduleInfo{
+ m(b("A", nil, []string{"a", "b"}),
+ b("B", nil, []string{"a", "b"})),
+ m(b("C", nil, []string{"a", "c"}),
+ b("D", nil, []string{"a", "c"})),
+ },
+ expectedPhonys: []*buildDef{
+ b("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM", []string{"a", "b"}, nil),
+ b("dedup-9F3lHN7zCZFVHkHogt17VAR5lkigoAdT9E_JZuYVP8E", []string{"a", "c"}, nil)},
+ conversions: map[string][]ninjaString{
+ "A": outputs("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM"),
+ "B": outputs("dedup--44g_C5MPySMYMOb1lLzwTRymLuXe4tNWQO4UFViBgM"),
+ "C": outputs("dedup-9F3lHN7zCZFVHkHogt17VAR5lkigoAdT9E_JZuYVP8E"),
+ "D": outputs("dedup-9F3lHN7zCZFVHkHogt17VAR5lkigoAdT9E_JZuYVP8E"),
+ },
+ }}
+ for index, tc := range testCases {
+ t.Run(fmt.Sprintf("TestCase-%d", index), func(t *testing.T) {
+ ctx := NewContext()
+ actualPhonys := ctx.deduplicateOrderOnlyDeps(tc.modules)
+ if len(actualPhonys.variables) != 0 {
+ t.Errorf("No variables expected but found %v", actualPhonys.variables)
+ }
+ if len(actualPhonys.rules) != 0 {
+ t.Errorf("No rules expected but found %v", actualPhonys.rules)
+ }
+ if e, a := len(tc.expectedPhonys), len(actualPhonys.buildDefs); e != a {
+ t.Errorf("Expected %d build statements but got %d", e, a)
+ }
+ for i := 0; i < len(tc.expectedPhonys); i++ {
+ a := actualPhonys.buildDefs[i]
+ e := tc.expectedPhonys[i]
+ if !reflect.DeepEqual(e.Outputs, a.Outputs) {
+ t.Errorf("phonys expected %v but actualPhonys %v", e.Outputs, a.Outputs)
+ }
+ if !reflect.DeepEqual(e.Inputs, a.Inputs) {
+ t.Errorf("phonys expected %v but actualPhonys %v", e.Inputs, a.Inputs)
+ }
+ }
+ find := func(k string) *buildDef {
+ for _, m := range tc.modules {
+ for _, b := range m.actionDefs.buildDefs {
+ if reflect.DeepEqual(b.Outputs, outputs(k)) {
+ return b
+ }
+ }
+ }
+ return nil
+ }
+ for k, conversion := range tc.conversions {
+ actual := find(k)
+ if actual == nil {
+ t.Errorf("Couldn't find %s", k)
+ }
+ if !reflect.DeepEqual(actual.OrderOnly, conversion) {
+ t.Errorf("expected %s.OrderOnly = %v but got %v", k, conversion, actual.OrderOnly)
+ }
+ }
+ })
+ }
+}
+
+func TestSourceRootDirAllowed(t *testing.T) {
+ type pathCase struct {
+ path string
+ decidingPrefix string
+ allowed bool
+ }
+ testcases := []struct {
+ desc string
+ rootDirs []string
+ pathCases []pathCase
+ }{
+ {
+ desc: "simple case",
+ rootDirs: []string{
+ "a",
+ "b/c/d",
+ "-c",
+ "-d/c/a",
+ "c/some_single_file",
+ },
+ pathCases: []pathCase{
+ {
+ path: "a",
+ decidingPrefix: "a",
+ allowed: true,
+ },
+ {
+ path: "a/b/c",
+ decidingPrefix: "a",
+ allowed: true,
+ },
+ {
+ path: "b",
+ decidingPrefix: "",
+ allowed: true,
+ },
+ {
+ path: "b/c/d/a",
+ decidingPrefix: "b/c/d",
+ allowed: true,
+ },
+ {
+ path: "c",
+ decidingPrefix: "c",
+ allowed: false,
+ },
+ {
+ path: "c/a/b",
+ decidingPrefix: "c",
+ allowed: false,
+ },
+ {
+ path: "c/some_single_file",
+ decidingPrefix: "c/some_single_file",
+ allowed: true,
+ },
+ {
+ path: "d/c/a/abc",
+ decidingPrefix: "d/c/a",
+ allowed: false,
+ },
+ },
+ },
+ {
+ desc: "root directory order matters",
+ rootDirs: []string{
+ "-a",
+ "a/c/some_allowed_file",
+ "a/b/d/some_allowed_file",
+ "a/b",
+ "a/c",
+ "-a/b/d",
+ },
+ pathCases: []pathCase{
+ {
+ path: "a",
+ decidingPrefix: "a",
+ allowed: false,
+ },
+ {
+ path: "a/some_disallowed_file",
+ decidingPrefix: "a",
+ allowed: false,
+ },
+ {
+ path: "a/c/some_allowed_file",
+ decidingPrefix: "a/c/some_allowed_file",
+ allowed: true,
+ },
+ {
+ path: "a/b/d/some_allowed_file",
+ decidingPrefix: "a/b/d/some_allowed_file",
+ allowed: true,
+ },
+ {
+ path: "a/b/c",
+ decidingPrefix: "a/b",
+ allowed: true,
+ },
+ {
+ path: "a/b/c/some_allowed_file",
+ decidingPrefix: "a/b",
+ allowed: true,
+ },
+ {
+ path: "a/b/d",
+ decidingPrefix: "a/b/d",
+ allowed: false,
+ },
+ },
+ },
+ }
+ for _, tc := range testcases {
+ dirs := SourceRootDirs{}
+ dirs.Add(tc.rootDirs...)
+ for _, pc := range tc.pathCases {
+ t.Run(fmt.Sprintf("%s: %s", tc.desc, pc.path), func(t *testing.T) {
+ allowed, decidingPrefix := dirs.SourceRootDirAllowed(pc.path)
+ if allowed != pc.allowed {
+ if pc.allowed {
+ t.Errorf("expected path %q to be allowed, but was not; root allowlist: %q", pc.path, tc.rootDirs)
+ } else {
+ t.Errorf("path %q was allowed unexpectedly; root allowlist: %q", pc.path, tc.rootDirs)
+ }
+ }
+ if decidingPrefix != pc.decidingPrefix {
+ t.Errorf("expected decidingPrefix to be %q, but got %q", pc.decidingPrefix, decidingPrefix)
+ }
+ })
+ }
+ }
+}
+
+func TestSourceRootDirs(t *testing.T) {
+ root_foo_bp := `
+ foo_module {
+ name: "foo",
+ deps: ["foo_dir1", "foo_dir_ignored_special_case"],
+ }
+ `
+ dir1_foo_bp := `
+ foo_module {
+ name: "foo_dir1",
+ deps: ["foo_dir_ignored"],
+ }
+ `
+ dir_ignored_foo_bp := `
+ foo_module {
+ name: "foo_dir_ignored",
+ }
+ `
+ dir_ignored_special_case_foo_bp := `
+ foo_module {
+ name: "foo_dir_ignored_special_case",
+ }
+ `
+ mockFs := map[string][]byte{
+ "Android.bp": []byte(root_foo_bp),
+ "dir1/Android.bp": []byte(dir1_foo_bp),
+ "dir_ignored/Android.bp": []byte(dir_ignored_foo_bp),
+ "dir_ignored/special_case/Android.bp": []byte(dir_ignored_special_case_foo_bp),
+ }
+ fileList := []string{}
+ for f := range mockFs {
+ fileList = append(fileList, f)
+ }
+ testCases := []struct {
+ sourceRootDirs []string
+ expectedModuleDefs []string
+ unexpectedModuleDefs []string
+ expectedErrs []string
+ }{
+ {
+ sourceRootDirs: []string{},
+ expectedModuleDefs: []string{
+ "foo",
+ "foo_dir1",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ },
+ {
+ sourceRootDirs: []string{"-", ""},
+ unexpectedModuleDefs: []string{
+ "foo",
+ "foo_dir1",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ },
+ {
+ sourceRootDirs: []string{"-"},
+ unexpectedModuleDefs: []string{
+ "foo",
+ "foo_dir1",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ },
+ {
+ sourceRootDirs: []string{"dir1"},
+ expectedModuleDefs: []string{
+ "foo",
+ "foo_dir1",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ },
+ {
+ sourceRootDirs: []string{"-dir1"},
+ expectedModuleDefs: []string{
+ "foo",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ unexpectedModuleDefs: []string{
+ "foo_dir1",
+ },
+ expectedErrs: []string{
+ `Android.bp:2:2: module "foo" depends on skipped module "foo_dir1"; "foo_dir1" was defined in files(s) [dir1/Android.bp], but was skipped for reason(s) ["dir1/Android.bp" is a descendant of "dir1", and that path prefix was not included in PRODUCT_SOURCE_ROOT_DIRS]`,
+ },
+ },
+ {
+ sourceRootDirs: []string{"-", "dir1"},
+ expectedModuleDefs: []string{
+ "foo_dir1",
+ },
+ unexpectedModuleDefs: []string{
+ "foo",
+ "foo_dir_ignored",
+ "foo_dir_ignored_special_case",
+ },
+ expectedErrs: []string{
+ `dir1/Android.bp:2:2: module "foo_dir1" depends on skipped module "foo_dir_ignored"; "foo_dir_ignored" was defined in files(s) [dir_ignored/Android.bp], but was skipped for reason(s) ["dir_ignored/Android.bp" is a descendant of "", and that path prefix was not included in PRODUCT_SOURCE_ROOT_DIRS]`,
+ },
+ },
+ {
+ sourceRootDirs: []string{"-", "dir1", "dir_ignored/special_case/Android.bp"},
+ expectedModuleDefs: []string{
+ "foo_dir1",
+ "foo_dir_ignored_special_case",
+ },
+ unexpectedModuleDefs: []string{
+ "foo",
+ "foo_dir_ignored",
+ },
+ expectedErrs: []string{
+ "dir1/Android.bp:2:2: module \"foo_dir1\" depends on skipped module \"foo_dir_ignored\"; \"foo_dir_ignored\" was defined in files(s) [dir_ignored/Android.bp], but was skipped for reason(s) [\"dir_ignored/Android.bp\" is a descendant of \"\", and that path prefix was not included in PRODUCT_SOURCE_ROOT_DIRS]",
+ },
+ },
+ }
+ for _, tc := range testCases {
+ t.Run(fmt.Sprintf(`source root dirs are %q`, tc.sourceRootDirs), func(t *testing.T) {
+ ctx := NewContext()
+ ctx.MockFileSystem(mockFs)
+ ctx.RegisterModuleType("foo_module", newFooModule)
+ ctx.RegisterBottomUpMutator("deps", depsMutator)
+ ctx.AddSourceRootDirs(tc.sourceRootDirs...)
+ RegisterPackageIncludesModuleType(ctx)
+ ctx.ParseFileList(".", fileList, nil)
+ _, actualErrs := ctx.ResolveDependencies(nil)
+
+ stringErrs := []string(nil)
+ for _, err := range actualErrs {
+ stringErrs = append(stringErrs, err.Error())
+ }
+ if !reflect.DeepEqual(tc.expectedErrs, stringErrs) {
+ t.Errorf("expected to find errors %v; got %v", tc.expectedErrs, stringErrs)
+ }
+ for _, modName := range tc.expectedModuleDefs {
+ allMods := ctx.moduleGroupFromName(modName, nil)
+ if allMods == nil || len(allMods.modules) != 1 {
+ mods := modulesOrAliases{}
+ if allMods != nil {
+ mods = allMods.modules
+ }
+ t.Errorf("expected to find one definition for module %q, but got %v", modName, mods)
+ }
+ }
+
+ for _, modName := range tc.unexpectedModuleDefs {
+ allMods := ctx.moduleGroupFromName(modName, nil)
+ if allMods != nil {
+ t.Errorf("expected to find no definitions for module %q, but got %v", modName, allMods.modules)
+ }
+ }
+ })
+ }
+}
diff --git a/doc.go b/doc.go
index 4a9dfd7..9e917f6 100644
--- a/doc.go
+++ b/doc.go
@@ -35,17 +35,17 @@
// the module type looks like a function call, and the properties of the module
// look like optional arguments. For example, a simple module might look like:
//
-// cc_library {
-// name: "cmd",
-// srcs: [
-// "main.c",
-// ],
-// deps: [
-// "libc",
-// ],
-// }
+// cc_library {
+// name: "cmd",
+// srcs: [
+// "main.c",
+// ],
+// deps: [
+// "libc",
+// ],
+// }
//
-// subdirs = ["subdir1", "subdir2"]
+// subdirs = ["subdir1", "subdir2"]
//
// The modules from the top level Blueprints file and recursively through any
// subdirectories listed by the "subdirs" variable are read by Blueprint, and
diff --git a/go.mod b/go.mod
index fe96d45..e278f2f 100644
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,3 @@
module github.com/google/blueprint
-go 1.13
+go 1.18
diff --git a/levenshtein.go b/levenshtein.go
new file mode 100644
index 0000000..de5b75a
--- /dev/null
+++ b/levenshtein.go
@@ -0,0 +1,117 @@
+// Copyright 2021 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blueprint
+
+import (
+ "sort"
+)
+
+func abs(a int) int {
+ if a < 0 {
+ return -a
+ }
+ return a
+}
+
+// This implementation is written to be recursive, because
+// we know Soong names are short, so we shouldn't hit the stack
+// depth. Also, the buffer is indexed this way so that new
+// allocations aren't needed.
+func levenshtein(a, b string, ai, bi, max int, buf [][]int) int {
+ if max == 0 {
+ return 0
+ }
+ if ai >= len(a) {
+ return len(b) - bi
+ }
+ if bi >= len(b) {
+ return len(a) - ai
+ }
+ if buf[bi][ai] != 0 {
+ return buf[bi][ai]
+ }
+ if abs(len(a)-len(b)) >= max {
+ return max
+ }
+ var res = max
+ if a[ai] == b[bi] {
+ res = levenshtein(a, b, ai+1, bi+1, max, buf)
+ } else {
+ if c := levenshtein(a, b, ai+1, bi+1, max-1, buf); c < res {
+ res = c // replace
+ }
+ if c := levenshtein(a, b, ai+1, bi, max-1, buf); c < res {
+ res = c // delete from a
+ }
+ if c := levenshtein(a, b, ai, bi+1, max-1, buf); c < res {
+ res = c // delete from b
+ }
+ res += 1
+ }
+ buf[bi][ai] = res
+ return res
+}
+
+func stringIn(arr []string, str string) bool {
+ for _, a := range arr {
+ if a == str {
+ return true
+ }
+ }
+ return false
+}
+
+func namesLike(name string, unlike string, moduleGroups []*moduleGroup) []string {
+ const kAllowedDifferences = 10
+ buf := make([][]int, len(name)+kAllowedDifferences)
+ for i := range buf {
+ buf[i] = make([]int, len(name))
+ }
+
+ var best []string
+ bestVal := kAllowedDifferences + 1
+
+ for _, group := range moduleGroups {
+ other := group.name
+
+ if other == unlike {
+ continue
+ }
+
+ l := levenshtein(name, other, 0, 0, kAllowedDifferences, buf)
+ // fmt.Printf("levenshtein %q %q %d\n", name, other, l)
+
+ // slightly better to use a min-heap
+ if l == 0 {
+ // these are the same, so it must be in a different namespace
+ // ignore...
+ } else if l < bestVal {
+ bestVal = l
+ best = []string{other}
+ } else if l == bestVal && !stringIn(best, other) {
+ best = append(best, other)
+ }
+
+ // zero buffer once used
+ for _, v := range buf {
+ for j := range v {
+ v[j] = 0
+ }
+ }
+ }
+
+ sort.Strings(best)
+ return best
+}
diff --git a/levenshtein_test.go b/levenshtein_test.go
new file mode 100644
index 0000000..60f0293
--- /dev/null
+++ b/levenshtein_test.go
@@ -0,0 +1,54 @@
+// Copyright 2014 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package blueprint
+
+import (
+ "reflect"
+ "testing"
+)
+
+func mods(mods []string) []*moduleGroup {
+ ret := []*moduleGroup{}
+
+ for _, v := range mods {
+ m := moduleGroup{name: v}
+ ret = append(ret, &m)
+ }
+
+ return ret
+}
+
+func assertEqual(t *testing.T, a, b []string) {
+ if len(a) == 0 && len(b) == 0 {
+ return
+ }
+
+ if !reflect.DeepEqual(a, b) {
+ t.Errorf("Expected the following to be equal:\n\t%q\n\t%q", a, b)
+ }
+}
+
+func TestLevenshteinWontGuessUnlike(t *testing.T) {
+ assertEqual(t, namesLike("a", "test", mods([]string{"test"})), []string{})
+}
+func TestLevenshteinInsert(t *testing.T) {
+ assertEqual(t, namesLike("a", "test", mods([]string{"ab", "ac", "not_this"})), []string{"ab", "ac"})
+}
+func TestLevenshteinDelete(t *testing.T) {
+ assertEqual(t, namesLike("ab", "test", mods([]string{"a", "b", "not_this"})), []string{"a", "b"})
+}
+func TestLevenshteinReplace(t *testing.T) {
+ assertEqual(t, namesLike("aa", "test", mods([]string{"ab", "ac", "not_this"})), []string{"ab", "ac"})
+}
diff --git a/live_tracker.go b/live_tracker.go
index 1d48e58..ef9c8a9 100644
--- a/live_tracker.go
+++ b/live_tracker.go
@@ -23,14 +23,16 @@
type liveTracker struct {
sync.Mutex
config interface{} // Used to evaluate variable, rule, and pool values.
+ ctx *Context // Used to evaluate globs
variables map[Variable]ninjaString
pools map[Pool]*poolDef
rules map[Rule]*ruleDef
}
-func newLiveTracker(config interface{}) *liveTracker {
+func newLiveTracker(ctx *Context, config interface{}) *liveTracker {
return &liveTracker{
+ ctx: ctx,
config: config,
variables: make(map[Variable]ninjaString),
pools: make(map[Pool]*poolDef),
@@ -153,7 +155,9 @@
func (l *liveTracker) addVariable(v Variable) error {
_, ok := l.variables[v]
if !ok {
- value, err := v.value(l.config)
+ ctx := &variableFuncContext{l.ctx}
+
+ value, err := v.value(ctx, l.config)
if err == errVariableIsArg {
// This variable is a placeholder for an argument that can be passed
// to a rule. It has no value and thus doesn't reference any other
diff --git a/metrics/Android.bp b/metrics/Android.bp
index 3668668..1e3a6f0 100644
--- a/metrics/Android.bp
+++ b/metrics/Android.bp
@@ -24,4 +24,7 @@
srcs: [
"event_handler.go",
],
+ testSrcs: [
+ "event_handler_test.go",
+ ],
}
diff --git a/metrics/event_handler.go b/metrics/event_handler.go
index c19d039..35a6858 100644
--- a/metrics/event_handler.go
+++ b/metrics/event_handler.go
@@ -31,10 +31,8 @@
scopeStartTimes []time.Time
}
-// _now wraps the time.Now() function. _now is declared for unit testing purpose.
-var _now = func() time.Time {
- return time.Now()
-}
+// _now simply delegates to time.Now() function. _now is declared for unit testing purpose.
+var _now = time.Now
// Event holds the performance metrics data of a single build event.
type Event struct {
@@ -58,15 +56,25 @@
// call to End (though other events may begin and end before this event ends).
// Events within the same scope must have unique names.
func (h *EventHandler) Begin(name string) {
+ if strings.ContainsRune(name, '.') {
+ panic(fmt.Sprintf("illegal event name (avoid dot): %s", name))
+ }
h.scopeIds = append(h.scopeIds, name)
h.scopeStartTimes = append(h.scopeStartTimes, _now())
}
+// Do wraps a function with calls to Begin() and End().
+func (h *EventHandler) Do(name string, f func()) {
+ h.Begin(name)
+ defer h.End(name)
+ f()
+}
+
// End logs the end of an event. All events nested within this event must have
// themselves been marked completed.
func (h *EventHandler) End(name string) {
if len(h.scopeIds) == 0 || name != h.scopeIds[len(h.scopeIds)-1] {
- panic(fmt.Errorf("Unexpected scope end '%s'. Current scope: (%s)",
+ panic(fmt.Errorf("unexpected scope end '%s'. Current scope: (%s)",
name, h.scopeIds))
}
event := Event{
@@ -89,16 +97,16 @@
func (h *EventHandler) CompletedEvents() []Event {
if len(h.scopeIds) > 0 {
panic(fmt.Errorf(
- "Retrieving events before all events have been closed. Current scope: (%s)",
+ "retrieving events before all events have been closed. Current scope: (%s)",
h.scopeIds))
}
// Validate no two events have the same full id.
- ids := map[string]bool{}
+ ids := map[string]struct{}{}
for _, event := range h.completedEvents {
if _, containsId := ids[event.Id]; containsId {
- panic(fmt.Errorf("Duplicate event registered: %s", event.Id))
+ panic(fmt.Errorf("duplicate event registered: %s", event.Id))
}
- ids[event.Id] = true
+ ids[event.Id] = struct{}{}
}
return h.completedEvents
}
diff --git a/metrics/event_handler_test.go b/metrics/event_handler_test.go
new file mode 100644
index 0000000..08a59bd
--- /dev/null
+++ b/metrics/event_handler_test.go
@@ -0,0 +1,100 @@
+// Copyright 2022 Google Inc. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package metrics
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func Map[A any, B any](in []A, f func(A) B) []B {
+ r := make([]B, len(in))
+ for i, a := range in {
+ r[i] = f(a)
+ }
+ return r
+}
+
+func TestEventNameWithDot(t *testing.T) {
+ defer func() {
+ r := fmt.Sprintf("%v", recover())
+ if !strings.HasPrefix(r, "illegal event name") {
+ t.Errorf("The code did not panic in the expected manner: %s", r)
+ }
+ }()
+ eh := EventHandler{}
+ eh.Begin("a.")
+}
+
+func TestEventNesting(t *testing.T) {
+ eh := EventHandler{}
+ eh.Begin("a")
+ eh.Begin("b")
+ eh.End("b")
+ eh.Begin("c")
+ eh.End("c")
+ eh.End("a")
+ expected := []string{"a.b", "a.c", "a"}
+ actual := Map(eh.CompletedEvents(), func(e Event) string {
+ return e.Id
+ })
+ if !reflect.DeepEqual(expected, actual) {
+ t.Errorf("expected: %s actual %s", expected, actual)
+ }
+}
+
+func TestEventOverlap(t *testing.T) {
+ defer func() {
+ r := fmt.Sprintf("%v", recover())
+ if !strings.Contains(r, "unexpected scope end 'a'") {
+ t.Errorf("expected panic but: %s", r)
+ }
+ }()
+ eh := EventHandler{}
+ eh.Begin("a")
+ eh.Begin("b")
+ eh.End("a")
+}
+
+func TestEventDuplication(t *testing.T) {
+ eh := EventHandler{}
+ eh.Begin("a")
+ eh.Begin("b")
+ eh.End("b")
+ eh.Begin("b")
+ eh.End("b")
+ eh.End("a")
+ defer func() {
+ r := fmt.Sprintf("%v", recover())
+ if !strings.HasPrefix(r, "duplicate event") {
+ t.Errorf("expected panic but: %s", r)
+ }
+ }()
+ eh.CompletedEvents()
+}
+
+func TestIncompleteEvent(t *testing.T) {
+ eh := EventHandler{}
+ eh.Begin("a")
+ defer func() {
+ r := fmt.Sprintf("%v", recover())
+ if !strings.HasPrefix(r, "retrieving events before all events have been closed.") {
+ t.Errorf("expected panic but: %s", r)
+ }
+ }()
+ eh.CompletedEvents()
+}
diff --git a/microfactory/microfactory.bash b/microfactory/microfactory.bash
index ded7b33..f037c04 100644
--- a/microfactory/microfactory.bash
+++ b/microfactory/microfactory.bash
@@ -51,6 +51,7 @@
local gen_src_dir="${BUILDDIR}/.microfactory_$(uname)_intermediates/src"
mkdir -p "${gen_src_dir}"
sed "s/^package microfactory/package main/" "${mf_src}/microfactory.go" >"${gen_src_dir}/microfactory.go"
+ printf "\n//for use with go run\nfunc main() { Main() }\n" >>"${gen_src_dir}/microfactory.go"
mf_cmd="${GOROOT}/bin/go run ${gen_src_dir}/microfactory.go"
else
diff --git a/microfactory/microfactory.go b/microfactory/microfactory.go
index a0c9a14..faa0d73 100644
--- a/microfactory/microfactory.go
+++ b/microfactory/microfactory.go
@@ -16,8 +16,8 @@
// to `go install`, but doesn't require a GOPATH. A package->path mapping can
// be specified as command line options:
//
-// -pkg-path android/soong=build/soong
-// -pkg-path github.com/google/blueprint=build/blueprint
+// -pkg-path android/soong=build/soong
+// -pkg-path github.com/google/blueprint=build/blueprint
//
// The paths can be relative to the current working directory, or an absolute
// path. Both packages and paths are compared with full directory names, so the
@@ -617,8 +617,6 @@
return true
}
-// microfactory.bash will make a copy of this file renamed into the main package for use with `go run`
-func main() { Main() }
func Main() {
var output, mybin string
var config Config
diff --git a/module_ctx.go b/module_ctx.go
index 53ee405..a1388b4 100644
--- a/module_ctx.go
+++ b/module_ctx.go
@@ -58,27 +58,27 @@
// that other modules can link against. The library Module might implement the
// following interface:
//
-// type LibraryProducer interface {
-// LibraryFileName() string
-// }
+// type LibraryProducer interface {
+// LibraryFileName() string
+// }
//
-// func IsLibraryProducer(module blueprint.Module) {
-// _, ok := module.(LibraryProducer)
-// return ok
-// }
+// func IsLibraryProducer(module blueprint.Module) {
+// _, ok := module.(LibraryProducer)
+// return ok
+// }
//
// A binary-producing Module that depends on the library Module could then do:
//
-// func (m *myBinaryModule) GenerateBuildActions(ctx blueprint.ModuleContext) {
-// ...
-// var libraryFiles []string
-// ctx.VisitDepsDepthFirstIf(IsLibraryProducer,
-// func(module blueprint.Module) {
-// libProducer := module.(LibraryProducer)
-// libraryFiles = append(libraryFiles, libProducer.LibraryFileName())
-// })
-// ...
-// }
+// func (m *myBinaryModule) GenerateBuildActions(ctx blueprint.ModuleContext) {
+// ...
+// var libraryFiles []string
+// ctx.VisitDepsDepthFirstIf(IsLibraryProducer,
+// func(module blueprint.Module) {
+// libProducer := module.(LibraryProducer)
+// libraryFiles = append(libraryFiles, libProducer.LibraryFileName())
+// })
+// ...
+// }
//
// to build the list of library file names that should be included in its link
// command.
@@ -300,6 +300,9 @@
// There are no guarantees about which variant of the module will be returned.
// Prefer retrieving the module using GetDirectDep or a visit function, when possible, as
// this will guarantee the appropriate module-variant dependency is returned.
+ //
+ // WARNING: This should _only_ be used within the context of bp2build, where variants and
+ // dependencies are not created.
ModuleFromName(name string) (Module, bool)
// OtherModuleDependencyVariantExists returns true if a module with the
@@ -836,7 +839,7 @@
// CreateModule creates a new module by calling the factory method for the specified moduleType, and applies
// the specified property structs to it as if the properties were set in a blueprint file.
- CreateModule(ModuleFactory, ...interface{}) Module
+ CreateModule(ModuleFactory, string, ...interface{}) Module
}
type BottomUpMutatorContext interface {
@@ -991,11 +994,17 @@
}
func (mctx *mutatorContext) CreateVariations(variationNames ...string) []Module {
- return mctx.createVariations(variationNames, false)
+ depChooser := chooseDepInherit(mctx.name, mctx.defaultVariation)
+ return mctx.createVariations(variationNames, depChooser, false)
+}
+
+func (mctx *mutatorContext) createVariationsWithTransition(transition Transition, variationNames ...string) []Module {
+ return mctx.createVariations(variationNames, chooseDepByTransition(mctx.name, transition), false)
}
func (mctx *mutatorContext) CreateLocalVariations(variationNames ...string) []Module {
- return mctx.createVariations(variationNames, true)
+ depChooser := chooseDepInherit(mctx.name, mctx.defaultVariation)
+ return mctx.createVariations(variationNames, depChooser, true)
}
func (mctx *mutatorContext) SetVariationProvider(module Module, provider ProviderKey, value interface{}) {
@@ -1008,9 +1017,9 @@
panic(fmt.Errorf("module %q is not a newly created variant of %q", module, mctx.module))
}
-func (mctx *mutatorContext) createVariations(variationNames []string, local bool) []Module {
+func (mctx *mutatorContext) createVariations(variationNames []string, depChooser depChooser, local bool) []Module {
var ret []Module
- modules, errs := mctx.context.createVariations(mctx.module, mctx.name, mctx.defaultVariation, variationNames, local)
+ modules, errs := mctx.context.createVariations(mctx.module, mctx.name, depChooser, variationNames, local)
if len(errs) > 0 {
mctx.errs = append(mctx.errs, errs...)
}
@@ -1091,8 +1100,13 @@
panic(fmt.Errorf("no %q variation in module variations %q", targetVariationName, foundVariations))
}
+func (mctx *mutatorContext) applyTransition(transition Transition) {
+ mctx.context.convertDepsToVariation(mctx.module, chooseDepByTransition(mctx.name, transition))
+}
+
func (mctx *mutatorContext) SetDependencyVariation(variationName string) {
- mctx.context.convertDepsToVariation(mctx.module, mctx.name, variationName, nil)
+ mctx.context.convertDepsToVariation(mctx.module, chooseDepExplicit(
+ mctx.name, variationName, nil))
}
func (mctx *mutatorContext) SetDefaultDependencyVariation(variationName *string) {
@@ -1201,13 +1215,14 @@
mctx.rename = append(mctx.rename, rename{mctx.module.group, name})
}
-func (mctx *mutatorContext) CreateModule(factory ModuleFactory, props ...interface{}) Module {
+func (mctx *mutatorContext) CreateModule(factory ModuleFactory, typeName string, props ...interface{}) Module {
module := newModule(factory)
module.relBlueprintsFile = mctx.module.relBlueprintsFile
module.pos = mctx.module.pos
module.propertyPos = mctx.module.propertyPos
module.createdBy = mctx.module
+ module.typeName = typeName
for _, p := range props {
err := proptools.AppendMatchingProperties(module.properties, p, nil)
diff --git a/name_interface.go b/name_interface.go
index 5e7e16e..db82453 100644
--- a/name_interface.go
+++ b/name_interface.go
@@ -17,6 +17,7 @@
import (
"fmt"
"sort"
+ "strings"
)
// This file exposes the logic of locating a module via a query string, to enable
@@ -54,12 +55,18 @@
// Gets called when a new module is created
NewModule(ctx NamespaceContext, group ModuleGroup, module Module) (namespace Namespace, err []error)
+ // Gets called when a module was pruned from the build tree by SourceRootDirs
+ NewSkippedModule(ctx NamespaceContext, name string, skipInfo SkippedModuleInfo)
+
// Finds the module with the given name
ModuleFromName(moduleName string, namespace Namespace) (group ModuleGroup, found bool)
+ // Finds if the module with the given name was skipped
+ SkippedModuleFromName(moduleName string, namespace Namespace) (skipInfos []SkippedModuleInfo, skipped bool)
+
// Returns an error indicating that the given module could not be found.
// The error contains some diagnostic information about where the dependency can be found.
- MissingDependencyError(depender string, dependerNamespace Namespace, depName string) (err error)
+ MissingDependencyError(depender string, dependerNamespace Namespace, depName string, guess []string) (err error)
// Rename
Rename(oldName string, newName string, namespace Namespace) []error
@@ -88,18 +95,29 @@
return &namespaceContextImpl{moduleInfo.pos.Filename}
}
+func newNamespaceContextFromFilename(filename string) NamespaceContext {
+ return &namespaceContextImpl{filename}
+}
+
func (ctx *namespaceContextImpl) ModulePath() string {
return ctx.modulePath
}
+type SkippedModuleInfo struct {
+ filename string
+ reason string
+}
+
// a SimpleNameInterface just stores all modules in a map based on name
type SimpleNameInterface struct {
- modules map[string]ModuleGroup
+ modules map[string]ModuleGroup
+ skippedModules map[string][]SkippedModuleInfo
}
func NewSimpleNameInterface() *SimpleNameInterface {
return &SimpleNameInterface{
- modules: make(map[string]ModuleGroup),
+ modules: make(map[string]ModuleGroup),
+ skippedModules: make(map[string][]SkippedModuleInfo),
}
}
@@ -118,11 +136,23 @@
return nil, []error{}
}
+func (s *SimpleNameInterface) NewSkippedModule(ctx NamespaceContext, name string, info SkippedModuleInfo) {
+ if name == "" {
+ return
+ }
+ s.skippedModules[name] = append(s.skippedModules[name], info)
+}
+
func (s *SimpleNameInterface) ModuleFromName(moduleName string, namespace Namespace) (group ModuleGroup, found bool) {
group, found = s.modules[moduleName]
return group, found
}
+func (s *SimpleNameInterface) SkippedModuleFromName(moduleName string, namespace Namespace) (skipInfos []SkippedModuleInfo, skipped bool) {
+ skipInfos, skipped = s.skippedModules[moduleName]
+ return
+}
+
func (s *SimpleNameInterface) Rename(oldName string, newName string, namespace Namespace) (errs []error) {
existingGroup, exists := s.modules[newName]
if exists {
@@ -167,8 +197,30 @@
return groups
}
-func (s *SimpleNameInterface) MissingDependencyError(depender string, dependerNamespace Namespace, dependency string) (err error) {
- return fmt.Errorf("%q depends on undefined module %q", depender, dependency)
+func (s *SimpleNameInterface) MissingDependencyError(depender string, dependerNamespace Namespace, dependency string, guess []string) (err error) {
+ skipInfos, skipped := s.SkippedModuleFromName(dependency, dependerNamespace)
+ if skipped {
+ filesFound := make([]string, 0, len(skipInfos))
+ reasons := make([]string, 0, len(skipInfos))
+ for _, info := range skipInfos {
+ filesFound = append(filesFound, info.filename)
+ reasons = append(reasons, info.reason)
+ }
+ return fmt.Errorf(
+ "module %q depends on skipped module %q; %q was defined in files(s) [%v], but was skipped for reason(s) [%v]",
+ depender,
+ dependency,
+ dependency,
+ strings.Join(filesFound, ", "),
+ strings.Join(reasons, "; "),
+ )
+ }
+
+ guessString := ""
+ if len(guess) > 0 {
+ guessString = fmt.Sprintf(" Did you mean %q?", guess)
+ }
+ return fmt.Errorf("%q depends on undefined module %q.%s", depender, dependency, guessString)
}
func (s *SimpleNameInterface) GetNamespace(ctx NamespaceContext) Namespace {
diff --git a/ninja_strings.go b/ninja_strings.go
index 51a167d..9e83a4d 100644
--- a/ninja_strings.go
+++ b/ninja_strings.go
@@ -118,7 +118,7 @@
r := rune(str[i])
state, err = state(parseState, i, r)
if err != nil {
- return nil, err
+ return nil, fmt.Errorf("error parsing ninja string %q: %s", str, err)
}
}
@@ -325,11 +325,11 @@
return n.variables
}
-func (l literalNinjaString) Value(pkgNames map[*packageContext]string) string {
+func (l literalNinjaString) Value(_ map[*packageContext]string) string {
return defaultEscaper.Replace(string(l))
}
-func (l literalNinjaString) ValueWithEscaper(w io.StringWriter, pkgNames map[*packageContext]string,
+func (l literalNinjaString) ValueWithEscaper(w io.StringWriter, _ map[*packageContext]string,
escaper *strings.Replacer) {
w.WriteString(escaper.Replace(string(l)))
}
diff --git a/ninja_strings_test.go b/ninja_strings_test.go
index c1e05f7..f400074 100644
--- a/ninja_strings_test.go
+++ b/ninja_strings_test.go
@@ -87,7 +87,7 @@
strs: []string{"$ ", " "},
}, {
input: "foo $ bar",
- err: "invalid character after '$' at byte offset 5",
+ err: `error parsing ninja string "foo $ bar": invalid character after '$' at byte offset 5`,
},
{
input: "foo $",
@@ -95,11 +95,11 @@
},
{
input: "foo ${} bar",
- err: "empty variable name at byte offset 6",
+ err: `error parsing ninja string "foo ${} bar": empty variable name at byte offset 6`,
},
{
input: "foo ${abc!} bar",
- err: "invalid character in variable name at byte offset 9",
+ err: `error parsing ninja string "foo ${abc!} bar": invalid character in variable name at byte offset 9`,
},
{
input: "foo ${abc",
diff --git a/package_ctx.go b/package_ctx.go
index 1eafdb9..f8a7cc4 100644
--- a/package_ctx.go
+++ b/package_ctx.go
@@ -31,35 +31,35 @@
// passed to all calls to define module- or singleton-specific Ninja
// definitions. For example:
//
-// package blah
+// package blah
//
-// import (
-// "blueprint"
-// )
+// import (
+// "blueprint"
+// )
//
-// var (
-// pctx = NewPackageContext("path/to/blah")
+// var (
+// pctx = NewPackageContext("path/to/blah")
//
-// myPrivateVar = pctx.StaticVariable("myPrivateVar", "abcdef")
-// MyExportedVar = pctx.StaticVariable("MyExportedVar", "$myPrivateVar 123456!")
+// myPrivateVar = pctx.StaticVariable("myPrivateVar", "abcdef")
+// MyExportedVar = pctx.StaticVariable("MyExportedVar", "$myPrivateVar 123456!")
//
-// SomeRule = pctx.StaticRule(...)
-// )
+// SomeRule = pctx.StaticRule(...)
+// )
//
-// // ...
+// // ...
//
-// func (m *MyModule) GenerateBuildActions(ctx blueprint.Module) {
-// ctx.Build(pctx, blueprint.BuildParams{
-// Rule: SomeRule,
-// Outputs: []string{"$myPrivateVar"},
-// })
-// }
+// func (m *MyModule) GenerateBuildActions(ctx blueprint.Module) {
+// ctx.Build(pctx, blueprint.BuildParams{
+// Rule: SomeRule,
+// Outputs: []string{"$myPrivateVar"},
+// })
+// }
type PackageContext interface {
Import(pkgPath string)
ImportAs(as, pkgPath string)
StaticVariable(name, value string) Variable
- VariableFunc(name string, f func(config interface{}) (string, error)) Variable
+ VariableFunc(name string, f func(ctx VariableFuncContext, config interface{}) (string, error)) Variable
VariableConfigMethod(name string, method interface{}) Variable
StaticPool(name string, params PoolParams) Pool
@@ -190,25 +190,25 @@
// "${pkg.Variable}", while the imported rules can simply be accessed as
// exported Go variables from the package. For example:
//
-// import (
-// "blueprint"
-// "foo/bar"
-// )
+// import (
+// "blueprint"
+// "foo/bar"
+// )
//
-// var pctx = NewPackagePath("blah")
+// var pctx = NewPackagePath("blah")
//
-// func init() {
-// pctx.Import("foo/bar")
-// }
+// func init() {
+// pctx.Import("foo/bar")
+// }
//
-// ...
+// ...
//
-// func (m *MyModule) GenerateBuildActions(ctx blueprint.Module) {
-// ctx.Build(pctx, blueprint.BuildParams{
-// Rule: bar.SomeRule,
-// Outputs: []string{"${bar.SomeVariable}"},
-// })
-// }
+// func (m *MyModule) GenerateBuildActions(ctx blueprint.Module) {
+// ctx.Build(pctx, blueprint.BuildParams{
+// Rule: bar.SomeRule,
+// Outputs: []string{"${bar.SomeVariable}"},
+// })
+// }
//
// Note that the local name used to refer to the package in Ninja variable names
// is derived from pkgPath by extracting the last path component. This differs
@@ -304,7 +304,7 @@
v.fullName_ = v.fullName(pkgNames)
}
-func (v *staticVariable) value(interface{}) (ninjaString, error) {
+func (v *staticVariable) value(VariableFuncContext, interface{}) (ninjaString, error) {
ninjaStr, err := parseNinjaString(v.pctx.scope, v.value_)
if err != nil {
err = fmt.Errorf("error parsing variable %s value: %s", v, err)
@@ -320,10 +320,30 @@
type variableFunc struct {
pctx *packageContext
name_ string
- value_ func(interface{}) (string, error)
+ value_ func(VariableFuncContext, interface{}) (string, error)
fullName_ string
}
+// VariableFuncContext is passed to VariableFunc functions.
+type VariableFuncContext interface {
+ // GlobWithDeps returns a list of files and directories that match the
+ // specified pattern but do not match any of the patterns in excludes.
+ // Any directories will have a '/' suffix. It also adds efficient
+ // dependencies to rerun the primary builder whenever a file matching
+ // the pattern as added or removed, without rerunning if a file that
+ // does not match the pattern is added to a searched directory.
+ GlobWithDeps(globPattern string, excludes []string) ([]string, error)
+}
+
+type variableFuncContext struct {
+ context *Context
+}
+
+func (v *variableFuncContext) GlobWithDeps(pattern string,
+ excludes []string) ([]string, error) {
+ return v.context.glob(pattern, excludes)
+}
+
// VariableFunc returns a Variable whose value is determined by a function that
// takes a config object as input and returns either the variable value or an
// error. It may only be called during a Go package's initialization - either
@@ -336,7 +356,7 @@
// reference other Ninja variables that are visible within the calling Go
// package.
func (p *packageContext) VariableFunc(name string,
- f func(config interface{}) (string, error)) Variable {
+ f func(ctx VariableFuncContext, config interface{}) (string, error)) Variable {
checkCalledFromInit()
@@ -382,7 +402,7 @@
methodValue := reflect.ValueOf(method)
validateVariableMethod(name, methodValue)
- fun := func(config interface{}) (string, error) {
+ fun := func(ctx VariableFuncContext, config interface{}) (string, error) {
result := methodValue.Call([]reflect.Value{reflect.ValueOf(config)})
resultStr := result[0].Interface().(string)
return resultStr, nil
@@ -420,8 +440,8 @@
v.fullName_ = v.fullName(pkgNames)
}
-func (v *variableFunc) value(config interface{}) (ninjaString, error) {
- value, err := v.value_(config)
+func (v *variableFunc) value(ctx VariableFuncContext, config interface{}) (ninjaString, error) {
+ value, err := v.value_(ctx, config)
if err != nil {
return nil, err
}
@@ -484,7 +504,7 @@
// Nothing to do, full name is known at initialization.
}
-func (v *argVariable) value(config interface{}) (ninjaString, error) {
+func (v *argVariable) value(ctx VariableFuncContext, config interface{}) (ninjaString, error) {
return nil, errVariableIsArg
}
diff --git a/parser/ast.go b/parser/ast.go
index cb311ee..ea774e6 100644
--- a/parser/ast.go
+++ b/parser/ast.go
@@ -60,6 +60,8 @@
Type string
TypePos scanner.Position
Map
+ //TODO(delmerico) make this a private field once ag/21588220 lands
+ Name__internal_only *string
}
func (m *Module) Copy() *Module {
@@ -86,6 +88,28 @@
func (m *Module) Pos() scanner.Position { return m.TypePos }
func (m *Module) End() scanner.Position { return m.Map.End() }
+func (m *Module) Name() string {
+ if m.Name__internal_only != nil {
+ return *m.Name__internal_only
+ }
+ for _, prop := range m.Properties {
+ if prop.Name == "name" {
+ if stringProp, ok := prop.Value.(*String); ok {
+ name := stringProp.Value
+ m.Name__internal_only = &name
+ } else {
+ name := prop.Value.String()
+ m.Name__internal_only = &name
+ }
+ }
+ }
+ if m.Name__internal_only == nil {
+ name := ""
+ m.Name__internal_only = &name
+ }
+ return *m.Name__internal_only
+}
+
// A Property is a name: value pair within a Map, which may be a top level Module.
type Property struct {
Name string
@@ -107,29 +131,6 @@
func (p *Property) Pos() scanner.Position { return p.NamePos }
func (p *Property) End() scanner.Position { return p.Value.End() }
-// A MapItem is a key: value pair within a Map, corresponding to map type, rather than a struct.
-type MapItem struct {
- ColonPos scanner.Position
- Key *String
- Value Expression
-}
-
-func (m *MapItem) Copy() *MapItem {
- ret := MapItem{
- ColonPos: m.ColonPos,
- Key: m.Key.Copy().(*String),
- Value: m.Value.Copy(),
- }
- return &ret
-}
-
-func (m *MapItem) String() string {
- return fmt.Sprintf("%s@%s: %s", m.Key, m.ColonPos, m.Value)
-}
-
-func (m *MapItem) Pos() scanner.Position { return m.Key.Pos() }
-func (m *MapItem) End() scanner.Position { return m.Value.End() }
-
// An Expression is a Value in a Property or Assignment. It can be a literal (String or Bool), a
// Map, a List, an Operator that combines two expressions of the same type, or a Variable that
// references and Assignment.
@@ -138,7 +139,7 @@
// Copy returns a copy of the Expression that will not affect the original if mutated
Copy() Expression
String() string
- // Type returns the underlying Type enum of the Expression if it were to be evalutated
+ // Type returns the underlying Type enum of the Expression if it were to be evaluated
Type() Type
// Eval returns an expression that is fully evaluated to a simple type (List, Map, String, or
// Bool). It will return the same object for every call to Eval().
@@ -267,7 +268,6 @@
LBracePos scanner.Position
RBracePos scanner.Position
Properties []*Property
- MapItems []*MapItem
}
func (x *Map) Pos() scanner.Position { return x.LBracePos }
@@ -279,36 +279,20 @@
for i := range x.Properties {
ret.Properties[i] = x.Properties[i].Copy()
}
- ret.MapItems = make([]*MapItem, len(x.MapItems))
- for i := range x.MapItems {
- ret.MapItems[i] = x.MapItems[i].Copy()
- }
return &ret
}
func (x *Map) Eval() Expression {
- if len(x.Properties) > 0 && len(x.MapItems) > 0 {
- panic("Cannot support both Properties and MapItems")
- }
return x
}
func (x *Map) String() string {
- var s string
- if len(x.MapItems) > 0 {
- mapStrings := make([]string, len(x.MapItems))
- for i, mapItem := range x.MapItems {
- mapStrings[i] = mapItem.String()
- }
- s = strings.Join(mapStrings, ", ")
- } else {
- propertyStrings := make([]string, len(x.Properties))
- for i, property := range x.Properties {
- propertyStrings[i] = property.String()
- }
- s = strings.Join(propertyStrings, ", ")
+ propertyStrings := make([]string, len(x.Properties))
+ for i, property := range x.Properties {
+ propertyStrings[i] = property.String()
}
- return fmt.Sprintf("@%s-%s{%s}", x.LBracePos, x.RBracePos, s)
+ return fmt.Sprintf("@%s-%s{%s}", x.LBracePos, x.RBracePos,
+ strings.Join(propertyStrings, ", "))
}
func (x *Map) Type() Type { return MapType }
@@ -338,6 +322,42 @@
return found
}
+// MovePropertyContents moves the contents of propertyName into property newLocation
+// If property newLocation doesn't exist, MovePropertyContents renames propertyName as newLocation.
+// Otherwise, MovePropertyContents only supports moving contents that are a List of String.
+func (x *Map) MovePropertyContents(propertyName string, newLocation string) (removed bool) {
+ oldProp, oldFound, _ := x.getPropertyImpl(propertyName)
+ newProp, newFound, _ := x.getPropertyImpl(newLocation)
+
+ // newLoc doesn't exist, simply renaming property
+ if oldFound && !newFound {
+ oldProp.Name = newLocation
+ return oldFound
+ }
+
+ if oldFound {
+ old, oldOk := oldProp.Value.(*List)
+ new, newOk := newProp.Value.(*List)
+ if oldOk && newOk {
+ toBeMoved := make([]string, len(old.Values)) //
+ for i, p := range old.Values {
+ toBeMoved[i] = p.(*String).Value
+ }
+
+ for _, moved := range toBeMoved {
+ RemoveStringFromList(old, moved)
+ AddStringToList(new, moved)
+ }
+ // oldProp should now be empty and needs to be deleted
+ x.RemoveProperty(oldProp.Name)
+ } else {
+ print(`MovePropertyContents currently only supports moving PropertyName
+ with List of Strings into an existing newLocation with List of Strings\n`)
+ }
+ }
+ return oldFound
+}
+
type List struct {
LBracePos scanner.Position
RBracePos scanner.Position
diff --git a/parser/modify.go b/parser/modify.go
index 3051f66..a28fbe6 100644
--- a/parser/modify.go
+++ b/parser/modify.go
@@ -56,6 +56,24 @@
return false
}
+func ReplaceStringsInList(list *List, replacements map[string]string) (replaced bool) {
+ modified := false
+ for i, v := range list.Values {
+ if v.Type() != StringType {
+ panic(fmt.Errorf("expected string in list, got %s", v.Type()))
+ }
+ if sv, ok := v.(*String); ok && replacements[sv.Value] != "" {
+ pos := list.Values[i].Pos()
+ list.Values[i] = &String{
+ LiteralPos: pos,
+ Value: replacements[sv.Value],
+ }
+ modified = true
+ }
+ }
+ return modified
+}
+
// A Patch represents a region of a text buffer to be replaced [Start, End) and its Replacement
type Patch struct {
Start, End int
diff --git a/parser/parser.go b/parser/parser.go
index bb8817e..63a6ac1 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -70,6 +70,7 @@
}
}()
+ p.next()
defs := p.parseDefinitions()
p.accept(scanner.EOF)
errs = p.errors
@@ -100,6 +101,7 @@
func ParseExpression(r io.Reader) (value Expression, errs []error) {
p := newParser(r, NewScope(nil))
+ p.next()
value = p.parseExpression()
p.accept(scanner.EOF)
errs = p.errors
@@ -124,7 +126,6 @@
}
p.scanner.Mode = scanner.ScanIdents | scanner.ScanInts | scanner.ScanStrings |
scanner.ScanRawStrings | scanner.ScanComments
- p.next()
return p
}
@@ -301,37 +302,6 @@
return
}
-func (p *parser) parseMapItemList() []*MapItem {
- var items []*MapItem
- // this is a map, not a struct, we only know we're at the end if we hit a '}'
- for p.tok != '}' {
- items = append(items, p.parseMapItem())
-
- if p.tok != ',' {
- // There was no comma, so the list is done.
- break
- }
- p.accept(',')
- }
- return items
-}
-
-func (p *parser) parseMapItem() *MapItem {
- keyExpression := p.parseExpression()
- if keyExpression.Type() != StringType {
- p.errorf("only strings are supported as map keys: %s (%s)", keyExpression.Type(), keyExpression.String())
- }
- key := keyExpression.(*String)
- p.accept(':')
- pos := p.scanner.Position
- value := p.parseExpression()
- return &MapItem{
- ColonPos: pos,
- Key: key,
- Value: value,
- }
-}
-
func (p *parser) parseProperty(isModule, compat bool) (property *Property) {
property = new(Property)
@@ -614,15 +584,7 @@
return nil
}
- var properties []*Property
- var mapItems []*MapItem
- // if the next item is an identifier, this is a property
- if p.tok == scanner.Ident {
- properties = p.parsePropertyList(false, false)
- } else {
- // otherwise, we assume that this is a map
- mapItems = p.parseMapItemList()
- }
+ properties := p.parsePropertyList(false, false)
rBracePos := p.scanner.Position
p.accept('}')
@@ -631,7 +593,6 @@
LBracePos: lBracePos,
RBracePos: rBracePos,
Properties: properties,
- MapItems: mapItems,
}
}
diff --git a/parser/parser_test.go b/parser/parser_test.go
index b32581e..b393792 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -194,121 +194,6 @@
nil,
},
- {`
- foo {
- stuff: {
- "key1": 1,
- "key2": 2,
- },
- }
- `,
- []Definition{
- &Module{
- Type: "foo",
- TypePos: mkpos(3, 2, 3),
- Map: Map{
- LBracePos: mkpos(7, 2, 7),
- RBracePos: mkpos(59, 7, 3),
- Properties: []*Property{
- {
- Name: "stuff",
- NamePos: mkpos(12, 3, 4),
- ColonPos: mkpos(17, 3, 9),
- Value: &Map{
- LBracePos: mkpos(19, 3, 11),
- RBracePos: mkpos(54, 6, 4),
- MapItems: []*MapItem{
- &MapItem{
- ColonPos: mkpos(33, 4, 13),
- Key: &String{
- LiteralPos: mkpos(25, 4, 5),
- Value: "key1",
- },
- Value: &Int64{
- LiteralPos: mkpos(33, 4, 13),
- Value: 1,
- Token: "1",
- },
- },
- &MapItem{
- ColonPos: mkpos(48, 5, 13),
- Key: &String{
- LiteralPos: mkpos(40, 5, 5),
- Value: "key2",
- },
- Value: &Int64{
- LiteralPos: mkpos(48, 5, 13),
- Value: 2,
- Token: "2",
- },
- },
- },
- },
- },
- },
- },
- },
- },
- nil,
- },
-
- {`
- foo {
- stuff: {
- "key1": {
- a: "abc",
- },
- },
- }
- `,
- []Definition{
- &Module{
- Type: "foo",
- TypePos: mkpos(3, 2, 3),
- Map: Map{
- LBracePos: mkpos(7, 2, 7),
- RBracePos: mkpos(65, 8, 3),
- Properties: []*Property{
- {
- Name: "stuff",
- NamePos: mkpos(12, 3, 4),
- ColonPos: mkpos(17, 3, 9),
- Value: &Map{
- LBracePos: mkpos(19, 3, 11),
- RBracePos: mkpos(60, 7, 4),
- MapItems: []*MapItem{
- &MapItem{
- ColonPos: mkpos(33, 4, 13),
- Key: &String{
- LiteralPos: mkpos(25, 4, 5),
- Value: "key1",
- },
- Value: &Map{
- LBracePos: mkpos(33, 4, 13),
- RBracePos: mkpos(54, 6, 5),
- Properties: []*Property{
- &Property{
- Name: "a",
- NamePos: mkpos(40, 5, 6),
- ColonPos: mkpos(41, 5, 7),
- Value: &String{
- LiteralPos: mkpos(43, 5, 9),
- Value: "abc",
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- },
- nil,
- },
-
{
`
foo {
@@ -1254,7 +1139,7 @@
Comments: []*Comment{
&Comment{
Comment: []string{"/* comment3", " comment4 */"},
- Slash: mkpos(32, 5, 3),
+ Slash: mkpos(32, 5, 3),
},
&Comment{
Comment: []string{"// comment5"},
@@ -1329,27 +1214,34 @@
}
}
-// TODO: Test error strings
-
-func TestMapParserError(t *testing.T) {
- input :=
- `
- foo {
- stuff: {
- 1: "value1",
- 2: "value2",
- },
- }
- `
- expectedErr := `<input>:4:6: only strings are supported as map keys: int64 ('\x01'@<input>:4:5)`
- _, errs := ParseAndEval("", bytes.NewBufferString(input), NewScope(nil))
- if len(errs) == 0 {
- t.Fatalf("Expected errors, got none.")
+func TestParserError(t *testing.T) {
+ testcases := []struct {
+ name string
+ input string
+ err string
+ }{
+ {
+ name: "invalid first token",
+ input: "\x00",
+ err: "invalid character NUL",
+ },
+ // TODO: test more parser errors
}
- for _, err := range errs {
- if expectedErr != err.Error() {
- t.Errorf("Unexpected err: %s", err)
- }
+
+ for _, tt := range testcases {
+ t.Run(tt.name, func(t *testing.T) {
+ r := bytes.NewBufferString(tt.input)
+ _, errs := ParseAndEval("", r, NewScope(nil))
+ if len(errs) == 0 {
+ t.Fatalf("missing expected error")
+ }
+ if g, w := errs[0], tt.err; !strings.Contains(g.Error(), w) {
+ t.Errorf("expected error %q, got %q", w, g)
+ }
+ for _, err := range errs[1:] {
+ t.Errorf("got unexpected extra error %q", err)
+ }
+ })
}
}
diff --git a/pathtools/lists_test.go b/pathtools/lists_test.go
index cce8786..87f24f9 100644
--- a/pathtools/lists_test.go
+++ b/pathtools/lists_test.go
@@ -4,7 +4,7 @@
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
-// http://www.apache.org/licenses/LICENSE-2.0
+// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
diff --git a/proptools/unpack.go b/proptools/unpack.go
index 28a68b5..e1d733c 100644
--- a/proptools/unpack.go
+++ b/proptools/unpack.go
@@ -27,12 +27,6 @@
const maxUnpackErrors = 10
-var (
- // Hard-coded list of allowlisted property names of type map. This is to limit use of maps to
- // where absolutely necessary.
- validMapProperties = []string{}
-)
-
type UnpackError struct {
Err error
Pos scanner.Position
@@ -51,18 +45,21 @@
// unpackContext keeps compound names and their values in a map. It is initialized from
// parsed properties.
type unpackContext struct {
- propertyMap map[string]*packedProperty
- validMapProperties map[string]bool
- errs []error
+ propertyMap map[string]*packedProperty
+ errs []error
}
// UnpackProperties populates the list of runtime values ("property structs") from the parsed properties.
// If a property a.b.c has a value, a field with the matching name in each runtime value is initialized
// from it. See PropertyNameForField for field and property name matching.
// For instance, if the input contains
-// { foo: "abc", bar: {x: 1},}
+//
+// { foo: "abc", bar: {x: 1},}
+//
// and a runtime value being has been declared as
-// var v struct { Foo string; Bar int }
+//
+// var v struct { Foo string; Bar int }
+//
// then v.Foo will be set to "abc" and v.Bar will be set to 1
// (cf. unpack_test.go for further examples)
//
@@ -74,19 +71,11 @@
// The same property can initialize fields in multiple runtime values. It is an error if any property
// value was not used to initialize at least one field.
func UnpackProperties(properties []*parser.Property, objects ...interface{}) (map[string]*parser.Property, []error) {
- return unpackProperties(properties, validMapProperties, objects...)
-}
-
-func unpackProperties(properties []*parser.Property, validMapProps []string, objects ...interface{}) (map[string]*parser.Property, []error) {
var unpackContext unpackContext
unpackContext.propertyMap = make(map[string]*packedProperty)
if !unpackContext.buildPropertyMap("", properties) {
return nil, unpackContext.errs
}
- unpackContext.validMapProperties = make(map[string]bool, len(validMapProps))
- for _, p := range validMapProps {
- unpackContext.validMapProperties[p] = true
- }
for _, obj := range objects {
valueObject := reflect.ValueOf(obj)
@@ -153,33 +142,7 @@
ctx.propertyMap[name] = &packedProperty{property, false}
switch propValue := property.Value.Eval().(type) {
case *parser.Map:
- // If this is a map and the values are not primitive types, we need to unroll it for further
- // mapping. Keys are limited to string types.
ctx.buildPropertyMap(name, propValue.Properties)
- if len(propValue.MapItems) == 0 {
- continue
- }
- items := propValue.MapItems
- keysType := items[0].Key.Type()
- valsAreBasic := primitiveType(items[0].Value.Type())
- if keysType != parser.StringType {
- ctx.addError(&UnpackError{Err: fmt.Errorf("complex key types are unsupported: %s", keysType)})
- return false
- } else if valsAreBasic {
- continue
- }
- itemProperties := make([]*parser.Property, len(items), len(items))
- for i, item := range items {
- itemProperties[i] = &parser.Property{
- Name: fmt.Sprintf("%s{value:%d}", property.Name, i),
- NamePos: property.NamePos,
- ColonPos: property.ColonPos,
- Value: item.Value,
- }
- }
- if !ctx.buildPropertyMap(prefix, itemProperties) {
- return false
- }
case *parser.List:
// If it is a list, unroll it unless its elements are of primitive type
// (no further mapping will be needed in that case, so we avoid cluttering
@@ -187,7 +150,7 @@
if len(propValue.Values) == 0 {
continue
}
- if primitiveType(propValue.Values[0].Type()) {
+ if t := propValue.Values[0].Type(); t == parser.StringType || t == parser.Int64Type || t == parser.BoolType {
continue
}
@@ -209,11 +172,6 @@
return len(ctx.errs) == nOldErrors
}
-// primitiveType returns whether typ is a primitive type
-func primitiveType(typ parser.Type) bool {
- return typ == parser.StringType || typ == parser.Int64Type || typ == parser.BoolType
-}
-
func fieldPath(prefix, fieldName string) string {
if prefix == "" {
return fieldName
@@ -265,15 +223,6 @@
switch kind := fieldValue.Kind(); kind {
case reflect.Bool, reflect.String, reflect.Struct, reflect.Slice:
// Do nothing
- case reflect.Map:
- // Restrict names of map properties that _can_ be set in bp files
- if _, ok := ctx.validMapProperties[propertyName]; !ok {
- if !HasTag(field, "blueprint", "mutated") {
- ctx.addError(&UnpackError{
- Err: fmt.Errorf("Uses of maps for properties must be allowlisted. %q is an unsupported use case", propertyName),
- })
- }
- }
case reflect.Interface:
if fieldValue.IsNil() {
panic(fmt.Errorf("field %s contains a nil interface", propertyName))
@@ -354,13 +303,6 @@
if len(ctx.errs) >= maxUnpackErrors {
return
}
- } else if fieldValue.Type().Kind() == reflect.Map {
- if unpackedValue, ok := ctx.unpackToMap(propertyName, property, fieldValue.Type()); ok {
- ExtendBasicType(fieldValue, unpackedValue, Append)
- }
- if len(ctx.errs) >= maxUnpackErrors {
- return
- }
} else {
unpackedValue, err := propertyToValue(fieldValue.Type(), property)
@@ -372,61 +314,6 @@
}
}
-// unpackToMap unpacks given parser.property into a go map of type mapType
-func (ctx *unpackContext) unpackToMap(mapName string, property *parser.Property, mapType reflect.Type) (reflect.Value, bool) {
- propValueAsMap, ok := property.Value.Eval().(*parser.Map)
- // Verify this property is a map
- if !ok {
- ctx.addError(&UnpackError{
- fmt.Errorf("can't assign %q value to map property %q", property.Value.Type(), property.Name),
- property.Value.Pos(),
- })
- return reflect.MakeMap(mapType), false
- }
- // And is not a struct
- if len(propValueAsMap.Properties) > 0 {
- ctx.addError(&UnpackError{
- fmt.Errorf("can't assign property to a map (%s) property %q", property.Value.Type(), property.Name),
- property.Value.Pos(),
- })
- return reflect.MakeMap(mapType), false
- }
-
- items := propValueAsMap.MapItems
- m := reflect.MakeMap(mapType)
- if len(items) == 0 {
- return m, true
- }
- keyConstructor := ctx.itemConstructor(items[0].Key.Type())
- keyType := mapType.Key()
- valueConstructor := ctx.itemConstructor(items[0].Value.Type())
- valueType := mapType.Elem()
-
- itemProperty := &parser.Property{NamePos: property.NamePos, ColonPos: property.ColonPos}
- for i, item := range items {
- itemProperty.Name = fmt.Sprintf("%s{key:%d}", mapName, i)
- itemProperty.Value = item.Key
- if packedProperty, ok := ctx.propertyMap[itemProperty.Name]; ok {
- packedProperty.used = true
- }
- keyValue, ok := itemValue(keyConstructor, itemProperty, keyType)
- if !ok {
- continue
- }
- itemProperty.Name = fmt.Sprintf("%s{value:%d}", mapName, i)
- itemProperty.Value = item.Value
- if packedProperty, ok := ctx.propertyMap[itemProperty.Name]; ok {
- packedProperty.used = true
- }
- value, ok := itemValue(valueConstructor, itemProperty, valueType)
- if ok {
- m.SetMapIndex(keyValue, value)
- }
- }
-
- return m, true
-}
-
// unpackSlice creates a value of a given slice type from the property which should be a list
func (ctx *unpackContext) unpackToSlice(
sliceName string, property *parser.Property, sliceType reflect.Type) (reflect.Value, bool) {
@@ -445,50 +332,11 @@
return value, true
}
- itemConstructor := ctx.itemConstructor(exprs[0].Type())
- itemType := sliceType.Elem()
-
- itemProperty := &parser.Property{NamePos: property.NamePos, ColonPos: property.ColonPos}
- for i, expr := range exprs {
- itemProperty.Name = sliceName + "[" + strconv.Itoa(i) + "]"
- itemProperty.Value = expr
- if packedProperty, ok := ctx.propertyMap[itemProperty.Name]; ok {
- packedProperty.used = true
- }
- if itemValue, ok := itemValue(itemConstructor, itemProperty, itemType); ok {
- value = reflect.Append(value, itemValue)
- }
- }
- return value, true
-}
-
-// constructItem is a function to construct a reflect.Value from given parser.Property of reflect.Type
-type constructItem func(*parser.Property, reflect.Type) (reflect.Value, bool)
-
-// itemValue creates a new item of type t with value determined by f
-func itemValue(f constructItem, property *parser.Property, t reflect.Type) (reflect.Value, bool) {
- isPtr := t.Kind() == reflect.Ptr
- if isPtr {
- t = t.Elem()
- }
- val, ok := f(property, t)
- if !ok {
- return val, ok
- }
- if isPtr {
- ptrValue := reflect.New(val.Type())
- ptrValue.Elem().Set(val)
- return ptrValue, true
- }
- return val, true
-}
-
-// itemConstructor returns a function to construct an item of typ
-func (ctx *unpackContext) itemConstructor(typ parser.Type) constructItem {
// The function to construct an item value depends on the type of list elements.
- switch typ {
+ var getItemFunc func(*parser.Property, reflect.Type) (reflect.Value, bool)
+ switch exprs[0].Type() {
case parser.BoolType, parser.StringType, parser.Int64Type:
- return func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
+ getItemFunc = func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
value, err := propertyToValue(t, property)
if err != nil {
ctx.addError(err)
@@ -497,26 +345,46 @@
return value, true
}
case parser.ListType:
- return func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
+ getItemFunc = func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
return ctx.unpackToSlice(property.Name, property, t)
}
case parser.MapType:
- return func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
- if t.Kind() == reflect.Map {
- return ctx.unpackToMap(property.Name, property, t)
- } else {
- itemValue := reflect.New(t).Elem()
- ctx.unpackToStruct(property.Name, itemValue)
- return itemValue, true
- }
+ getItemFunc = func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
+ itemValue := reflect.New(t).Elem()
+ ctx.unpackToStruct(property.Name, itemValue)
+ return itemValue, true
}
case parser.NotEvaluatedType:
- return func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
+ getItemFunc = func(property *parser.Property, t reflect.Type) (reflect.Value, bool) {
return reflect.New(t), false
}
default:
- panic(fmt.Errorf("bizarre property expression type: %v", typ))
+ panic(fmt.Errorf("bizarre property expression type: %v", exprs[0].Type()))
}
+
+ itemProperty := &parser.Property{NamePos: property.NamePos, ColonPos: property.ColonPos}
+ elemType := sliceType.Elem()
+ isPtr := elemType.Kind() == reflect.Ptr
+
+ for i, expr := range exprs {
+ itemProperty.Name = sliceName + "[" + strconv.Itoa(i) + "]"
+ itemProperty.Value = expr
+ if packedProperty, ok := ctx.propertyMap[itemProperty.Name]; ok {
+ packedProperty.used = true
+ }
+ if isPtr {
+ if itemValue, ok := getItemFunc(itemProperty, elemType.Elem()); ok {
+ ptrValue := reflect.New(itemValue.Type())
+ ptrValue.Elem().Set(itemValue)
+ value = reflect.Append(value, ptrValue)
+ }
+ } else {
+ if itemValue, ok := getItemFunc(itemProperty, elemType); ok {
+ value = reflect.Append(value, itemValue)
+ }
+ }
+ }
+ return value, true
}
// propertyToValue creates a value of a given value type from the property.
diff --git a/proptools/unpack_test.go b/proptools/unpack_test.go
index 5c6e3d0..7e2751d 100644
--- a/proptools/unpack_test.go
+++ b/proptools/unpack_test.go
@@ -129,82 +129,6 @@
},
{
- name: "map",
- input: `
- m {
- stuff: { "asdf": "jkl;", "qwert": "uiop"},
- empty: {},
- nested: {
- other_stuff: {},
- },
- }
- `,
- output: []interface{}{
- &struct {
- Stuff map[string]string
- Empty map[string]string
- Nil map[string]string
- NonString map[string]struct{ S string } `blueprint:"mutated"`
- Nested struct {
- Other_stuff map[string]string
- }
- }{
- Stuff: map[string]string{"asdf": "jkl;", "qwert": "uiop"},
- Empty: map[string]string{},
- Nil: nil,
- NonString: nil,
- Nested: struct{ Other_stuff map[string]string }{
- Other_stuff: map[string]string{},
- },
- },
- },
- },
-
- {
- name: "map with slice",
- input: `
- m {
- stuff: { "asdf": ["jkl;"], "qwert": []},
- empty: {},
- }
- `,
- output: []interface{}{
- &struct {
- Stuff map[string][]string
- Empty map[string][]string
- Nil map[string][]string
- NonString map[string]struct{ S string } `blueprint:"mutated"`
- }{
- Stuff: map[string][]string{"asdf": []string{"jkl;"}, "qwert": []string{}},
- Empty: map[string][]string{},
- Nil: nil,
- NonString: nil,
- },
- },
- },
-
- {
- name: "map with struct",
- input: `
- m {
- stuff: { "asdf": {s:"a"}},
- empty: {},
- }
- `,
- output: []interface{}{
- &struct {
- Stuff map[string]struct{ S string }
- Empty map[string]struct{ S string }
- Nil map[string]struct{ S string }
- }{
- Stuff: map[string]struct{ S string }{"asdf": struct{ S string }{"a"}},
- Empty: map[string]struct{ S string }{},
- Nil: nil,
- },
- },
- },
-
- {
name: "double nested",
input: `
m {
@@ -831,7 +755,7 @@
}
}
- _, errs = unpackProperties(module.Properties, []string{"stuff", "empty", "nil", "nested.other_stuff"}, output...)
+ _, errs = UnpackProperties(module.Properties, output...)
if len(errs) != 0 && len(testCase.errs) == 0 {
t.Errorf("test case: %s", testCase.input)
t.Errorf("unexpected unpack errors:")
@@ -1038,37 +962,6 @@
`<input>:3:16: can't assign string value to list property "map_list"`,
},
},
- {
- name: "invalid use of maps",
- input: `
- m {
- map: {"foo": "bar"},
- }
- `,
- output: []interface{}{
- &struct {
- Map map[string]string
- }{},
- },
- errors: []string{
- `<input>: Uses of maps for properties must be allowlisted. "map" is an unsupported use case`,
- },
- },
- {
- name: "invalid use of maps, not used in bp file",
- input: `
- m {
- }
- `,
- output: []interface{}{
- &struct {
- Map map[string]string
- }{},
- },
- errors: []string{
- `<input>: Uses of maps for properties must be allowlisted. "map" is an unsupported use case`,
- },
- },
}
for _, testCase := range testCases {
diff --git a/scope.go b/scope.go
index 3f39eb7..9585559 100644
--- a/scope.go
+++ b/scope.go
@@ -29,7 +29,7 @@
name() string // "foo"
fullName(pkgNames map[*packageContext]string) string // "pkg.foo" or "path.to.pkg.foo"
memoizeFullName(pkgNames map[*packageContext]string) // precompute fullName if desired
- value(config interface{}) (ninjaString, error)
+ value(ctx VariableFuncContext, config interface{}) (ninjaString, error)
String() string
}
@@ -373,7 +373,7 @@
// Nothing to do, full name is known at initialization.
}
-func (l *localVariable) value(interface{}) (ninjaString, error) {
+func (l *localVariable) value(VariableFuncContext, interface{}) (ninjaString, error) {
return l.value_, nil
}
diff --git a/visit_test.go b/visit_test.go
index 798e289..34e67d1 100644
--- a/visit_test.go
+++ b/visit_test.go
@@ -74,18 +74,18 @@
}
}
-// A
-// |
-// B
-// |\
-// C \
-// \|
-// D
-// |
-// E
-// / \
-// \ /
-// F
+// > A
+// > |
+// > B
+// > |\
+// > C \
+// > \|
+// > D
+// > |
+// > E
+// > / \
+// > \ /
+// > F
func setupVisitTest(t *testing.T) *Context {
ctx := NewContext()
ctx.RegisterModuleType("visit_module", newVisitModule)
@@ -98,22 +98,22 @@
name: "A",
visit: ["B"],
}
-
+
visit_module {
name: "B",
visit: ["C", "D"],
}
-
+
visit_module {
name: "C",
visit: ["D"],
}
-
+
visit_module {
name: "D",
visit: ["E"],
}
-
+
visit_module {
name: "E",
visit: ["F", "F"],
