Merge "Check UseRBE is set before replacing any template with the RE version." am: 3e0b9c031c am: 4e2977ee81
Original change: https://googleplex-android-review.googlesource.com/c/platform/build/soong/+/15747538
Change-Id: Ifa33cdad01dac8e81724d97f6373b745ff775ca9
diff --git a/Android.bp b/Android.bp
index c9ed95b..959cbf1 100644
--- a/Android.bp
+++ b/Android.bp
@@ -37,6 +37,7 @@
"blueprint-bootstrap",
"soong",
"soong-env",
+ "soong-shared",
],
srcs: [
"android/androidmk.go",
@@ -285,7 +286,6 @@
"java/support_libraries.go",
"java/system_modules.go",
"java/testing.go",
- "java/tradefed.go",
],
testSrcs: [
"java/app_test.go",
diff --git a/android/mutator.go b/android/mutator.go
index 31a3789..68ba6f4 100644
--- a/android/mutator.go
+++ b/android/mutator.go
@@ -161,6 +161,7 @@
CreateVariations(...string) []blueprint.Module
CreateLocalVariations(...string) []blueprint.Module
SetDependencyVariation(string)
+ SetDefaultDependencyVariation(*string)
AddVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string)
AddFarVariationDependencies([]blueprint.Variation, blueprint.DependencyTag, ...string)
AddInterVariantDependency(tag blueprint.DependencyTag, from, to blueprint.Module)
diff --git a/android/paths.go b/android/paths.go
index 8cc7057..0f20b84 100644
--- a/android/paths.go
+++ b/android/paths.go
@@ -1267,16 +1267,23 @@
// MaybeRel performs the same function as filepath.Rel, but reports errors to a PathContext, and returns false if
// targetPath is not inside basePath.
func MaybeRel(ctx PathContext, basePath string, targetPath string) (string, bool) {
+ rel, isRel, err := maybeRelErr(basePath, targetPath)
+ if err != nil {
+ reportPathError(ctx, err)
+ }
+ return rel, isRel
+}
+
+func maybeRelErr(basePath string, targetPath string) (string, bool, error) {
// filepath.Rel returns an error if one path is absolute and the other is not, handle that case first.
if filepath.IsAbs(basePath) != filepath.IsAbs(targetPath) {
- return "", false
+ return "", false, nil
}
rel, err := filepath.Rel(basePath, targetPath)
if err != nil {
- reportPathError(ctx, err)
- return "", false
+ return "", false, err
} else if rel == ".." || strings.HasPrefix(rel, "../") || strings.HasPrefix(rel, "/") {
- return "", false
+ return "", false, nil
}
- return rel, true
+ return rel, true, nil
}
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 2d0fac1..b3fccea 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -21,6 +21,8 @@
"github.com/google/blueprint"
"github.com/google/blueprint/proptools"
+
+ "android/soong/shared"
)
// RuleBuilder provides an alternative to ModuleContext.Rule and ModuleContext.Build to add a command line to the build
@@ -30,6 +32,8 @@
installs RuleBuilderInstalls
temporariesSet map[WritablePath]bool
restat bool
+ sbox bool
+ sboxOutDir WritablePath
missingDeps []string
}
@@ -73,11 +77,36 @@
}
// Restat marks the rule as a restat rule, which will be passed to ModuleContext.Rule in BuildParams.Restat.
+//
+// Restat is not compatible with Sbox()
func (r *RuleBuilder) Restat() *RuleBuilder {
+ if r.sbox {
+ panic("Restat() is not compatible with Sbox()")
+ }
r.restat = true
return r
}
+// Sbox marks the rule as needing to be wrapped by sbox. The WritablePath should point to the output
+// directory that sbox will wipe. It should not be written to by any other rule. sbox will ensure
+// that all outputs have been written, and will discard any output files that were not specified.
+//
+// Sbox is not compatible with Restat()
+func (r *RuleBuilder) Sbox(outputDir WritablePath) *RuleBuilder {
+ if r.sbox {
+ panic("Sbox() may not be called more than once")
+ }
+ if len(r.commands) > 0 {
+ panic("Sbox() may not be called after Command()")
+ }
+ if r.restat {
+ panic("Sbox() is not compatible with Restat()")
+ }
+ r.sbox = true
+ r.sboxOutDir = outputDir
+ return r
+}
+
// Install associates an output of the rule with an install location, which can be retrieved later using
// RuleBuilder.Installs.
func (r *RuleBuilder) Install(from Path, to string) {
@@ -88,7 +117,10 @@
// created by this method. That can be mutated through their methods in any order, as long as the mutations do not
// race with any call to Build.
func (r *RuleBuilder) Command() *RuleBuilderCommand {
- command := &RuleBuilderCommand{}
+ command := &RuleBuilderCommand{
+ sbox: r.sbox,
+ sboxOutDir: r.sboxOutDir,
+ }
r.commands = append(r.commands, command)
return command
}
@@ -120,12 +152,16 @@
// that are also outputs of another command in the same RuleBuilder are filtered out.
func (r *RuleBuilder) Inputs() Paths {
outputs := r.outputSet()
+ depFiles := r.depFileSet()
inputs := make(map[string]Path)
for _, c := range r.commands {
for _, input := range c.inputs {
- if _, isOutput := outputs[input.String()]; !isOutput {
- inputs[input.String()] = input
+ inputStr := input.String()
+ if _, isOutput := outputs[inputStr]; !isOutput {
+ if _, isDepFile := depFiles[inputStr]; !isDepFile {
+ inputs[input.String()] = input
+ }
}
}
}
@@ -171,6 +207,16 @@
return outputList
}
+func (r *RuleBuilder) depFileSet() map[string]WritablePath {
+ depFiles := make(map[string]WritablePath)
+ for _, c := range r.commands {
+ for _, depFile := range c.depFiles {
+ depFiles[depFile.String()] = depFile
+ }
+ }
+ return depFiles
+}
+
// DepFiles returns the list of paths that were passed to the RuleBuilderCommand methods that take depfile paths, such
// as RuleBuilderCommand.DepFile or RuleBuilderCommand.FlagWithDepFile.
func (r *RuleBuilder) DepFiles() WritablePaths {
@@ -237,9 +283,9 @@
var _ BuilderContext = SingletonContext(nil)
func (r *RuleBuilder) depFileMergerCmd(ctx PathContext, depFiles WritablePaths) *RuleBuilderCommand {
- return (&RuleBuilderCommand{}).
+ return r.Command().
Tool(ctx.Config().HostToolPath(ctx, "dep_fixer")).
- Flags(depFiles.Strings())
+ Inputs(depFiles.Paths())
}
// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
@@ -259,9 +305,6 @@
return
}
- tools := r.Tools()
- commands := r.Commands()
-
var depFile WritablePath
var depFormat blueprint.Deps
if depFiles := r.DepFiles(); len(depFiles) > 0 {
@@ -269,37 +312,76 @@
depFormat = blueprint.DepsGCC
if len(depFiles) > 1 {
// Add a command locally that merges all depfiles together into the first depfile.
- cmd := r.depFileMergerCmd(ctx, depFiles)
- commands = append(commands, string(cmd.buf))
- tools = append(tools, cmd.tools...)
+ r.depFileMergerCmd(ctx, depFiles)
+
+ if r.sbox {
+ // Check for Rel() errors, as all depfiles should be in the output dir
+ for _, path := range depFiles[1:] {
+ Rel(ctx, r.sboxOutDir.String(), path.String())
+ }
+ }
}
}
+ tools := r.Tools()
+ commands := r.Commands()
+ outputs := r.Outputs()
+
+ if len(commands) == 0 {
+ return
+ }
+ if len(outputs) == 0 {
+ panic("No outputs specified from any Commands")
+ }
+
+ commandString := strings.Join(proptools.NinjaEscapeList(commands), " && ")
+
+ if r.sbox {
+ sboxOutputs := make([]string, len(outputs))
+ for i, output := range outputs {
+ sboxOutputs[i] = "__SBOX_OUT_DIR__/" + Rel(ctx, r.sboxOutDir.String(), output.String())
+ }
+
+ commandString = proptools.ShellEscape(commandString)
+ if !strings.HasPrefix(commandString, `'`) {
+ commandString = `'` + commandString + `'`
+ }
+
+ sboxCmd := &RuleBuilderCommand{}
+ sboxCmd.Tool(ctx.Config().HostToolPath(ctx, "sbox")).
+ Flag("-c").Text(commandString).
+ Flag("--sandbox-path").Text(shared.TempDirForOutDir(PathForOutput(ctx).String())).
+ Flag("--output-root").Text(r.sboxOutDir.String())
+
+ if depFile != nil {
+ sboxCmd.Flag("--depfile-out").Text(depFile.String())
+ }
+
+ sboxCmd.Flags(sboxOutputs)
+
+ commandString = string(sboxCmd.buf)
+ tools = append(tools, sboxCmd.tools...)
+ }
+
// Ninja doesn't like multiple outputs when depfiles are enabled, move all but the first output to
// ImplicitOutputs. RuleBuilder never uses "$out", so the distinction between Outputs and ImplicitOutputs
// doesn't matter.
- var output WritablePath
- var implicitOutputs WritablePaths
- if outputs := r.Outputs(); len(outputs) > 0 {
- output = outputs[0]
- implicitOutputs = outputs[1:]
- }
+ output := outputs[0]
+ implicitOutputs := outputs[1:]
- if len(commands) > 0 {
- ctx.Build(pctx, BuildParams{
- Rule: ctx.Rule(pctx, name, blueprint.RuleParams{
- Command: strings.Join(proptools.NinjaEscapeList(commands), " && "),
- CommandDeps: tools.Strings(),
- Restat: r.restat,
- }),
- Implicits: r.Inputs(),
- Output: output,
- ImplicitOutputs: implicitOutputs,
- Depfile: depFile,
- Deps: depFormat,
- Description: desc,
- })
- }
+ ctx.Build(pctx, BuildParams{
+ Rule: ctx.Rule(pctx, name, blueprint.RuleParams{
+ Command: commandString,
+ CommandDeps: tools.Strings(),
+ Restat: r.restat,
+ }),
+ Implicits: r.Inputs(),
+ Output: output,
+ ImplicitOutputs: implicitOutputs,
+ Depfile: depFile,
+ Deps: depFormat,
+ Description: desc,
+ })
}
// RuleBuilderCommand is a builder for a command in a command line. It can be mutated by its methods to add to the
@@ -312,6 +394,28 @@
outputs WritablePaths
depFiles WritablePaths
tools Paths
+
+ sbox bool
+ sboxOutDir WritablePath
+}
+
+func (c *RuleBuilderCommand) addInput(path Path) string {
+ if c.sbox {
+ if rel, isRel, _ := maybeRelErr(c.sboxOutDir.String(), path.String()); isRel {
+ return "__SBOX_OUT_DIR__/" + rel
+ }
+ }
+ c.inputs = append(c.inputs, path)
+ return path.String()
+}
+
+func (c *RuleBuilderCommand) outputStr(path Path) string {
+ if c.sbox {
+ // Errors will be handled in RuleBuilder.Build where we have a context to report them
+ rel, _, _ := maybeRelErr(c.sboxOutDir.String(), path.String())
+ return "__SBOX_OUT_DIR__/" + rel
+ }
+ return path.String()
}
// Text adds the specified raw text to the command line. The text should not contain input or output paths or the
@@ -378,8 +482,7 @@
// Input adds the specified input path to the command line. The path will also be added to the dependencies returned by
// RuleBuilder.Inputs.
func (c *RuleBuilderCommand) Input(path Path) *RuleBuilderCommand {
- c.inputs = append(c.inputs, path)
- return c.Text(path.String())
+ return c.Text(c.addInput(path))
}
// Inputs adds the specified input paths to the command line, separated by spaces. The paths will also be added to the
@@ -394,14 +497,16 @@
// Implicit adds the specified input path to the dependencies returned by RuleBuilder.Inputs without modifying the
// command line.
func (c *RuleBuilderCommand) Implicit(path Path) *RuleBuilderCommand {
- c.inputs = append(c.inputs, path)
+ c.addInput(path)
return c
}
// Implicits adds the specified input paths to the dependencies returned by RuleBuilder.Inputs without modifying the
// command line.
func (c *RuleBuilderCommand) Implicits(paths Paths) *RuleBuilderCommand {
- c.inputs = append(c.inputs, paths...)
+ for _, path := range paths {
+ c.addInput(path)
+ }
return c
}
@@ -409,7 +514,7 @@
// RuleBuilder.Outputs.
func (c *RuleBuilderCommand) Output(path WritablePath) *RuleBuilderCommand {
c.outputs = append(c.outputs, path)
- return c.Text(path.String())
+ return c.Text(c.outputStr(path))
}
// Outputs adds the specified output paths to the command line, separated by spaces. The paths will also be added to
@@ -426,7 +531,7 @@
// commands in a single RuleBuilder then RuleBuilder.Build will add an extra command to merge the depfiles together.
func (c *RuleBuilderCommand) DepFile(path WritablePath) *RuleBuilderCommand {
c.depFiles = append(c.depFiles, path)
- return c.Text(path.String())
+ return c.Text(c.outputStr(path))
}
// ImplicitOutput adds the specified output path to the dependencies returned by RuleBuilder.Outputs without modifying
@@ -455,16 +560,18 @@
// FlagWithInput adds the specified flag and input path to the command line, with no separator between them. The path
// will also be added to the dependencies returned by RuleBuilder.Inputs.
func (c *RuleBuilderCommand) FlagWithInput(flag string, path Path) *RuleBuilderCommand {
- c.inputs = append(c.inputs, path)
- return c.Text(flag + path.String())
+ return c.Text(flag + c.addInput(path))
}
// FlagWithInputList adds the specified flag and input paths to the command line, with the inputs joined by sep
// and no separator between the flag and inputs. The input paths will also be added to the dependencies returned by
// RuleBuilder.Inputs.
func (c *RuleBuilderCommand) FlagWithInputList(flag string, paths Paths, sep string) *RuleBuilderCommand {
- c.inputs = append(c.inputs, paths...)
- return c.FlagWithList(flag, paths.Strings(), sep)
+ strs := make([]string, len(paths))
+ for i, path := range paths {
+ strs[i] = c.addInput(path)
+ }
+ return c.FlagWithList(flag, strs, sep)
}
// FlagForEachInput adds the specified flag joined with each input path to the command line. The input paths will also
@@ -481,14 +588,14 @@
// will also be added to the outputs returned by RuleBuilder.Outputs.
func (c *RuleBuilderCommand) FlagWithOutput(flag string, path WritablePath) *RuleBuilderCommand {
c.outputs = append(c.outputs, path)
- return c.Text(flag + path.String())
+ return c.Text(flag + c.outputStr(path))
}
// FlagWithDepFile adds the specified flag and depfile path to the command line, with no separator between them. The path
// will also be added to the outputs returned by RuleBuilder.Outputs.
func (c *RuleBuilderCommand) FlagWithDepFile(flag string, path WritablePath) *RuleBuilderCommand {
c.depFiles = append(c.depFiles, path)
- return c.Text(flag + path.String())
+ return c.Text(flag + c.outputStr(path))
}
// String returns the command line.
diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go
index 7bad025..d122a42 100644
--- a/android/rule_builder_test.go
+++ b/android/rule_builder_test.go
@@ -22,6 +22,10 @@
"reflect"
"strings"
"testing"
+
+ "github.com/google/blueprint"
+
+ "android/soong/shared"
)
func pathContext() PathContext {
@@ -234,8 +238,6 @@
}
func TestRuleBuilder(t *testing.T) {
- rule := NewRuleBuilder()
-
fs := map[string][]byte{
"dep_fixer": nil,
"input": nil,
@@ -249,73 +251,114 @@
ctx := PathContextForTesting(TestConfig("out", nil), fs)
- cmd := rule.Command().
- DepFile(PathForOutput(ctx, "DepFile")).
- Flag("Flag").
- FlagWithArg("FlagWithArg=", "arg").
- FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "depfile")).
- FlagWithInput("FlagWithInput=", PathForSource(ctx, "input")).
- FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "output")).
- Implicit(PathForSource(ctx, "Implicit")).
- ImplicitDepFile(PathForOutput(ctx, "ImplicitDepFile")).
- ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")).
- Input(PathForSource(ctx, "Input")).
- Output(PathForOutput(ctx, "Output")).
- Text("Text").
- Tool(PathForSource(ctx, "Tool"))
+ addCommands := func(rule *RuleBuilder) {
+ cmd := rule.Command().
+ DepFile(PathForOutput(ctx, "DepFile")).
+ Flag("Flag").
+ FlagWithArg("FlagWithArg=", "arg").
+ FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "depfile")).
+ FlagWithInput("FlagWithInput=", PathForSource(ctx, "input")).
+ FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "output")).
+ Implicit(PathForSource(ctx, "Implicit")).
+ ImplicitDepFile(PathForOutput(ctx, "ImplicitDepFile")).
+ ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")).
+ Input(PathForSource(ctx, "Input")).
+ Output(PathForOutput(ctx, "Output")).
+ Text("Text").
+ Tool(PathForSource(ctx, "Tool"))
- rule.Command().
- Text("command2").
- DepFile(PathForOutput(ctx, "depfile2")).
- Input(PathForSource(ctx, "input2")).
- Output(PathForOutput(ctx, "output2")).
- Tool(PathForSource(ctx, "tool2"))
+ rule.Command().
+ Text("command2").
+ DepFile(PathForOutput(ctx, "depfile2")).
+ Input(PathForSource(ctx, "input2")).
+ Output(PathForOutput(ctx, "output2")).
+ Tool(PathForSource(ctx, "tool2"))
- // Test updates to the first command after the second command has been started
- cmd.Text("after command2")
- // Test updating a command when the previous update did not replace the cmd variable
- cmd.Text("old cmd")
+ // Test updates to the first command after the second command has been started
+ cmd.Text("after command2")
+ // Test updating a command when the previous update did not replace the cmd variable
+ cmd.Text("old cmd")
- // Test a command that uses the output of a previous command as an input
- rule.Command().
- Text("command3").
- Input(PathForSource(ctx, "input3")).
- Input(PathForOutput(ctx, "output2")).
- Output(PathForOutput(ctx, "output3"))
-
- wantCommands := []string{
- "out/DepFile Flag FlagWithArg=arg FlagWithDepFile=out/depfile FlagWithInput=input FlagWithOutput=out/output Input out/Output Text Tool after command2 old cmd",
- "command2 out/depfile2 input2 out/output2 tool2",
- "command3 input3 out/output2 out/output3",
+ // Test a command that uses the output of a previous command as an input
+ rule.Command().
+ Text("command3").
+ Input(PathForSource(ctx, "input3")).
+ Input(PathForOutput(ctx, "output2")).
+ Output(PathForOutput(ctx, "output3"))
}
- wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer out/DepFile out/depfile out/ImplicitDepFile out/depfile2"
-
wantInputs := PathsForSource(ctx, []string{"Implicit", "Input", "input", "input2", "input3"})
wantOutputs := PathsForOutput(ctx, []string{"ImplicitOutput", "Output", "output", "output2", "output3"})
wantDepFiles := PathsForOutput(ctx, []string{"DepFile", "depfile", "ImplicitDepFile", "depfile2"})
wantTools := PathsForSource(ctx, []string{"Tool", "tool2"})
- if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) {
- t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g)
- }
+ t.Run("normal", func(t *testing.T) {
+ rule := NewRuleBuilder()
+ addCommands(rule)
- if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w {
- t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g)
- }
+ wantCommands := []string{
+ "out/DepFile Flag FlagWithArg=arg FlagWithDepFile=out/depfile FlagWithInput=input FlagWithOutput=out/output Input out/Output Text Tool after command2 old cmd",
+ "command2 out/depfile2 input2 out/output2 tool2",
+ "command3 input3 out/output2 out/output3",
+ }
- if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) {
- t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g)
- }
- if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) {
- t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g)
- }
- if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) {
- t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g)
- }
- if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) {
- t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g)
- }
+ wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer out/DepFile out/depfile out/ImplicitDepFile out/depfile2"
+
+ if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) {
+ t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g)
+ }
+
+ if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g)
+ }
+
+ if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w {
+ t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g)
+ }
+ })
+
+ t.Run("sbox", func(t *testing.T) {
+ rule := NewRuleBuilder().Sbox(PathForOutput(ctx))
+ addCommands(rule)
+
+ wantCommands := []string{
+ "__SBOX_OUT_DIR__/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_OUT_DIR__/depfile FlagWithInput=input FlagWithOutput=__SBOX_OUT_DIR__/output Input __SBOX_OUT_DIR__/Output Text Tool after command2 old cmd",
+ "command2 __SBOX_OUT_DIR__/depfile2 input2 __SBOX_OUT_DIR__/output2 tool2",
+ "command3 input3 __SBOX_OUT_DIR__/output2 __SBOX_OUT_DIR__/output3",
+ }
+
+ wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_OUT_DIR__/DepFile __SBOX_OUT_DIR__/depfile __SBOX_OUT_DIR__/ImplicitDepFile __SBOX_OUT_DIR__/depfile2"
+
+ if g, w := rule.Commands(), wantCommands; !reflect.DeepEqual(g, w) {
+ t.Errorf("\nwant rule.Commands() = %#v\n got %#v", w, g)
+ }
+
+ if g, w := rule.Inputs(), wantInputs; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Inputs() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.Outputs(), wantOutputs; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Outputs() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.DepFiles(), wantDepFiles; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.DepFiles() = %#v\n got %#v", w, g)
+ }
+ if g, w := rule.Tools(), wantTools; !reflect.DeepEqual(w, g) {
+ t.Errorf("\nwant rule.Tools() = %#v\n got %#v", w, g)
+ }
+
+ if g, w := rule.depFileMergerCmd(ctx, rule.DepFiles()).String(), wantDepMergerCommand; g != w {
+ t.Errorf("\nwant rule.depFileMergerCmd() = %#v\n got %#v", w, g)
+ }
+ })
}
func testRuleBuilderFactory() Module {
@@ -329,14 +372,19 @@
ModuleBase
properties struct {
Src string
+
+ Restat bool
+ Sbox bool
}
}
func (t *testRuleBuilderModule) GenerateAndroidBuildActions(ctx ModuleContext) {
in := PathForSource(ctx, t.properties.Src)
out := PathForModuleOut(ctx, ctx.ModuleName())
+ outDep := PathForModuleOut(ctx, ctx.ModuleName()+".d")
+ outDir := PathForModuleOut(ctx)
- testRuleBuilder_Build(ctx, in, out)
+ testRuleBuilder_Build(ctx, in, out, outDep, outDir, t.properties.Restat, t.properties.Sbox)
}
type testRuleBuilderSingleton struct{}
@@ -348,15 +396,23 @@
func (t *testRuleBuilderSingleton) GenerateBuildActions(ctx SingletonContext) {
in := PathForSource(ctx, "bar")
out := PathForOutput(ctx, "baz")
- testRuleBuilder_Build(ctx, in, out)
+ outDep := PathForOutput(ctx, "baz.d")
+ outDir := PathForOutput(ctx)
+ testRuleBuilder_Build(ctx, in, out, outDep, outDir, true, false)
}
-func testRuleBuilder_Build(ctx BuilderContext, in Path, out WritablePath) {
+func testRuleBuilder_Build(ctx BuilderContext, in Path, out, outDep, outDir WritablePath, restat, sbox bool) {
rule := NewRuleBuilder()
- rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out)
+ if sbox {
+ rule.Sbox(outDir)
+ }
- rule.Restat()
+ rule.Command().Tool(PathForSource(ctx, "cp")).Input(in).Output(out).ImplicitDepFile(outDep)
+
+ if restat {
+ rule.Restat()
+ }
rule.Build(pctx, ctx, "rule", "desc")
}
@@ -372,6 +428,12 @@
rule_builder_test {
name: "foo",
src: "bar",
+ restat: true,
+ }
+ rule_builder_test {
+ name: "foo_sbox",
+ src: "bar",
+ sbox: true,
}
`
@@ -391,9 +453,19 @@
_, errs = ctx.PrepareBuildActions(config)
FailIfErrored(t, errs)
- check := func(t *testing.T, params TestingBuildParams, wantOutput string) {
- if len(params.RuleParams.CommandDeps) != 1 || params.RuleParams.CommandDeps[0] != "cp" {
- t.Errorf("want RuleParams.CommandDeps = [%q], got %q", "cp", params.RuleParams.CommandDeps)
+ check := func(t *testing.T, params TestingBuildParams, wantCommand, wantOutput, wantDepfile string, wantRestat bool, extraCmdDeps []string) {
+ t.Helper()
+ if params.RuleParams.Command != wantCommand {
+ t.Errorf("\nwant RuleParams.Command = %q\n got %q", wantCommand, params.RuleParams.Command)
+ }
+
+ wantDeps := append([]string{"cp"}, extraCmdDeps...)
+ if !reflect.DeepEqual(params.RuleParams.CommandDeps, wantDeps) {
+ t.Errorf("\nwant RuleParams.CommandDeps = %q\n got %q", wantDeps, params.RuleParams.CommandDeps)
+ }
+
+ if params.RuleParams.Restat != wantRestat {
+ t.Errorf("want RuleParams.Restat = %v, got %v", wantRestat, params.RuleParams.Restat)
}
if len(params.Implicits) != 1 || params.Implicits[0].String() != "bar" {
@@ -404,17 +476,40 @@
t.Errorf("want Output = %q, got %q", wantOutput, params.Output)
}
- if !params.RuleParams.Restat {
- t.Errorf("want RuleParams.Restat = true, got %v", params.RuleParams.Restat)
+ if len(params.ImplicitOutputs) != 0 {
+ t.Errorf("want ImplicitOutputs = [], got %q", params.ImplicitOutputs.Strings())
+ }
+
+ if params.Depfile.String() != wantDepfile {
+ t.Errorf("want Depfile = %q, got %q", wantDepfile, params.Depfile)
+ }
+
+ if params.Deps != blueprint.DepsGCC {
+ t.Errorf("want Deps = %q, got %q", blueprint.DepsGCC, params.Deps)
}
}
t.Run("module", func(t *testing.T) {
+ outFile := filepath.Join(buildDir, ".intermediates", "foo", "foo")
check(t, ctx.ModuleForTests("foo", "").Rule("rule"),
- filepath.Join(buildDir, ".intermediates", "foo", "foo"))
+ "cp bar "+outFile,
+ outFile, outFile+".d", true, nil)
+ })
+ t.Run("sbox", func(t *testing.T) {
+ outDir := filepath.Join(buildDir, ".intermediates", "foo_sbox")
+ outFile := filepath.Join(outDir, "foo_sbox")
+ depFile := filepath.Join(outDir, "foo_sbox.d")
+ sbox := filepath.Join(buildDir, "host", config.PrebuiltOS(), "bin/sbox")
+ sandboxPath := shared.TempDirForOutDir(buildDir)
+
+ cmd := sbox + ` -c 'cp bar __SBOX_OUT_DIR__/foo_sbox' --sandbox-path ` + sandboxPath + " --output-root " + outDir + " --depfile-out " + depFile + " __SBOX_OUT_DIR__/foo_sbox"
+
+ check(t, ctx.ModuleForTests("foo_sbox", "").Rule("rule"),
+ cmd, outFile, depFile, false, []string{sbox})
})
t.Run("singleton", func(t *testing.T) {
+ outFile := filepath.Join(buildDir, "baz")
check(t, ctx.SingletonForTests("rule_builder_test").Rule("rule"),
- filepath.Join(buildDir, "baz"))
+ "cp bar "+outFile, outFile, outFile+".d", true, nil)
})
}
diff --git a/android/util.go b/android/util.go
index f9dce6f..b17422d 100644
--- a/android/util.go
+++ b/android/util.go
@@ -243,3 +243,36 @@
}
return strings.HasPrefix(str, pat[:i]) && strings.HasSuffix(str, pat[i+1:])
}
+
+// ShardPaths takes a Paths, and returns a slice of Paths where each one has at most shardSize paths.
+func ShardPaths(paths Paths, shardSize int) []Paths {
+ if len(paths) == 0 {
+ return nil
+ }
+ ret := make([]Paths, 0, (len(paths)+shardSize-1)/shardSize)
+ for len(paths) > shardSize {
+ ret = append(ret, paths[0:shardSize])
+ paths = paths[shardSize:]
+ }
+ if len(paths) > 0 {
+ ret = append(ret, paths)
+ }
+ return ret
+}
+
+// ShardStrings takes a slice of strings, and returns a slice of slices of strings where each one has at most shardSize
+// elements.
+func ShardStrings(s []string, shardSize int) [][]string {
+ if len(s) == 0 {
+ return nil
+ }
+ ret := make([][]string, 0, (len(s)+shardSize-1)/shardSize)
+ for len(s) > shardSize {
+ ret = append(ret, s[0:shardSize])
+ s = s[shardSize:]
+ }
+ if len(s) > 0 {
+ ret = append(ret, s)
+ }
+ return ret
+}
diff --git a/android/util_test.go b/android/util_test.go
index 2e5eb07..7f0d331 100644
--- a/android/util_test.go
+++ b/android/util_test.go
@@ -404,3 +404,102 @@
// b = ["foo" "bar"]
// c = ["foo" "baz"]
}
+
+func Test_Shard(t *testing.T) {
+ type args struct {
+ strings []string
+ shardSize int
+ }
+ tests := []struct {
+ name string
+ args args
+ want [][]string
+ }{
+ {
+ name: "empty",
+ args: args{
+ strings: nil,
+ shardSize: 1,
+ },
+ want: [][]string(nil),
+ },
+ {
+ name: "single shard",
+ args: args{
+ strings: []string{"a", "b"},
+ shardSize: 2,
+ },
+ want: [][]string{{"a", "b"}},
+ },
+ {
+ name: "single short shard",
+ args: args{
+ strings: []string{"a", "b"},
+ shardSize: 3,
+ },
+ want: [][]string{{"a", "b"}},
+ },
+ {
+ name: "shard per input",
+ args: args{
+ strings: []string{"a", "b", "c"},
+ shardSize: 1,
+ },
+ want: [][]string{{"a"}, {"b"}, {"c"}},
+ },
+ {
+ name: "balanced shards",
+ args: args{
+ strings: []string{"a", "b", "c", "d"},
+ shardSize: 2,
+ },
+ want: [][]string{{"a", "b"}, {"c", "d"}},
+ },
+ {
+ name: "unbalanced shards",
+ args: args{
+ strings: []string{"a", "b", "c"},
+ shardSize: 2,
+ },
+ want: [][]string{{"a", "b"}, {"c"}},
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Run("strings", func(t *testing.T) {
+ if got := ShardStrings(tt.args.strings, tt.args.shardSize); !reflect.DeepEqual(got, tt.want) {
+ t.Errorf("ShardStrings(%v, %v) = %v, want %v",
+ tt.args.strings, tt.args.shardSize, got, tt.want)
+ }
+ })
+
+ t.Run("paths", func(t *testing.T) {
+ stringsToPaths := func(strings []string) Paths {
+ if strings == nil {
+ return nil
+ }
+ paths := make(Paths, len(strings))
+ for i, s := range strings {
+ paths[i] = PathForTesting(s)
+ }
+ return paths
+ }
+
+ paths := stringsToPaths(tt.args.strings)
+
+ var want []Paths
+ if sWant := tt.want; sWant != nil {
+ want = make([]Paths, len(sWant))
+ for i, w := range sWant {
+ want[i] = stringsToPaths(w)
+ }
+ }
+
+ if got := ShardPaths(paths, tt.args.shardSize); !reflect.DeepEqual(got, want) {
+ t.Errorf("ShardPaths(%v, %v) = %v, want %v",
+ paths, tt.args.shardSize, got, want)
+ }
+ })
+ })
+ }
+}
diff --git a/apex/apex.go b/apex/apex.go
index 4dca961..8ee8035 100644
--- a/apex/apex.go
+++ b/apex/apex.go
@@ -530,6 +530,17 @@
a.properties.Multilib.Prefer32.Binaries, target.String(),
a.getImageVariation(config))
}
+
+ if strings.HasPrefix(ctx.ModuleName(), "com.android.runtime") && target.Os.Class == android.Device {
+ for _, sanitizer := range ctx.Config().SanitizeDevice() {
+ if sanitizer == "hwaddress" {
+ addDependenciesForNativeModules(ctx,
+ []string{"libclang_rt.hwasan-aarch64-android"},
+ nil, target.String(), a.getImageVariation(config))
+ break
+ }
+ }
+ }
}
}
diff --git a/cc/binary.go b/cc/binary.go
index 99fb795..2a6ceb8 100644
--- a/cc/binary.go
+++ b/cc/binary.go
@@ -444,8 +444,8 @@
// Bionic binaries (e.g. linker) is installed to the bootstrap subdirectory.
// The original path becomes a symlink to the corresponding file in the
// runtime APEX.
- if isBionic(ctx.baseModuleName()) && ctx.Arch().Native && ctx.apexName() == "" && !ctx.inRecovery() {
- if ctx.Device() {
+ if installToBootstrap(ctx.baseModuleName(), ctx.Config()) && ctx.Arch().Native && ctx.apexName() == "" && !ctx.inRecovery() {
+ if ctx.Device() && isBionic(ctx.baseModuleName()) {
binary.installSymlinkToRuntimeApex(ctx, file)
}
binary.baseInstaller.subDir = "bootstrap"
diff --git a/cc/cc.go b/cc/cc.go
index 0b39634..49dce18 100644
--- a/cc/cc.go
+++ b/cc/cc.go
@@ -51,6 +51,8 @@
ctx.TopDown("hwasan_deps", sanitizerDepsMutator(hwasan))
ctx.BottomUp("hwasan", sanitizerMutator(hwasan)).Parallel()
+ // cfi mutator shouldn't run before sanitizers that return true for
+ // incompatibleWithCfi()
ctx.TopDown("cfi_deps", sanitizerDepsMutator(cfi))
ctx.BottomUp("cfi", sanitizerMutator(cfi)).Parallel()
@@ -239,6 +241,7 @@
type ModuleContextIntf interface {
static() bool
staticBinary() bool
+ header() bool
toolchain() config.Toolchain
useSdk() bool
sdkVersion() string
@@ -605,6 +608,9 @@
if library, ok := c.linker.(*libraryDecorator); ok {
return len(library.Properties.Stubs.Versions) > 0
}
+ if library, ok := c.linker.(*prebuiltLibraryLinker); ok {
+ return len(library.Properties.Stubs.Versions) > 0
+ }
return false
}
@@ -624,6 +630,13 @@
return false
}
+func installToBootstrap(name string, config android.Config) bool {
+ if name == "libclang_rt.hwasan-aarch64-android" {
+ return inList("hwaddress", config.SanitizeDevice())
+ }
+ return isBionic(name)
+}
+
type baseModuleContext struct {
android.BaseContext
moduleContextImpl
@@ -661,6 +674,10 @@
return ctx.mod.staticBinary()
}
+func (ctx *moduleContextImpl) header() bool {
+ return ctx.mod.header()
+}
+
func (ctx *moduleContextImpl) useSdk() bool {
if ctx.ctx.Device() && !ctx.useVndk() && !ctx.inRecovery() && !ctx.ctx.Fuchsia() {
return String(ctx.mod.Properties.Sdk_version) != ""
@@ -1924,6 +1941,15 @@
return false
}
+func (c *Module) header() bool {
+ if h, ok := c.linker.(interface {
+ header() bool
+ }); ok {
+ return h.header()
+ }
+ return false
+}
+
func (c *Module) getMakeLinkType() string {
if c.useVndk() {
if inList(c.Name(), vndkCoreLibraries) || inList(c.Name(), vndkSpLibraries) || inList(c.Name(), llndkLibraries) {
diff --git a/cc/config/global.go b/cc/config/global.go
index fde5cab..926b668 100644
--- a/cc/config/global.go
+++ b/cc/config/global.go
@@ -123,7 +123,7 @@
// prebuilts/clang default settings.
ClangDefaultBase = "prebuilts/clang/host"
- ClangDefaultVersion = "clang-r353983c"
+ ClangDefaultVersion = "clang-r353983c1"
ClangDefaultShortVersion = "9.0.3"
// Directories with warnings from Android.bp files.
diff --git a/cc/library.go b/cc/library.go
index f93f5af..39f7a72 100644
--- a/cc/library.go
+++ b/cc/library.go
@@ -938,12 +938,12 @@
library.baseInstaller.subDir += "-" + vndkVersion
}
}
- } else if len(library.Properties.Stubs.Versions) > 0 && android.DirectlyInAnyApex(ctx, ctx.ModuleName()) {
+ } else if len(library.Properties.Stubs.Versions) > 0 {
// Bionic libraries (e.g. libc.so) is installed to the bootstrap subdirectory.
// The original path becomes a symlink to the corresponding file in the
// runtime APEX.
- if isBionic(ctx.baseModuleName()) && !library.buildStubs() && ctx.Arch().Native && !ctx.inRecovery() {
- if ctx.Device() {
+ if installToBootstrap(ctx.baseModuleName(), ctx.Config()) && !library.buildStubs() && ctx.Arch().Native && !ctx.inRecovery() {
+ if ctx.Device() && isBionic(ctx.baseModuleName()) {
library.installSymlinkToRuntimeApex(ctx, file)
}
library.baseInstaller.subDir = "bootstrap"
diff --git a/cc/llndk_library.go b/cc/llndk_library.go
index 56ef2b6..52c58eb 100644
--- a/cc/llndk_library.go
+++ b/cc/llndk_library.go
@@ -221,7 +221,7 @@
&library.MutatedProperties,
&library.flagExporter.Properties)
- android.InitAndroidArchModule(module, android.DeviceSupported, android.MultilibBoth)
+ module.Init()
return module
}
diff --git a/cc/makevars.go b/cc/makevars.go
index aa6fdea..b03e170 100644
--- a/cc/makevars.go
+++ b/cc/makevars.go
@@ -95,7 +95,20 @@
ctx.Strict("VNDK_CORE_LIBRARIES", strings.Join(vndkCoreLibraries, " "))
ctx.Strict("VNDK_SAMEPROCESS_LIBRARIES", strings.Join(vndkSpLibraries, " "))
- ctx.Strict("LLNDK_LIBRARIES", strings.Join(llndkLibraries, " "))
+
+ // Make uses LLNDK_LIBRARIES to determine which libraries to install.
+ // HWASAN is only part of the LL-NDK in builds in which libc depends on HWASAN.
+ // Therefore, by removing the library here, we cause it to only be installed if libc
+ // depends on it.
+ installedLlndkLibraries := []string{}
+ for _, lib := range llndkLibraries {
+ if strings.HasPrefix(lib, "libclang_rt.hwasan-") {
+ continue
+ }
+ installedLlndkLibraries = append(installedLlndkLibraries, lib)
+ }
+ ctx.Strict("LLNDK_LIBRARIES", strings.Join(installedLlndkLibraries, " "))
+
ctx.Strict("VNDK_PRIVATE_LIBRARIES", strings.Join(vndkPrivateLibraries, " "))
ctx.Strict("VNDK_USING_CORE_VARIANT_LIBRARIES", strings.Join(vndkUsingCoreVariantLibraries, " "))
diff --git a/cc/sanitize.go b/cc/sanitize.go
index b7a36a6..c1b055a 100644
--- a/cc/sanitize.go
+++ b/cc/sanitize.go
@@ -120,6 +120,10 @@
}
}
+func (t sanitizerType) incompatibleWithCfi() bool {
+ return t == asan || t == hwasan
+}
+
type SanitizeProperties struct {
// enable AddressSanitizer, ThreadSanitizer, or UndefinedBehaviorSanitizer
Sanitize struct {
@@ -543,16 +547,18 @@
}
func (sanitize *sanitize) AndroidMk(ctx AndroidMkContext, ret *android.AndroidMkData) {
- // Add a suffix for CFI-enabled static libraries to allow surfacing both to make without a
- // name conflict.
- if ret.Class == "STATIC_LIBRARIES" && Bool(sanitize.Properties.Sanitize.Cfi) {
- ret.SubName += ".cfi"
- }
- if ret.Class == "STATIC_LIBRARIES" && Bool(sanitize.Properties.Sanitize.Hwaddress) {
- ret.SubName += ".hwasan"
- }
- if ret.Class == "STATIC_LIBRARIES" && Bool(sanitize.Properties.Sanitize.Scs) {
- ret.SubName += ".scs"
+ // Add a suffix for cfi/hwasan/scs-enabled static/header libraries to allow surfacing
+ // both the sanitized and non-sanitized variants to make without a name conflict.
+ if ret.Class == "STATIC_LIBRARIES" || ret.Class == "HEADER_LIBRARIES" {
+ if Bool(sanitize.Properties.Sanitize.Cfi) {
+ ret.SubName += ".cfi"
+ }
+ if Bool(sanitize.Properties.Sanitize.Hwaddress) {
+ ret.SubName += ".hwasan"
+ }
+ if Bool(sanitize.Properties.Sanitize.Scs) {
+ ret.SubName += ".scs"
+ }
}
}
@@ -841,7 +847,7 @@
{Mutator: "image", Variation: c.imageVariation()},
{Mutator: "arch", Variation: mctx.Target().String()},
}, staticDepTag, runtimeLibrary)
- } else if !c.static() {
+ } else if !c.static() && !c.header() {
// dynamic executable and shared libs get shared runtime libs
mctx.AddFarVariationDependencies([]blueprint.Variation{
{Mutator: "link", Variation: "shared"},
@@ -870,95 +876,68 @@
modules := mctx.CreateVariations(t.variationName())
modules[0].(*Module).sanitize.SetSanitizer(t, true)
} else if c.sanitize.isSanitizerEnabled(t) || c.sanitize.Properties.SanitizeDep {
- // Save original sanitizer status before we assign values to variant
- // 0 as that overwrites the original.
isSanitizerEnabled := c.sanitize.isSanitizerEnabled(t)
+ if mctx.Device() && t.incompatibleWithCfi() {
+ // TODO: Make sure that cfi mutator runs "after" any of the sanitizers that
+ // are incompatible with cfi
+ c.sanitize.SetSanitizer(cfi, false)
+ }
+ if c.static() || c.header() || t == asan {
+ // Static and header libs are split into non-sanitized and sanitized variants.
+ // Shared libs are not split. However, for asan, we split even for shared
+ // libs because a library sanitized for asan can't be linked from a library
+ // that isn't sanitized for asan.
+ //
+ // Note for defaultVariation: since we don't split for shared libs but for static/header
+ // libs, it is possible for the sanitized variant of a static/header lib to depend
+ // on non-sanitized variant of a shared lib. Such unfulfilled variation causes an
+ // error when the module is split. defaultVariation is the name of the variation that
+ // will be used when such a dangling dependency occurs during the split of the current
+ // module. By setting it to the name of the sanitized variation, the dangling dependency
+ // is redirected to the sanitized variant of the dependent module.
+ defaultVariation := t.variationName()
+ mctx.SetDefaultDependencyVariation(&defaultVariation)
+ modules := mctx.CreateVariations("", t.variationName())
+ modules[0].(*Module).sanitize.SetSanitizer(t, false)
+ modules[1].(*Module).sanitize.SetSanitizer(t, true)
+ modules[0].(*Module).sanitize.Properties.SanitizeDep = false
+ modules[1].(*Module).sanitize.Properties.SanitizeDep = false
- modules := mctx.CreateVariations("", t.variationName())
- modules[0].(*Module).sanitize.SetSanitizer(t, false)
- modules[1].(*Module).sanitize.SetSanitizer(t, true)
-
- modules[0].(*Module).sanitize.Properties.SanitizeDep = false
- modules[1].(*Module).sanitize.Properties.SanitizeDep = false
-
- // We don't need both variants active for anything but CFI-enabled
- // target static libraries, so suppress the appropriate variant in
- // all other cases.
- if t == cfi {
+ // For cfi/scs/hwasan, we can export both sanitized and un-sanitized variants
+ // to Make, because the sanitized version has a different suffix in name.
+ // For other types of sanitizers, suppress the variation that is disabled.
+ if t != cfi && t != scs && t != hwasan {
+ if isSanitizerEnabled {
+ modules[0].(*Module).Properties.PreventInstall = true
+ modules[0].(*Module).Properties.HideFromMake = true
+ } else {
+ modules[1].(*Module).Properties.PreventInstall = true
+ modules[1].(*Module).Properties.HideFromMake = true
+ }
+ }
+ // Export the static lib name to make
if c.static() {
- if !mctx.Device() {
- if isSanitizerEnabled {
- modules[0].(*Module).Properties.PreventInstall = true
- modules[0].(*Module).Properties.HideFromMake = true
+ if t == cfi {
+ appendStringSync(c.Name(), cfiStaticLibs(mctx.Config()), &cfiStaticLibsMutex)
+ } else if t == hwasan {
+ if c.useVndk() {
+ appendStringSync(c.Name(), hwasanVendorStaticLibs(mctx.Config()),
+ &hwasanStaticLibsMutex)
} else {
- modules[1].(*Module).Properties.PreventInstall = true
- modules[1].(*Module).Properties.HideFromMake = true
+ appendStringSync(c.Name(), hwasanStaticLibs(mctx.Config()),
+ &hwasanStaticLibsMutex)
}
- } else {
- cfiStaticLibs := cfiStaticLibs(mctx.Config())
+ }
+ }
+ } else {
+ // Shared libs are not split. Only the sanitized variant is created.
+ modules := mctx.CreateVariations(t.variationName())
+ modules[0].(*Module).sanitize.SetSanitizer(t, true)
+ modules[0].(*Module).sanitize.Properties.SanitizeDep = false
- cfiStaticLibsMutex.Lock()
- *cfiStaticLibs = append(*cfiStaticLibs, c.Name())
- cfiStaticLibsMutex.Unlock()
- }
- } else {
- modules[0].(*Module).Properties.PreventInstall = true
- modules[0].(*Module).Properties.HideFromMake = true
- }
- } else if t == asan {
- if mctx.Device() {
- // CFI and ASAN are currently mutually exclusive so disable
- // CFI if this is an ASAN variant.
- modules[1].(*Module).sanitize.Properties.InSanitizerDir = true
- modules[1].(*Module).sanitize.SetSanitizer(cfi, false)
- }
- if isSanitizerEnabled {
- modules[0].(*Module).Properties.PreventInstall = true
- modules[0].(*Module).Properties.HideFromMake = true
- } else {
- modules[1].(*Module).Properties.PreventInstall = true
- modules[1].(*Module).Properties.HideFromMake = true
- }
- } else if t == scs {
- // We don't currently link any static libraries built with make into
- // libraries built with SCS, so we don't need logic for propagating
- // SCSness of dependencies into make.
- if !c.static() {
- if isSanitizerEnabled {
- modules[0].(*Module).Properties.PreventInstall = true
- modules[0].(*Module).Properties.HideFromMake = true
- } else {
- modules[1].(*Module).Properties.PreventInstall = true
- modules[1].(*Module).Properties.HideFromMake = true
- }
- }
- } else if t == hwasan {
- if mctx.Device() {
- // CFI and HWASAN are currently mutually exclusive so disable
- // CFI if this is an HWASAN variant.
- modules[1].(*Module).sanitize.SetSanitizer(cfi, false)
- }
-
- if c.static() {
- if c.useVndk() {
- hwasanVendorStaticLibs := hwasanVendorStaticLibs(mctx.Config())
- hwasanStaticLibsMutex.Lock()
- *hwasanVendorStaticLibs = append(*hwasanVendorStaticLibs, c.Name())
- hwasanStaticLibsMutex.Unlock()
- } else {
- hwasanStaticLibs := hwasanStaticLibs(mctx.Config())
- hwasanStaticLibsMutex.Lock()
- *hwasanStaticLibs = append(*hwasanStaticLibs, c.Name())
- hwasanStaticLibsMutex.Unlock()
- }
- } else {
- if isSanitizerEnabled {
- modules[0].(*Module).Properties.PreventInstall = true
- modules[0].(*Module).Properties.HideFromMake = true
- } else {
- modules[1].(*Module).Properties.PreventInstall = true
- modules[1].(*Module).Properties.HideFromMake = true
- }
+ // locate the asan libraries under /data/asan
+ if mctx.Device() && t == asan && isSanitizerEnabled {
+ modules[0].(*Module).sanitize.Properties.InSanitizerDir = true
}
}
}
@@ -994,6 +973,12 @@
}).(*[]string)
}
+func appendStringSync(item string, list *[]string, mutex *sync.Mutex) {
+ mutex.Lock()
+ *list = append(*list, item)
+ mutex.Unlock()
+}
+
func enableMinimalRuntime(sanitize *sanitize) bool {
if !Bool(sanitize.Properties.Sanitize.Address) &&
!Bool(sanitize.Properties.Sanitize.Hwaddress) &&
diff --git a/cmd/dep_fixer/Android.bp b/cmd/dep_fixer/Android.bp
index d2d1113..97364d5 100644
--- a/cmd/dep_fixer/Android.bp
+++ b/cmd/dep_fixer/Android.bp
@@ -14,10 +14,6 @@
blueprint_go_binary {
name: "dep_fixer",
- deps: ["androidmk-parser"],
- srcs: [
- "main.go",
- "deps.go",
- ],
- testSrcs: ["deps_test.go"],
+ deps: ["soong-makedeps"],
+ srcs: ["main.go"],
}
diff --git a/cmd/dep_fixer/main.go b/cmd/dep_fixer/main.go
index f94cf2f..d1bd139 100644
--- a/cmd/dep_fixer/main.go
+++ b/cmd/dep_fixer/main.go
@@ -25,6 +25,8 @@
"io/ioutil"
"log"
"os"
+
+ "android/soong/makedeps"
)
func main() {
@@ -39,7 +41,7 @@
log.Fatal("Expected at least one input file as an argument")
}
- var mergedDeps *Deps
+ var mergedDeps *makedeps.Deps
var firstInput []byte
for i, arg := range flag.Args() {
@@ -48,7 +50,7 @@
log.Fatalf("Error opening %q: %v", arg, err)
}
- deps, err := Parse(arg, bytes.NewBuffer(append([]byte(nil), input...)))
+ deps, err := makedeps.Parse(arg, bytes.NewBuffer(append([]byte(nil), input...)))
if err != nil {
log.Fatalf("Failed to parse: %v", err)
}
diff --git a/cmd/sbox/Android.bp b/cmd/sbox/Android.bp
index fe4c7bb..a706810d 100644
--- a/cmd/sbox/Android.bp
+++ b/cmd/sbox/Android.bp
@@ -14,6 +14,7 @@
blueprint_go_binary {
name: "sbox",
+ deps: ["soong-makedeps"],
srcs: [
"sbox.go",
],
diff --git a/cmd/sbox/sbox.go b/cmd/sbox/sbox.go
index 4167edb..7057b33 100644
--- a/cmd/sbox/sbox.go
+++ b/cmd/sbox/sbox.go
@@ -15,6 +15,7 @@
package main
import (
+ "bytes"
"errors"
"flag"
"fmt"
@@ -25,6 +26,8 @@
"path/filepath"
"strings"
"time"
+
+ "android/soong/makedeps"
)
var (
@@ -56,7 +59,7 @@
}
fmt.Fprintf(os.Stderr,
- "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> --overwrite [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+
+ "Usage: sbox -c <commandToRun> --sandbox-path <sandboxPath> --output-root <outputRoot> [--depfile-out depFile] <outputFile> [<outputFile>...]\n"+
"\n"+
"Deletes <outputRoot>,"+
"runs <commandToRun>,"+
@@ -152,9 +155,6 @@
return err
}
allOutputs = append(allOutputs, sandboxedDepfile)
- if !strings.Contains(rawCommand, "__SBOX_DEPFILE__") {
- return fmt.Errorf("the --depfile-out argument only makes sense if the command contains the text __SBOX_DEPFILE__")
- }
rawCommand = strings.Replace(rawCommand, "__SBOX_DEPFILE__", filepath.Join(tempDir, sandboxedDepfile), -1)
}
@@ -281,6 +281,26 @@
}
}
+ // Rewrite the depfile so that it doesn't include the (randomized) sandbox directory
+ if depfileOut != "" {
+ in, err := ioutil.ReadFile(depfileOut)
+ if err != nil {
+ return err
+ }
+
+ deps, err := makedeps.Parse(depfileOut, bytes.NewBuffer(in))
+ if err != nil {
+ return err
+ }
+
+ deps.Output = "outputfile"
+
+ err = ioutil.WriteFile(depfileOut, deps.Print(), 0666)
+ if err != nil {
+ return err
+ }
+ }
+
// TODO(jeffrygaston) if a process creates more output files than it declares, should there be a warning?
return nil
}
diff --git a/genrule/genrule.go b/genrule/genrule.go
index 87e6747..55c7e62 100644
--- a/genrule/genrule.go
+++ b/genrule/genrule.go
@@ -17,6 +17,7 @@
import (
"fmt"
"io"
+ "strconv"
"strings"
"github.com/google/blueprint"
@@ -37,10 +38,20 @@
var (
pctx = android.NewPackageContext("android/soong/genrule")
+
+ gensrcsMerge = pctx.AndroidStaticRule("gensrcsMerge", blueprint.RuleParams{
+ Command: "${soongZip} -o ${tmpZip} @${tmpZip}.rsp && ${zipSync} -d ${genDir} ${tmpZip}",
+ CommandDeps: []string{"${soongZip}", "${zipSync}"},
+ Rspfile: "${tmpZip}.rsp",
+ RspfileContent: "${zipArgs}",
+ }, "tmpZip", "genDir", "zipArgs")
)
func init() {
pctx.HostBinToolVariable("sboxCmd", "sbox")
+
+ pctx.HostBinToolVariable("soongZip", "soong_zip")
+ pctx.HostBinToolVariable("zipSync", "zipsync")
}
type SourceFileGenerator interface {
@@ -110,9 +121,9 @@
taskGenerator taskFunc
- deps android.Paths
- rule blueprint.Rule
- rawCommand string
+ deps android.Paths
+ rule blueprint.Rule
+ rawCommands []string
exportedIncludeDirs android.Paths
@@ -120,15 +131,20 @@
outputDeps android.Paths
subName string
+ subDir string
}
-type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask
+type taskFunc func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask
type generateTask struct {
in android.Paths
out android.WritablePaths
+ copyTo android.WritablePaths
+ genDir android.WritablePath
sandboxOuts []string
cmd string
+ shard int
+ shards int
}
func (g *Module) GeneratedSourceFiles() android.Paths {
@@ -167,10 +183,10 @@
if len(g.properties.Export_include_dirs) > 0 {
for _, dir := range g.properties.Export_include_dirs {
g.exportedIncludeDirs = append(g.exportedIncludeDirs,
- android.PathForModuleGen(ctx, ctx.ModuleDir(), dir))
+ android.PathForModuleGen(ctx, g.subDir, ctx.ModuleDir(), dir))
}
} else {
- g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, ""))
+ g.exportedIncludeDirs = append(g.exportedIncludeDirs, android.PathForModuleGen(ctx, g.subDir))
}
locationLabels := map[string][]string{}
@@ -275,120 +291,154 @@
}
}
- task := g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles)
+ var copyFrom android.Paths
+ var outputFiles android.WritablePaths
+ var zipArgs strings.Builder
- for _, out := range task.out {
- addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())})
- }
-
- referencedDepfile := false
-
- rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
- // report the error directly without returning an error to android.Expand to catch multiple errors in a
- // single run
- reportError := func(fmt string, args ...interface{}) (string, error) {
- ctx.PropertyErrorf("cmd", fmt, args...)
- return "SOONG_ERROR", nil
+ for _, task := range g.taskGenerator(ctx, String(g.properties.Cmd), srcFiles) {
+ for _, out := range task.out {
+ addLocationLabel(out.Rel(), []string{filepath.Join("__SBOX_OUT_DIR__", out.Rel())})
}
- switch name {
- case "location":
- if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
- return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
+ referencedDepfile := false
+
+ rawCommand, err := android.Expand(task.cmd, func(name string) (string, error) {
+ // report the error directly without returning an error to android.Expand to catch multiple errors in a
+ // single run
+ reportError := func(fmt string, args ...interface{}) (string, error) {
+ ctx.PropertyErrorf("cmd", fmt, args...)
+ return "SOONG_ERROR", nil
}
- paths := locationLabels[firstLabel]
- if len(paths) == 0 {
- return reportError("default label %q has no files", firstLabel)
- } else if len(paths) > 1 {
- return reportError("default label %q has multiple files, use $(locations %s) to reference it",
- firstLabel, firstLabel)
- }
- return locationLabels[firstLabel][0], nil
- case "in":
- return "${in}", nil
- case "out":
- return "__SBOX_OUT_FILES__", nil
- case "depfile":
- referencedDepfile = true
- if !Bool(g.properties.Depfile) {
- return reportError("$(depfile) used without depfile property")
- }
- return "__SBOX_DEPFILE__", nil
- case "genDir":
- return "__SBOX_OUT_DIR__", nil
- default:
- if strings.HasPrefix(name, "location ") {
- label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
- if paths, ok := locationLabels[label]; ok {
- if len(paths) == 0 {
- return reportError("label %q has no files", label)
- } else if len(paths) > 1 {
- return reportError("label %q has multiple files, use $(locations %s) to reference it",
- label, label)
- }
- return paths[0], nil
- } else {
- return reportError("unknown location label %q", label)
+
+ switch name {
+ case "location":
+ if len(g.properties.Tools) == 0 && len(g.properties.Tool_files) == 0 {
+ return reportError("at least one `tools` or `tool_files` is required if $(location) is used")
}
- } else if strings.HasPrefix(name, "locations ") {
- label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
- if paths, ok := locationLabels[label]; ok {
- if len(paths) == 0 {
- return reportError("label %q has no files", label)
- }
- return strings.Join(paths, " "), nil
- } else {
- return reportError("unknown locations label %q", label)
+ paths := locationLabels[firstLabel]
+ if len(paths) == 0 {
+ return reportError("default label %q has no files", firstLabel)
+ } else if len(paths) > 1 {
+ return reportError("default label %q has multiple files, use $(locations %s) to reference it",
+ firstLabel, firstLabel)
}
- } else {
- return reportError("unknown variable '$(%s)'", name)
+ return locationLabels[firstLabel][0], nil
+ case "in":
+ return "${in}", nil
+ case "out":
+ return "__SBOX_OUT_FILES__", nil
+ case "depfile":
+ referencedDepfile = true
+ if !Bool(g.properties.Depfile) {
+ return reportError("$(depfile) used without depfile property")
+ }
+ return "__SBOX_DEPFILE__", nil
+ case "genDir":
+ return "__SBOX_OUT_DIR__", nil
+ default:
+ if strings.HasPrefix(name, "location ") {
+ label := strings.TrimSpace(strings.TrimPrefix(name, "location "))
+ if paths, ok := locationLabels[label]; ok {
+ if len(paths) == 0 {
+ return reportError("label %q has no files", label)
+ } else if len(paths) > 1 {
+ return reportError("label %q has multiple files, use $(locations %s) to reference it",
+ label, label)
+ }
+ return paths[0], nil
+ } else {
+ return reportError("unknown location label %q", label)
+ }
+ } else if strings.HasPrefix(name, "locations ") {
+ label := strings.TrimSpace(strings.TrimPrefix(name, "locations "))
+ if paths, ok := locationLabels[label]; ok {
+ if len(paths) == 0 {
+ return reportError("label %q has no files", label)
+ }
+ return strings.Join(paths, " "), nil
+ } else {
+ return reportError("unknown locations label %q", label)
+ }
+ } else {
+ return reportError("unknown variable '$(%s)'", name)
+ }
}
+ })
+
+ if err != nil {
+ ctx.PropertyErrorf("cmd", "%s", err.Error())
+ return
}
- })
- if err != nil {
- ctx.PropertyErrorf("cmd", "%s", err.Error())
- return
+ if Bool(g.properties.Depfile) && !referencedDepfile {
+ ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
+ return
+ }
+
+ // tell the sbox command which directory to use as its sandbox root
+ buildDir := android.PathForOutput(ctx).String()
+ sandboxPath := shared.TempDirForOutDir(buildDir)
+
+ // recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written,
+ // to be replaced later by ninja_strings.go
+ depfilePlaceholder := ""
+ if Bool(g.properties.Depfile) {
+ depfilePlaceholder = "$depfileArgs"
+ }
+
+ // Escape the command for the shell
+ rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'"
+ g.rawCommands = append(g.rawCommands, rawCommand)
+ sandboxCommand := fmt.Sprintf("rm -rf %s && $sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts",
+ task.genDir, sandboxPath, task.genDir, rawCommand, depfilePlaceholder)
+
+ ruleParams := blueprint.RuleParams{
+ Command: sandboxCommand,
+ CommandDeps: []string{"$sboxCmd"},
+ }
+ args := []string{"allouts"}
+ if Bool(g.properties.Depfile) {
+ ruleParams.Deps = blueprint.DepsGCC
+ args = append(args, "depfileArgs")
+ }
+ name := "generator"
+ if task.shards > 1 {
+ name += strconv.Itoa(task.shard)
+ }
+ rule := ctx.Rule(pctx, name, ruleParams, args...)
+
+ g.generateSourceFile(ctx, task, rule)
+
+ if len(task.copyTo) > 0 {
+ outputFiles = append(outputFiles, task.copyTo...)
+ copyFrom = append(copyFrom, task.out.Paths()...)
+ zipArgs.WriteString(" -C " + task.genDir.String())
+ zipArgs.WriteString(android.JoinWithPrefix(task.out.Strings(), " -f "))
+ } else {
+ outputFiles = append(outputFiles, task.out...)
+ }
}
- if Bool(g.properties.Depfile) && !referencedDepfile {
- ctx.PropertyErrorf("cmd", "specified depfile=true but did not include a reference to '${depfile}' in cmd")
+ if len(copyFrom) > 0 {
+ ctx.Build(pctx, android.BuildParams{
+ Rule: gensrcsMerge,
+ Implicits: copyFrom,
+ Outputs: outputFiles,
+ Args: map[string]string{
+ "zipArgs": zipArgs.String(),
+ "tmpZip": android.PathForModuleGen(ctx, g.subDir+".zip").String(),
+ "genDir": android.PathForModuleGen(ctx, g.subDir).String(),
+ },
+ })
}
- // tell the sbox command which directory to use as its sandbox root
- buildDir := android.PathForOutput(ctx).String()
- sandboxPath := shared.TempDirForOutDir(buildDir)
-
- // recall that Sprintf replaces percent sign expressions, whereas dollar signs expressions remain as written,
- // to be replaced later by ninja_strings.go
- depfilePlaceholder := ""
- if Bool(g.properties.Depfile) {
- depfilePlaceholder = "$depfileArgs"
+ g.outputFiles = outputFiles.Paths()
+ if len(g.outputFiles) > 0 {
+ g.outputDeps = append(g.outputDeps, g.outputFiles[0])
}
-
- genDir := android.PathForModuleGen(ctx)
- // Escape the command for the shell
- rawCommand = "'" + strings.Replace(rawCommand, "'", `'\''`, -1) + "'"
- g.rawCommand = rawCommand
- sandboxCommand := fmt.Sprintf("$sboxCmd --sandbox-path %s --output-root %s -c %s %s $allouts",
- sandboxPath, genDir, rawCommand, depfilePlaceholder)
-
- ruleParams := blueprint.RuleParams{
- Command: sandboxCommand,
- CommandDeps: []string{"$sboxCmd"},
- }
- args := []string{"allouts"}
- if Bool(g.properties.Depfile) {
- ruleParams.Deps = blueprint.DepsGCC
- args = append(args, "depfileArgs")
- }
- g.rule = ctx.Rule(pctx, "generator", ruleParams, args...)
-
- g.generateSourceFile(ctx, task)
-
}
-func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask) {
+func (g *Module) generateSourceFile(ctx android.ModuleContext, task generateTask, rule blueprint.Rule) {
desc := "generate"
if len(task.out) == 0 {
ctx.ModuleErrorf("must have at least one output file")
@@ -403,9 +453,13 @@
depFile = android.PathForModuleGen(ctx, task.out[0].Rel()+".d")
}
+ if task.shards > 1 {
+ desc += " " + strconv.Itoa(task.shard)
+ }
+
params := android.BuildParams{
- Rule: g.rule,
- Description: "generate",
+ Rule: rule,
+ Description: desc,
Output: task.out[0],
ImplicitOutputs: task.out[1:],
Inputs: task.in,
@@ -420,11 +474,6 @@
}
ctx.Build(pctx, params)
-
- for _, outputFile := range task.out {
- g.outputFiles = append(g.outputFiles, outputFile)
- }
- g.outputDeps = append(g.outputDeps, task.out[0])
}
// Collect information for opening IDE project files in java/jdeps.go.
@@ -446,7 +495,7 @@
SubName: g.subName,
Extra: []android.AndroidMkExtraFunc{
func(w io.Writer, outputFile android.Path) {
- fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=", strings.Join(g.outputFiles.Strings(), " "))
+ fmt.Fprintln(w, "LOCAL_ADDITIONAL_DEPENDENCIES :=", strings.Join(g.outputDeps.Strings(), " "))
},
},
Custom: func(w io.Writer, name, prefix, moduleDir string, data android.AndroidMkData) {
@@ -483,47 +532,80 @@
func NewGenSrcs() *Module {
properties := &genSrcsProperties{}
- taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask {
- commands := []string{}
- outFiles := android.WritablePaths{}
- genDir := android.PathForModuleGen(ctx)
- sandboxOuts := []string{}
- for _, in := range srcFiles {
- outFile := android.GenPathWithExt(ctx, "", in, String(properties.Output_extension))
- outFiles = append(outFiles, outFile)
+ taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
+ genDir := android.PathForModuleGen(ctx, "gensrcs")
+ shardSize := defaultShardSize
+ if s := properties.Shard_size; s != nil {
+ shardSize = int(*s)
+ }
- sandboxOutfile := pathToSandboxOut(outFile, genDir)
- sandboxOuts = append(sandboxOuts, sandboxOutfile)
+ shards := android.ShardPaths(srcFiles, shardSize)
+ var generateTasks []generateTask
- command, err := android.Expand(rawCommand, func(name string) (string, error) {
- switch name {
- case "in":
- return in.String(), nil
- case "out":
- return sandboxOutfile, nil
- default:
- return "$(" + name + ")", nil
- }
- })
- if err != nil {
- ctx.PropertyErrorf("cmd", err.Error())
+ for i, shard := range shards {
+ var commands []string
+ var outFiles android.WritablePaths
+ var copyTo android.WritablePaths
+ var shardDir android.WritablePath
+ var sandboxOuts []string
+
+ if len(shards) > 1 {
+ shardDir = android.PathForModuleGen(ctx, strconv.Itoa(i))
+ } else {
+ shardDir = genDir
}
- // escape the command in case for example it contains '#', an odd number of '"', etc
- command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
- commands = append(commands, command)
- }
- fullCommand := strings.Join(commands, " && ")
+ for _, in := range shard {
+ outFile := android.GenPathWithExt(ctx, "gensrcs", in, String(properties.Output_extension))
+ sandboxOutfile := pathToSandboxOut(outFile, genDir)
- return generateTask{
- in: srcFiles,
- out: outFiles,
- sandboxOuts: sandboxOuts,
- cmd: fullCommand,
+ if len(shards) > 1 {
+ shardFile := android.GenPathWithExt(ctx, strconv.Itoa(i), in, String(properties.Output_extension))
+ copyTo = append(copyTo, outFile)
+ outFile = shardFile
+ }
+
+ outFiles = append(outFiles, outFile)
+ sandboxOuts = append(sandboxOuts, sandboxOutfile)
+
+ command, err := android.Expand(rawCommand, func(name string) (string, error) {
+ switch name {
+ case "in":
+ return in.String(), nil
+ case "out":
+ return sandboxOutfile, nil
+ default:
+ return "$(" + name + ")", nil
+ }
+ })
+ if err != nil {
+ ctx.PropertyErrorf("cmd", err.Error())
+ }
+
+ // escape the command in case for example it contains '#', an odd number of '"', etc
+ command = fmt.Sprintf("bash -c %v", proptools.ShellEscape(command))
+ commands = append(commands, command)
+ }
+ fullCommand := strings.Join(commands, " && ")
+
+ generateTasks = append(generateTasks, generateTask{
+ in: shard,
+ out: outFiles,
+ copyTo: copyTo,
+ genDir: shardDir,
+ sandboxOuts: sandboxOuts,
+ cmd: fullCommand,
+ shard: i,
+ shards: len(shards),
+ })
}
+
+ return generateTasks
}
- return generatorFactory(taskGenerator, properties)
+ g := generatorFactory(taskGenerator, properties)
+ g.subDir = "gensrcs"
+ return g
}
func GenSrcsFactory() android.Module {
@@ -535,12 +617,17 @@
type genSrcsProperties struct {
// extension that will be substituted for each output file
Output_extension *string
+
+ // maximum number of files that will be passed on a single command line.
+ Shard_size *int64
}
+const defaultShardSize = 100
+
func NewGenRule() *Module {
properties := &genRuleProperties{}
- taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) generateTask {
+ taskGenerator := func(ctx android.ModuleContext, rawCommand string, srcFiles android.Paths) []generateTask {
outs := make(android.WritablePaths, len(properties.Out))
sandboxOuts := make([]string, len(properties.Out))
genDir := android.PathForModuleGen(ctx)
@@ -548,12 +635,13 @@
outs[i] = android.PathForModuleGen(ctx, out)
sandboxOuts[i] = pathToSandboxOut(outs[i], genDir)
}
- return generateTask{
+ return []generateTask{{
in: srcFiles,
out: outs,
+ genDir: android.PathForModuleGen(ctx),
sandboxOuts: sandboxOuts,
cmd: rawCommand,
- }
+ }}
}
return generatorFactory(taskGenerator, properties)
diff --git a/genrule/genrule_test.go b/genrule/genrule_test.go
index 0b6952f..e8dc3b5 100644
--- a/genrule/genrule_test.go
+++ b/genrule/genrule_test.go
@@ -57,6 +57,7 @@
ctx := android.NewTestArchContext()
ctx.RegisterModuleType("filegroup", android.ModuleFactoryAdaptor(android.FileGroupFactory))
ctx.RegisterModuleType("genrule", android.ModuleFactoryAdaptor(GenRuleFactory))
+ ctx.RegisterModuleType("gensrcs", android.ModuleFactoryAdaptor(GenSrcsFactory))
ctx.RegisterModuleType("genrule_defaults", android.ModuleFactoryAdaptor(defaultsFactory))
ctx.RegisterModuleType("tool", android.ModuleFactoryAdaptor(toolFactory))
ctx.PreArchMutators(android.RegisterDefaultsPreArchMutators)
@@ -109,6 +110,9 @@
"tool_file2": nil,
"in1": nil,
"in2": nil,
+ "in1.txt": nil,
+ "in2.txt": nil,
+ "in3.txt": nil,
}
for k, v := range fs {
@@ -491,11 +495,102 @@
}
gen := ctx.ModuleForTests("gen", "").Module().(*Module)
- if g, w := gen.rawCommand, "'"+test.expect+"'"; w != g {
+ if g, w := gen.rawCommands[0], "'"+test.expect+"'"; w != g {
t.Errorf("want %q, got %q", w, g)
}
})
}
+}
+
+func TestGenSrcs(t *testing.T) {
+ testcases := []struct {
+ name string
+ prop string
+
+ allowMissingDependencies bool
+
+ err string
+ cmds []string
+ deps []string
+ files []string
+ }{
+ {
+ name: "gensrcs",
+ prop: `
+ tools: ["tool"],
+ srcs: ["in1.txt", "in2.txt"],
+ cmd: "$(location) $(in) > $(out)",
+ `,
+ cmds: []string{
+ "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''",
+ },
+ deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"},
+ files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h"},
+ },
+ {
+ name: "shards",
+ prop: `
+ tools: ["tool"],
+ srcs: ["in1.txt", "in2.txt", "in3.txt"],
+ cmd: "$(location) $(in) > $(out)",
+ shard_size: 2,
+ `,
+ cmds: []string{
+ "'bash -c '\\''out/tool in1.txt > __SBOX_OUT_DIR__/in1.h'\\'' && bash -c '\\''out/tool in2.txt > __SBOX_OUT_DIR__/in2.h'\\'''",
+ "'bash -c '\\''out/tool in3.txt > __SBOX_OUT_DIR__/in3.h'\\'''",
+ },
+ deps: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h"},
+ files: []string{buildDir + "/.intermediates/gen/gen/gensrcs/in1.h", buildDir + "/.intermediates/gen/gen/gensrcs/in2.h", buildDir + "/.intermediates/gen/gen/gensrcs/in3.h"},
+ },
+ }
+
+ for _, test := range testcases {
+ t.Run(test.name, func(t *testing.T) {
+ config := android.TestArchConfig(buildDir, nil)
+ bp := "gensrcs {\n"
+ bp += `name: "gen",` + "\n"
+ bp += `output_extension: "h",` + "\n"
+ bp += test.prop
+ bp += "}\n"
+
+ ctx := testContext(config, bp, nil)
+
+ _, errs := ctx.ParseFileList(".", []string{"Android.bp"})
+ if errs == nil {
+ _, errs = ctx.PrepareBuildActions(config)
+ }
+ if errs == nil && test.err != "" {
+ t.Fatalf("want error %q, got no error", test.err)
+ } else if errs != nil && test.err == "" {
+ android.FailIfErrored(t, errs)
+ } else if test.err != "" {
+ if len(errs) != 1 {
+ t.Errorf("want 1 error, got %d errors:", len(errs))
+ for _, err := range errs {
+ t.Errorf(" %s", err.Error())
+ }
+ t.FailNow()
+ }
+ if !strings.Contains(errs[0].Error(), test.err) {
+ t.Fatalf("want %q, got %q", test.err, errs[0].Error())
+ }
+ return
+ }
+
+ gen := ctx.ModuleForTests("gen", "").Module().(*Module)
+ if g, w := gen.rawCommands, test.cmds; !reflect.DeepEqual(w, g) {
+ t.Errorf("want %q, got %q", w, g)
+ }
+
+ if g, w := gen.outputDeps.Strings(), test.deps; !reflect.DeepEqual(w, g) {
+ t.Errorf("want deps %q, got %q", w, g)
+ }
+
+ if g, w := gen.outputFiles.Strings(), test.files; !reflect.DeepEqual(w, g) {
+ t.Errorf("want files %q, got %q", w, g)
+ }
+ })
+ }
}
@@ -529,8 +624,8 @@
gen := ctx.ModuleForTests("gen", "").Module().(*Module)
expectedCmd := "'cp ${in} __SBOX_OUT_FILES__'"
- if gen.rawCommand != expectedCmd {
- t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommand)
+ if gen.rawCommands[0] != expectedCmd {
+ t.Errorf("Expected cmd: %q, actual: %q", expectedCmd, gen.rawCommands[0])
}
expectedSrcs := []string{"in1"}
diff --git a/java/aapt2.go b/java/aapt2.go
index bcc8e97..f21408f 100644
--- a/java/aapt2.go
+++ b/java/aapt2.go
@@ -61,7 +61,7 @@
"outDir", "cFlags")
func aapt2Compile(ctx android.ModuleContext, dir android.Path, paths android.Paths) android.WritablePaths {
- shards := shardPaths(paths, AAPT2_SHARD_SIZE)
+ shards := android.ShardPaths(paths, AAPT2_SHARD_SIZE)
ret := make(android.WritablePaths, 0, len(paths))
diff --git a/java/java.go b/java/java.go
index 480518e..9ac38c9 100644
--- a/java/java.go
+++ b/java/java.go
@@ -530,18 +530,6 @@
return false
}
-func shardPaths(paths android.Paths, shardSize int) []android.Paths {
- ret := make([]android.Paths, 0, (len(paths)+shardSize-1)/shardSize)
- for len(paths) > shardSize {
- ret = append(ret, paths[0:shardSize])
- paths = paths[shardSize:]
- }
- if len(paths) > 0 {
- ret = append(ret, paths)
- }
- return ret
-}
-
func (j *Module) hasSrcExt(ext string) bool {
return hasSrcExt(j.properties.Srcs, ext)
}
@@ -1088,7 +1076,7 @@
shardSize := int(*(j.properties.Javac_shard_size))
var shardSrcs []android.Paths
if len(uniqueSrcFiles) > 0 {
- shardSrcs = shardPaths(uniqueSrcFiles, shardSize)
+ shardSrcs = android.ShardPaths(uniqueSrcFiles, shardSize)
for idx, shardSrc := range shardSrcs {
classes := android.PathForModuleOut(ctx, "javac", jarName+strconv.Itoa(idx))
TransformJavaToClasses(ctx, classes, idx, shardSrc, nil, flags, extraJarDeps)
@@ -1465,8 +1453,6 @@
type Library struct {
Module
-
- InstallMixin func(ctx android.ModuleContext, installPath android.Path) (extraInstallDeps android.Paths)
}
func shouldUncompressDex(ctx android.ModuleContext, dexpreopter *dexpreopter) bool {
@@ -1496,12 +1482,8 @@
j.compile(ctx)
if (Bool(j.properties.Installable) || ctx.Host()) && !android.DirectlyInAnyApex(ctx, ctx.ModuleName()) {
- var extraInstallDeps android.Paths
- if j.InstallMixin != nil {
- extraInstallDeps = j.InstallMixin(ctx, j.outputFile)
- }
j.installFile = ctx.InstallFile(android.PathForModuleInstall(ctx, "framework"),
- ctx.ModuleName()+".jar", j.outputFile, extraInstallDeps...)
+ ctx.ModuleName()+".jar", j.outputFile)
}
}
diff --git a/java/tradefed.go b/java/tradefed.go
deleted file mode 100644
index ebbdec1..0000000
--- a/java/tradefed.go
+++ /dev/null
@@ -1,37 +0,0 @@
-// Copyright 2019 Google Inc. All rights reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package java
-
-import (
- "android/soong/android"
-)
-
-func init() {
- android.RegisterModuleType("tradefed_java_library_host", tradefedJavaLibraryFactory)
-}
-
-// tradefed_java_library_factory wraps java_library and installs an additional
-// copy of the output jar to $HOST_OUT/tradefed.
-func tradefedJavaLibraryFactory() android.Module {
- module := LibraryHostFactory().(*Library)
- module.InstallMixin = tradefedJavaLibraryInstall
- return module
-}
-
-func tradefedJavaLibraryInstall(ctx android.ModuleContext, path android.Path) android.Paths {
- installedPath := ctx.InstallFile(android.PathForModuleInstall(ctx, "tradefed"),
- ctx.ModuleName()+".jar", path)
- return android.Paths{installedPath}
-}
diff --git a/makedeps/Android.bp b/makedeps/Android.bp
new file mode 100644
index 0000000..b77b08f
--- /dev/null
+++ b/makedeps/Android.bp
@@ -0,0 +1,21 @@
+// Copyright 2019 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+bootstrap_go_package {
+ name: "soong-makedeps",
+ pkgPath: "android/soong/makedeps",
+ deps: ["androidmk-parser"],
+ srcs: ["deps.go"],
+ testSrcs: ["deps_test.go"],
+}
diff --git a/cmd/dep_fixer/deps.go b/makedeps/deps.go
similarity index 98%
rename from cmd/dep_fixer/deps.go
rename to makedeps/deps.go
index 64c97f5..e64e6f7 100644
--- a/cmd/dep_fixer/deps.go
+++ b/makedeps/deps.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package main
+package makedeps
import (
"bytes"
diff --git a/cmd/dep_fixer/deps_test.go b/makedeps/deps_test.go
similarity index 99%
rename from cmd/dep_fixer/deps_test.go
rename to makedeps/deps_test.go
index 0a779b7..a32df65 100644
--- a/cmd/dep_fixer/deps_test.go
+++ b/makedeps/deps_test.go
@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
-package main
+package makedeps
import (
"bytes"
