Support sandboxing inputs in RuleBuilder
When RuleBuilder.SandboxInputs() is called configure sbox to copy
all the input files into the sandbox directory and then change the
working directory there when running the command.
Copying input files into the sandbox directory gets tricky when
the input file is the output file from another rule, and could
be at an arbitrary, possibly absolute path based on the value
of OUT_DIR. They will need to be copied to a directory in the
sandbox using the path relative to OUT_DIR.
RSP files need special handling, they need to both be copied into
the sandbox as an input, rewritten to contain paths as seen in the
sandbox, and references to them on the command line need to use
sandbox paths.
Bug: 182612695
Test: rule_builder_test.go
Change-Id: Ic0db961961b186e4ed9b76246881e3f04971825c
diff --git a/android/rule_builder.go b/android/rule_builder.go
index 75f1b5d..cc6c9b4 100644
--- a/android/rule_builder.go
+++ b/android/rule_builder.go
@@ -50,6 +50,7 @@
remoteable RemoteRuleSupports
outDir WritablePath
sboxTools bool
+ sboxInputs bool
sboxManifestPath WritablePath
missingDeps []string
}
@@ -155,6 +156,25 @@
return r
}
+// SandboxInputs enables input sandboxing for the rule by copying any referenced inputs into the
+// sandbox. It also implies SandboxTools().
+//
+// Sandboxing inputs requires RuleBuilder to be aware of all references to input paths. Paths
+// that are passed to RuleBuilder outside of the methods that expect inputs, for example
+// FlagWithArg, must use RuleBuilderCommand.PathForInput to translate the path to one that matches
+// the sandbox layout.
+func (r *RuleBuilder) SandboxInputs() *RuleBuilder {
+ if !r.sbox {
+ panic("SandboxInputs() must be called after Sbox()")
+ }
+ if len(r.commands) > 0 {
+ panic("SandboxInputs() may not be called after Command()")
+ }
+ r.sboxTools = true
+ r.sboxInputs = true
+ return r
+}
+
// Install associates an output of the rule with an install location, which can be retrieved later using
// RuleBuilder.Installs.
func (r *RuleBuilder) Install(from Path, to string) {
@@ -425,6 +445,26 @@
Inputs(depFiles.Paths())
}
+// composeRspFileContent returns a string that will serve as the contents of the rsp file to pass
+// the listed input files to the command running in the sandbox.
+func (r *RuleBuilder) composeRspFileContent(rspFileInputs Paths) string {
+ if r.sboxInputs {
+ if len(rspFileInputs) > 0 {
+ // When SandboxInputs is used the paths need to be rewritten to be relative to the sandbox
+ // directory so that they are valid after sbox chdirs into the sandbox directory.
+ return proptools.NinjaEscape(strings.Join(r.sboxPathsForInputsRel(rspFileInputs), " "))
+ } else {
+ // If the list of inputs is empty fall back to "$in" so that the rspfilecontent Ninja
+ // variable is set to something non-empty, otherwise ninja will complain. The inputs
+ // will be empty (all the non-rspfile inputs are implicits), so $in will evaluate to
+ // an empty string.
+ return "$in"
+ }
+ } else {
+ return "$in"
+ }
+}
+
// Build adds the built command line to the build graph, with dependencies on Inputs and Tools, and output files for
// Outputs.
func (r *RuleBuilder) Build(name string, desc string) {
@@ -511,6 +551,27 @@
}
}
+ // If sandboxing inputs is enabled, add copy rules to the manifest to copy each input
+ // into the sbox directory.
+ if r.sboxInputs {
+ for _, input := range inputs {
+ command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
+ From: proto.String(input.String()),
+ To: proto.String(r.sboxPathForInputRel(input)),
+ })
+ }
+
+ // If using an rsp file copy it into the sbox directory.
+ if rspFilePath != nil {
+ command.CopyBefore = append(command.CopyBefore, &sbox_proto.Copy{
+ From: proto.String(rspFilePath.String()),
+ To: proto.String(r.sboxPathForInputRel(rspFilePath)),
+ })
+ }
+
+ command.Chdir = proto.Bool(true)
+ }
+
// Add copy rules to the manifest to copy each output file from the sbox directory.
// to the output directory after running the commands.
sboxOutputs := make([]string, len(outputs))
@@ -580,7 +641,7 @@
var rspFile, rspFileContent string
if rspFilePath != nil {
rspFile = rspFilePath.String()
- rspFileContent = "$in"
+ rspFileContent = r.composeRspFileContent(rspFileInputs)
}
var pool blueprint.Pool
@@ -636,29 +697,45 @@
}
func (c *RuleBuilderCommand) addInput(path Path) string {
- if c.rule.sbox {
- if rel, isRel, _ := maybeRelErr(c.rule.outDir.String(), path.String()); isRel {
- return filepath.Join(sboxOutDir, rel)
- }
- }
c.inputs = append(c.inputs, path)
- return path.String()
+ return c.PathForInput(path)
}
-func (c *RuleBuilderCommand) addImplicit(path Path) string {
- if c.rule.sbox {
- if rel, isRel, _ := maybeRelErr(c.rule.outDir.String(), path.String()); isRel {
- return filepath.Join(sboxOutDir, rel)
- }
- }
+func (c *RuleBuilderCommand) addImplicit(path Path) {
c.implicits = append(c.implicits, path)
- return path.String()
}
func (c *RuleBuilderCommand) addOrderOnly(path Path) {
c.orderOnlys = append(c.orderOnlys, path)
}
+// PathForInput takes an input path and returns the appropriate path to use on the command line. If
+// sbox was enabled via a call to RuleBuilder.Sbox() and the path was an output path it returns a
+// path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
+// original path.
+func (c *RuleBuilderCommand) PathForInput(path Path) string {
+ if c.rule.sbox {
+ rel, inSandbox := c.rule._sboxPathForInputRel(path)
+ if inSandbox {
+ rel = filepath.Join(sboxSandboxBaseDir, rel)
+ }
+ return rel
+ }
+ return path.String()
+}
+
+// PathsForInputs takes a list of input paths and returns the appropriate paths to use on the
+// command line. If sbox was enabled via a call to RuleBuilder.Sbox() a path was an output path, it
+// returns the path with the placeholder prefix used for outputs in sbox. If sbox is not enabled it
+// returns the original paths.
+func (c *RuleBuilderCommand) PathsForInputs(paths Paths) []string {
+ ret := make([]string, len(paths))
+ for i, path := range paths {
+ ret[i] = c.PathForInput(path)
+ }
+ return ret
+}
+
// PathForOutput takes an output path and returns the appropriate path to use on the command
// line. If sbox was enabled via a call to RuleBuilder.Sbox(), it returns a path with the
// placeholder prefix used for outputs in sbox. If sbox is not enabled it returns the
@@ -690,6 +767,37 @@
return filepath.Join(sboxToolsSubDir, "src", path.String())
}
+func (r *RuleBuilder) _sboxPathForInputRel(path Path) (rel string, inSandbox bool) {
+ // Errors will be handled in RuleBuilder.Build where we have a context to report them
+ rel, isRelSboxOut, _ := maybeRelErr(r.outDir.String(), path.String())
+ if isRelSboxOut {
+ return filepath.Join(sboxOutSubDir, rel), true
+ }
+ if r.sboxInputs {
+ // When sandboxing inputs all inputs have to be copied into the sandbox. Input files that
+ // are outputs of other rules could be an arbitrary absolute path if OUT_DIR is set, so they
+ // will be copied to relative paths under __SBOX_OUT_DIR__/out.
+ rel, isRelOut, _ := maybeRelErr(PathForOutput(r.ctx).String(), path.String())
+ if isRelOut {
+ return filepath.Join(sboxOutSubDir, rel), true
+ }
+ }
+ return path.String(), false
+}
+
+func (r *RuleBuilder) sboxPathForInputRel(path Path) string {
+ rel, _ := r._sboxPathForInputRel(path)
+ return rel
+}
+
+func (r *RuleBuilder) sboxPathsForInputsRel(paths Paths) []string {
+ ret := make([]string, len(paths))
+ for i, path := range paths {
+ ret[i] = r.sboxPathForInputRel(path)
+ }
+ return ret
+}
+
// SboxPathForPackagedTool takes a PackageSpec for a tool and returns the corresponding path for the
// tool after copying it into the sandbox. This can be used on the RuleBuilder command line to
// reference the tool.
@@ -1053,7 +1161,7 @@
}
}
- c.FlagWithArg(flag, rspFile.String())
+ c.FlagWithArg(flag, c.PathForInput(rspFile))
return c
}
diff --git a/android/rule_builder_test.go b/android/rule_builder_test.go
index bd35820..3415aed 100644
--- a/android/rule_builder_test.go
+++ b/android/rule_builder_test.go
@@ -296,35 +296,40 @@
"input3": nil,
}
- pathCtx := PathContextForTesting(TestConfig("out", nil, "", fs))
+ pathCtx := PathContextForTesting(TestConfig("out_local", nil, "", fs))
ctx := builderContextForTests{
PathContext: pathCtx,
}
addCommands := func(rule *RuleBuilder) {
cmd := rule.Command().
- DepFile(PathForOutput(ctx, "DepFile")).
+ DepFile(PathForOutput(ctx, "module/DepFile")).
Flag("Flag").
FlagWithArg("FlagWithArg=", "arg").
- FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "depfile")).
+ FlagWithDepFile("FlagWithDepFile=", PathForOutput(ctx, "module/depfile")).
FlagWithInput("FlagWithInput=", PathForSource(ctx, "input")).
- FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "output")).
+ FlagWithOutput("FlagWithOutput=", PathForOutput(ctx, "module/output")).
+ FlagWithRspFileInputList("FlagWithRspFileInputList=", PathForOutput(ctx, "rsp"),
+ Paths{
+ PathForSource(ctx, "RspInput"),
+ PathForOutput(ctx, "other/RspOutput2"),
+ }).
Implicit(PathForSource(ctx, "Implicit")).
- ImplicitDepFile(PathForOutput(ctx, "ImplicitDepFile")).
- ImplicitOutput(PathForOutput(ctx, "ImplicitOutput")).
+ ImplicitDepFile(PathForOutput(ctx, "module/ImplicitDepFile")).
+ ImplicitOutput(PathForOutput(ctx, "module/ImplicitOutput")).
Input(PathForSource(ctx, "Input")).
- Output(PathForOutput(ctx, "Output")).
+ Output(PathForOutput(ctx, "module/Output")).
OrderOnly(PathForSource(ctx, "OrderOnly")).
- SymlinkOutput(PathForOutput(ctx, "SymlinkOutput")).
- ImplicitSymlinkOutput(PathForOutput(ctx, "ImplicitSymlinkOutput")).
+ SymlinkOutput(PathForOutput(ctx, "module/SymlinkOutput")).
+ ImplicitSymlinkOutput(PathForOutput(ctx, "module/ImplicitSymlinkOutput")).
Text("Text").
Tool(PathForSource(ctx, "Tool"))
rule.Command().
Text("command2").
- DepFile(PathForOutput(ctx, "depfile2")).
+ DepFile(PathForOutput(ctx, "module/depfile2")).
Input(PathForSource(ctx, "input2")).
- Output(PathForOutput(ctx, "output2")).
+ Output(PathForOutput(ctx, "module/output2")).
OrderOnlys(PathsForSource(ctx, []string{"OrderOnlys"})).
Tool(PathForSource(ctx, "tool2"))
@@ -337,32 +342,46 @@
rule.Command().
Text("command3").
Input(PathForSource(ctx, "input3")).
- Input(PathForOutput(ctx, "output2")).
- Output(PathForOutput(ctx, "output3"))
+ Input(PathForOutput(ctx, "module/output2")).
+ Output(PathForOutput(ctx, "module/output3")).
+ Text(cmd.PathForInput(PathForSource(ctx, "input3"))).
+ Text(cmd.PathForOutput(PathForOutput(ctx, "module/output2")))
}
wantInputs := PathsForSource(ctx, []string{"Implicit", "Input", "input", "input2", "input3"})
- wantOutputs := PathsForOutput(ctx, []string{"ImplicitOutput", "ImplicitSymlinkOutput", "Output", "SymlinkOutput", "output", "output2", "output3"})
- wantDepFiles := PathsForOutput(ctx, []string{"DepFile", "depfile", "ImplicitDepFile", "depfile2"})
+ wantRspFileInputs := Paths{PathForSource(ctx, "RspInput"),
+ PathForOutput(ctx, "other/RspOutput2")}
+ wantOutputs := PathsForOutput(ctx, []string{
+ "module/ImplicitOutput", "module/ImplicitSymlinkOutput", "module/Output", "module/SymlinkOutput",
+ "module/output", "module/output2", "module/output3"})
+ wantDepFiles := PathsForOutput(ctx, []string{
+ "module/DepFile", "module/depfile", "module/ImplicitDepFile", "module/depfile2"})
wantTools := PathsForSource(ctx, []string{"Tool", "tool2"})
wantOrderOnlys := PathsForSource(ctx, []string{"OrderOnly", "OrderOnlys"})
- wantSymlinkOutputs := PathsForOutput(ctx, []string{"ImplicitSymlinkOutput", "SymlinkOutput"})
+ wantSymlinkOutputs := PathsForOutput(ctx, []string{
+ "module/ImplicitSymlinkOutput", "module/SymlinkOutput"})
t.Run("normal", func(t *testing.T) {
rule := NewRuleBuilder(pctx, ctx)
addCommands(rule)
wantCommands := []string{
- "out/DepFile Flag FlagWithArg=arg FlagWithDepFile=out/depfile FlagWithInput=input FlagWithOutput=out/output Input out/Output out/SymlinkOutput Text Tool after command2 old cmd",
- "command2 out/depfile2 input2 out/output2 tool2",
- "command3 input3 out/output2 out/output3",
+ "out_local/module/DepFile Flag FlagWithArg=arg FlagWithDepFile=out_local/module/depfile " +
+ "FlagWithInput=input FlagWithOutput=out_local/module/output FlagWithRspFileInputList=out_local/rsp " +
+ "Input out_local/module/Output out_local/module/SymlinkOutput Text Tool after command2 old cmd",
+ "command2 out_local/module/depfile2 input2 out_local/module/output2 tool2",
+ "command3 input3 out_local/module/output2 out_local/module/output3 input3 out_local/module/output2",
}
- wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer out/DepFile out/depfile out/ImplicitDepFile out/depfile2"
+ wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer " +
+ "out_local/module/DepFile out_local/module/depfile out_local/module/ImplicitDepFile out_local/module/depfile2"
+
+ wantRspFileContent := "$in"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
+ AssertDeepEquals(t, "rule.RspfileInputs()", wantRspFileInputs, rule.RspFileInputs())
AssertDeepEquals(t, "rule.Outputs()", wantOutputs, rule.Outputs())
AssertDeepEquals(t, "rule.SymlinkOutputs()", wantSymlinkOutputs, rule.SymlinkOutputs())
AssertDeepEquals(t, "rule.DepFiles()", wantDepFiles, rule.DepFiles())
@@ -370,54 +389,74 @@
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
+
+ AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
t.Run("sbox", func(t *testing.T) {
- rule := NewRuleBuilder(pctx, ctx).Sbox(PathForOutput(ctx, ""),
+ rule := NewRuleBuilder(pctx, ctx).Sbox(PathForOutput(ctx, "module"),
PathForOutput(ctx, "sbox.textproto"))
addCommands(rule)
wantCommands := []string{
- "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output Input __SBOX_SANDBOX_DIR__/out/Output __SBOX_SANDBOX_DIR__/out/SymlinkOutput Text Tool after command2 old cmd",
+ "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " +
+ "FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " +
+ "FlagWithRspFileInputList=out_local/rsp Input __SBOX_SANDBOX_DIR__/out/Output " +
+ "__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text Tool after command2 old cmd",
"command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 tool2",
- "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3",
+ "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2",
}
- wantDepMergerCommand := "out/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
+ wantDepMergerCommand := "out_local/host/" + ctx.Config().PrebuiltOS() + "/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
+
+ wantRspFileContent := "$in"
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
+ AssertDeepEquals(t, "rule.RspfileInputs()", wantRspFileInputs, rule.RspFileInputs())
AssertDeepEquals(t, "rule.Outputs()", wantOutputs, rule.Outputs())
+ AssertDeepEquals(t, "rule.SymlinkOutputs()", wantSymlinkOutputs, rule.SymlinkOutputs())
AssertDeepEquals(t, "rule.DepFiles()", wantDepFiles, rule.DepFiles())
AssertDeepEquals(t, "rule.Tools()", wantTools, rule.Tools())
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
+
+ AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
t.Run("sbox tools", func(t *testing.T) {
- rule := NewRuleBuilder(pctx, ctx).Sbox(PathForOutput(ctx, ""),
+ rule := NewRuleBuilder(pctx, ctx).Sbox(PathForOutput(ctx, "module"),
PathForOutput(ctx, "sbox.textproto")).SandboxTools()
addCommands(rule)
wantCommands := []string{
- "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output Input __SBOX_SANDBOX_DIR__/out/Output __SBOX_SANDBOX_DIR__/out/SymlinkOutput Text __SBOX_SANDBOX_DIR__/tools/src/Tool after command2 old cmd",
+ "__SBOX_SANDBOX_DIR__/out/DepFile Flag FlagWithArg=arg FlagWithDepFile=__SBOX_SANDBOX_DIR__/out/depfile " +
+ "FlagWithInput=input FlagWithOutput=__SBOX_SANDBOX_DIR__/out/output " +
+ "FlagWithRspFileInputList=out_local/rsp Input __SBOX_SANDBOX_DIR__/out/Output " +
+ "__SBOX_SANDBOX_DIR__/out/SymlinkOutput Text __SBOX_SANDBOX_DIR__/tools/src/Tool after command2 old cmd",
"command2 __SBOX_SANDBOX_DIR__/out/depfile2 input2 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/tools/src/tool2",
- "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3",
+ "command3 input3 __SBOX_SANDBOX_DIR__/out/output2 __SBOX_SANDBOX_DIR__/out/output3 input3 __SBOX_SANDBOX_DIR__/out/output2",
}
wantDepMergerCommand := "__SBOX_SANDBOX_DIR__/tools/out/bin/dep_fixer __SBOX_SANDBOX_DIR__/out/DepFile __SBOX_SANDBOX_DIR__/out/depfile __SBOX_SANDBOX_DIR__/out/ImplicitDepFile __SBOX_SANDBOX_DIR__/out/depfile2"
+ wantRspFileContent := "$in"
+
AssertDeepEquals(t, "rule.Commands()", wantCommands, rule.Commands())
AssertDeepEquals(t, "rule.Inputs()", wantInputs, rule.Inputs())
+ AssertDeepEquals(t, "rule.RspfileInputs()", wantRspFileInputs, rule.RspFileInputs())
AssertDeepEquals(t, "rule.Outputs()", wantOutputs, rule.Outputs())
+ AssertDeepEquals(t, "rule.SymlinkOutputs()", wantSymlinkOutputs, rule.SymlinkOutputs())
AssertDeepEquals(t, "rule.DepFiles()", wantDepFiles, rule.DepFiles())
AssertDeepEquals(t, "rule.Tools()", wantTools, rule.Tools())
AssertDeepEquals(t, "rule.OrderOnlys()", wantOrderOnlys, rule.OrderOnlys())
AssertSame(t, "rule.depFileMergerCmd()", wantDepMergerCommand, rule.depFileMergerCmd(rule.DepFiles()).String())
+
+ AssertSame(t, "rule.composeRspFileContent()", wantRspFileContent, rule.composeRspFileContent(rule.RspFileInputs()))
})
}
