Merge changes Ic3a55909,I70462293
* changes:
Always run asan ckati on the build servers
Add exec.Cmd wrapper for logging / sandboxing
diff --git a/ui/build/Android.bp b/ui/build/Android.bp
index 51aed2c..d44c112 100644
--- a/ui/build/Android.bp
+++ b/ui/build/Android.bp
@@ -24,6 +24,7 @@
"config.go",
"context.go",
"environment.go",
+ "exec.go",
"kati.go",
"make.go",
"ninja.go",
@@ -36,9 +37,15 @@
"util_test.go",
],
darwin: {
- srcs: ["util_darwin.go"],
+ srcs: [
+ "sandbox_darwin.go",
+ "util_darwin.go"
+ ],
},
linux: {
- srcs: ["util_linux.go"],
+ srcs: [
+ "sandbox_linux.go",
+ "util_linux.go"
+ ],
},
}
diff --git a/ui/build/build.go b/ui/build/build.go
index 506ff51..d6059c0 100644
--- a/ui/build/build.go
+++ b/ui/build/build.go
@@ -16,7 +16,6 @@
import (
"os"
- "os/exec"
"path/filepath"
"text/template"
)
@@ -67,13 +66,12 @@
ctx.Verboseln("Environment:", config.Environment().Environ())
if inList("help", config.Arguments()) {
- cmd := exec.CommandContext(ctx.Context, "make", "-f", "build/core/help.mk")
- cmd.Env = config.Environment().Environ()
+ cmd := Command(ctx, config, "make",
+ "make", "-f", "build/core/help.mk")
+ cmd.Sandbox = makeSandbox
cmd.Stdout = ctx.Stdout()
cmd.Stderr = ctx.Stderr()
- if err := cmd.Run(); err != nil {
- ctx.Fatalln("Failed to run make:", err)
- }
+ cmd.RunOrFatal()
return
}
diff --git a/ui/build/config.go b/ui/build/config.go
index f5007b1..e21e1fa 100644
--- a/ui/build/config.go
+++ b/ui/build/config.go
@@ -319,13 +319,21 @@
}
}
-func (c *configImpl) PrebuiltBuildTool(name string) string {
+func (c *configImpl) HostAsan() bool {
if v, ok := c.environ.Get("SANITIZE_HOST"); ok {
if sanitize := strings.Fields(v); inList("address", sanitize) {
- asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
- if _, err := os.Stat(asan); err == nil {
- return asan
- }
+ return true
+ }
+ }
+ return false
+}
+
+func (c *configImpl) PrebuiltBuildTool(name string) string {
+ // (b/36182021) We're seeing rare ckati crashes, so always enable asan kati on the build servers.
+ if c.HostAsan() || (c.Dist() && name == "ckati") {
+ asan := filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "asan/bin", name)
+ if _, err := os.Stat(asan); err == nil {
+ return asan
}
}
return filepath.Join("prebuilts/build-tools", c.HostPrebuiltTag(), "bin", name)
diff --git a/ui/build/exec.go b/ui/build/exec.go
new file mode 100644
index 0000000..4c45c50
--- /dev/null
+++ b/ui/build/exec.go
@@ -0,0 +1,107 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os/exec"
+)
+
+// Cmd is a wrapper of os/exec.Cmd that integrates with the build context for
+// logging, the config's Environment for simpler environment modification, and
+// implements hooks for sandboxing
+type Cmd struct {
+ *exec.Cmd
+
+ Environment *Environment
+ Sandbox Sandbox
+
+ ctx Context
+ config Config
+ name string
+}
+
+func Command(ctx Context, config Config, name string, executable string, args ...string) *Cmd {
+ ret := &Cmd{
+ Cmd: exec.CommandContext(ctx.Context, executable, args...),
+ Environment: config.Environment().Copy(),
+ Sandbox: noSandbox,
+
+ ctx: ctx,
+ config: config,
+ name: name,
+ }
+
+ return ret
+}
+
+func (c *Cmd) prepare() {
+ if c.Env == nil {
+ c.Env = c.Environment.Environ()
+ }
+ if c.sandboxSupported() {
+ c.wrapSandbox()
+ }
+
+ c.ctx.Verboseln(c.Path, c.Args)
+}
+
+func (c *Cmd) Start() error {
+ c.prepare()
+ return c.Cmd.Start()
+}
+
+func (c *Cmd) Run() error {
+ c.prepare()
+ return c.Cmd.Run()
+}
+
+func (c *Cmd) Output() ([]byte, error) {
+ c.prepare()
+ return c.Cmd.Output()
+}
+
+func (c *Cmd) CombinedOutput() ([]byte, error) {
+ c.prepare()
+ return c.Cmd.CombinedOutput()
+}
+
+// StartOrFatal is equivalent to Start, but handles the error with a call to ctx.Fatal
+func (c *Cmd) StartOrFatal() {
+ if err := c.Start(); err != nil {
+ c.ctx.Fatalf("Failed to run %s: %v", c.name, err)
+ }
+}
+
+// RunOrFatal is equivalent to Run, but handles the error with a call to ctx.Fatal
+func (c *Cmd) RunOrFatal() {
+ if err := c.Run(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ c.ctx.Fatalf("%s failed with: %v", c.name, e.ProcessState.String())
+ } else {
+ c.ctx.Fatalf("Failed to run %s: %v", c.name, err)
+ }
+ }
+}
+
+// WaitOrFatal is equivalent to Wait, but handles the error with a call to ctx.Fatal
+func (c *Cmd) WaitOrFatal() {
+ if err := c.Wait(); err != nil {
+ if e, ok := err.(*exec.ExitError); ok {
+ c.ctx.Fatalf("%s failed with: %v", c.name, e.ProcessState.String())
+ } else {
+ c.ctx.Fatalf("Failed to run %s: %v", c.name, err)
+ }
+ }
+}
diff --git a/ui/build/kati.go b/ui/build/kati.go
index 218ca69..972b559 100644
--- a/ui/build/kati.go
+++ b/ui/build/kati.go
@@ -20,7 +20,6 @@
"fmt"
"io"
"io/ioutil"
- "os/exec"
"path/filepath"
"regexp"
"strconv"
@@ -93,28 +92,20 @@
args = append(args, "-j"+strconv.Itoa(config.Parallel()))
}
- cmd := exec.CommandContext(ctx.Context, executable, args...)
- cmd.Env = config.Environment().Environ()
+ cmd := Command(ctx, config, "ckati", executable, args...)
+ cmd.Sandbox = katiSandbox
pipe, err := cmd.StdoutPipe()
if err != nil {
ctx.Fatalln("Error getting output pipe for ckati:", err)
}
cmd.Stderr = cmd.Stdout
- ctx.Verboseln(cmd.Path, cmd.Args)
- if err := cmd.Start(); err != nil {
- ctx.Fatalln("Failed to run ckati:", err)
- }
+ // Kati leaks memory, so ensure leak detection is turned off
+ cmd.Environment.Set("ASAN_OPTIONS", "detect_leaks=0")
+ cmd.StartOrFatal()
katiRewriteOutput(ctx, pipe)
-
- if err := cmd.Wait(); err != nil {
- if e, ok := err.(*exec.ExitError); ok {
- ctx.Fatalln("ckati failed with:", e.ProcessState.String())
- } else {
- ctx.Fatalln("Failed to run ckati:", err)
- }
- }
+ cmd.WaitOrFatal()
}
var katiIncludeRe = regexp.MustCompile(`^(\[\d+/\d+] )?including [^ ]+ ...$`)
diff --git a/ui/build/make.go b/ui/build/make.go
index a8d4483..32dc17b 100644
--- a/ui/build/make.go
+++ b/ui/build/make.go
@@ -16,7 +16,6 @@
import (
"fmt"
- "os/exec"
"path/filepath"
"strings"
)
@@ -38,7 +37,7 @@
ctx.BeginTrace("dumpvars")
defer ctx.EndTrace()
- cmd := exec.CommandContext(ctx.Context,
+ cmd := Command(ctx, config, "make",
"make",
"--no-print-directory",
"-f", "build/core/config.mk",
@@ -48,11 +47,10 @@
"MAKECMDGOALS="+strings.Join(goals, " "),
"DUMP_MANY_VARS="+strings.Join(vars, " "),
"OUT_DIR="+config.OutDir())
- cmd.Env = config.Environment().Environ()
cmd.Args = append(cmd.Args, extra_targets...)
+ cmd.Sandbox = makeSandbox
// TODO: error out when Stderr contains any content
cmd.Stderr = ctx.Stderr()
- ctx.Verboseln(cmd.Path, cmd.Args)
output, err := cmd.Output()
if err != nil {
return nil, err
diff --git a/ui/build/ninja.go b/ui/build/ninja.go
index 054af24..5787a00 100644
--- a/ui/build/ninja.go
+++ b/ui/build/ninja.go
@@ -15,7 +15,6 @@
package build
import (
- "os/exec"
"path/filepath"
"strconv"
"strings"
@@ -51,35 +50,26 @@
}
args = append(args, "-w", "dupbuild=err")
- env := config.Environment().Copy()
- env.AppendFromKati(config.KatiEnvFile())
+ cmd := Command(ctx, config, "ninja", executable, args...)
+ cmd.Environment.AppendFromKati(config.KatiEnvFile())
// Allow both NINJA_ARGS and NINJA_EXTRA_ARGS, since both have been
// used in the past to specify extra ninja arguments.
- if extra, ok := env.Get("NINJA_ARGS"); ok {
- args = append(args, strings.Fields(extra)...)
+ if extra, ok := cmd.Environment.Get("NINJA_ARGS"); ok {
+ cmd.Args = append(cmd.Args, strings.Fields(extra)...)
}
- if extra, ok := env.Get("NINJA_EXTRA_ARGS"); ok {
- args = append(args, strings.Fields(extra)...)
+ if extra, ok := cmd.Environment.Get("NINJA_EXTRA_ARGS"); ok {
+ cmd.Args = append(cmd.Args, strings.Fields(extra)...)
}
- if _, ok := env.Get("NINJA_STATUS"); !ok {
- env.Set("NINJA_STATUS", "[%p %f/%t] ")
+ if _, ok := cmd.Environment.Get("NINJA_STATUS"); !ok {
+ cmd.Environment.Set("NINJA_STATUS", "[%p %f/%t] ")
}
- cmd := exec.CommandContext(ctx.Context, executable, args...)
- cmd.Env = env.Environ()
cmd.Stdin = ctx.Stdin()
cmd.Stdout = ctx.Stdout()
cmd.Stderr = ctx.Stderr()
- ctx.Verboseln(cmd.Path, cmd.Args)
startTime := time.Now()
defer ctx.ImportNinjaLog(filepath.Join(config.OutDir(), ".ninja_log"), startTime)
- if err := cmd.Run(); err != nil {
- if e, ok := err.(*exec.ExitError); ok {
- ctx.Fatalln("ninja failed with:", e.ProcessState.String())
- } else {
- ctx.Fatalln("Failed to run ninja:", err)
- }
- }
+ cmd.RunOrFatal()
}
diff --git a/ui/build/sandbox/darwin/global.sb b/ui/build/sandbox/darwin/global.sb
new file mode 100644
index 0000000..47d0c43
--- /dev/null
+++ b/ui/build/sandbox/darwin/global.sb
@@ -0,0 +1,40 @@
+(version 1)
+
+; TODO: (deny default)
+(allow default (with report))
+
+; Import apple-defined rules for bsd daemons
+(import "bsd.sb")
+
+; Allow reading of any file
+(allow file-read*)
+
+; Allow writing to $OUT_DIR and $DIST_DIR
+(allow file-write*
+ (subpath (param "OUT_DIR"))
+ (subpath (param "DIST_DIR")))
+
+; Java attempts to write usage data to ~/.oracle_jre_usage, just ignore
+(deny file-write* (with no-log)
+ (subpath (string-append (param "HOME") "/.oracle_jre_usage")))
+
+; Allow writes to user-specific temp folders (Java stores hsperfdata there)
+(allow file-write*
+ (subpath "/private/var/folders"))
+
+; Allow writing to the terminal
+(allow file-write-data
+ (subpath "/dev/tty"))
+
+; Java
+(allow mach-lookup
+ (global-name "com.apple.SystemConfiguration.configd") ; Java
+ (global-name "com.apple.CoreServices.coreservicesd") ; xcodebuild in Soong
+ (global-name "com.apple.FSEvents") ; xcodebuild in Soong
+ (global-name "com.apple.lsd.mapdb") ; xcodebuild in Soong
+ (global-name-regex #"^com\.apple\.distributed_notifications") ; xcodebuild in Soong
+)
+
+; Allow executing any file
+(allow process-exec*)
+(allow process-fork)
diff --git a/ui/build/sandbox_darwin.go b/ui/build/sandbox_darwin.go
new file mode 100644
index 0000000..54c145c
--- /dev/null
+++ b/ui/build/sandbox_darwin.go
@@ -0,0 +1,69 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+import (
+ "os/exec"
+ "path/filepath"
+)
+
+type Sandbox string
+
+const (
+ noSandbox = ""
+ globalSandbox = "build/soong/ui/build/sandbox/darwin/global.sb"
+ makeSandbox = globalSandbox
+ soongSandbox = globalSandbox
+ katiSandbox = globalSandbox
+)
+
+var sandboxExecPath string
+
+func init() {
+ if p, err := exec.LookPath("sandbox-exec"); err == nil {
+ sandboxExecPath = p
+ }
+}
+
+func (c *Cmd) sandboxSupported() bool {
+ if c.Sandbox == "" {
+ return false
+ } else if sandboxExecPath == "" {
+ c.ctx.Verboseln("sandbox-exec not found, disabling sandboxing")
+ return false
+ }
+ return true
+}
+
+func (c *Cmd) wrapSandbox() {
+ homeDir, _ := c.Environment.Get("HOME")
+ outDir, err := filepath.Abs(c.config.OutDir())
+ if err != nil {
+ c.ctx.Fatalln("Failed to get absolute path of OUT_DIR:", err)
+ }
+ distDir, err := filepath.Abs(c.config.DistDir())
+ if err != nil {
+ c.ctx.Fatalln("Failed to get absolute path of DIST_DIR:", err)
+ }
+
+ c.Args[0] = c.Path
+ c.Path = sandboxExecPath
+ c.Args = append([]string{
+ "sandbox-exec", "-f", string(c.Sandbox),
+ "-D", "HOME=" + homeDir,
+ "-D", "OUT_DIR=" + outDir,
+ "-D", "DIST_DIR=" + distDir,
+ }, c.Args...)
+}
diff --git a/ui/build/sandbox_linux.go b/ui/build/sandbox_linux.go
new file mode 100644
index 0000000..fb48b9c
--- /dev/null
+++ b/ui/build/sandbox_linux.go
@@ -0,0 +1,32 @@
+// Copyright 2017 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package build
+
+type Sandbox bool
+
+const (
+ noSandbox = false
+ globalSandbox = false
+ makeSandbox = false
+ soongSandbox = false
+ katiSandbox = false
+)
+
+func (c *Cmd) sandboxSupported() bool {
+ return false
+}
+
+func (c *Cmd) wrapSandbox() {
+}
diff --git a/ui/build/soong.go b/ui/build/soong.go
index 6554f1d..ddfe666 100644
--- a/ui/build/soong.go
+++ b/ui/build/soong.go
@@ -15,7 +15,6 @@
package build
import (
- "os/exec"
"path/filepath"
)
@@ -23,42 +22,27 @@
ctx.BeginTrace("bootstrap soong")
defer ctx.EndTrace()
- cmd := exec.CommandContext(ctx.Context, "./bootstrap.bash")
- env := config.Environment().Copy()
- env.Set("BUILDDIR", config.SoongOutDir())
- cmd.Env = env.Environ()
+ cmd := Command(ctx, config, "soong bootstrap", "./bootstrap.bash")
+ cmd.Environment.Set("BUILDDIR", config.SoongOutDir())
+ cmd.Sandbox = soongSandbox
cmd.Stdout = ctx.Stdout()
cmd.Stderr = ctx.Stderr()
- ctx.Verboseln(cmd.Path, cmd.Args)
- if err := cmd.Run(); err != nil {
- if e, ok := err.(*exec.ExitError); ok {
- ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
- } else {
- ctx.Fatalln("Failed to run soong bootstrap:", err)
- }
- }
+ cmd.RunOrFatal()
}
func runSoong(ctx Context, config Config) {
ctx.BeginTrace("soong")
defer ctx.EndTrace()
- cmd := exec.CommandContext(ctx.Context, filepath.Join(config.SoongOutDir(), "soong"), "-w", "dupbuild=err")
+ cmd := Command(ctx, config, "soong",
+ filepath.Join(config.SoongOutDir(), "soong"), "-w", "dupbuild=err")
if config.IsVerbose() {
cmd.Args = append(cmd.Args, "-v")
}
- env := config.Environment().Copy()
- env.Set("SKIP_NINJA", "true")
- cmd.Env = env.Environ()
+ cmd.Environment.Set("SKIP_NINJA", "true")
+ cmd.Sandbox = soongSandbox
cmd.Stdin = ctx.Stdin()
cmd.Stdout = ctx.Stdout()
cmd.Stderr = ctx.Stderr()
- ctx.Verboseln(cmd.Path, cmd.Args)
- if err := cmd.Run(); err != nil {
- if e, ok := err.(*exec.ExitError); ok {
- ctx.Fatalln("soong bootstrap failed with:", e.ProcessState.String())
- } else {
- ctx.Fatalln("Failed to run soong bootstrap:", err)
- }
- }
+ cmd.RunOrFatal()
}
