| // Copyright 2017 The Bazel Authors. All rights reserved. |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package main |
| |
| import ( |
| "bytes" |
| "errors" |
| "flag" |
| "fmt" |
| "io" |
| "io/ioutil" |
| "log" |
| "os" |
| "os/exec" |
| "path/filepath" |
| "runtime" |
| "strconv" |
| "strings" |
| ) |
| |
| var ( |
| // cgoEnvVars is the list of all cgo environment variable |
| cgoEnvVars = []string{"CGO_CFLAGS", "CGO_CXXFLAGS", "CGO_CPPFLAGS", "CGO_LDFLAGS"} |
| // cgoAbsEnvFlags are all the flags that need absolute path in cgoEnvVars |
| cgoAbsEnvFlags = []string{"-I", "-L", "-isysroot", "-isystem", "-iquote", "-include", "-gcc-toolchain", "--sysroot", "-resource-dir", "-fsanitize-blacklist", "-fsanitize-ignorelist"} |
| ) |
| |
| // env holds a small amount of Go environment and toolchain information |
| // which is common to multiple builders. Most Bazel-agnostic build information |
| // is collected in go/build.Default though. |
| // |
| // See ./README.rst for more information about handling arguments and |
| // environment variables. |
| type env struct { |
| // sdk is the path to the Go SDK, which contains tools for the host |
| // platform. This may be different than GOROOT. |
| sdk string |
| |
| // installSuffix is the name of the directory below GOROOT/pkg that contains |
| // the .a files for the standard library we should build against. |
| // For example, linux_amd64_race. |
| installSuffix string |
| |
| // verbose indicates whether subprocess command lines should be printed. |
| verbose bool |
| |
| // workDirPath is a temporary work directory. It is created lazily. |
| workDirPath string |
| |
| shouldPreserveWorkDir bool |
| } |
| |
| // envFlags registers flags common to multiple builders and returns an env |
| // configured with those flags. |
| func envFlags(flags *flag.FlagSet) *env { |
| env := &env{} |
| flags.StringVar(&env.sdk, "sdk", "", "Path to the Go SDK.") |
| flags.Var(&tagFlag{}, "tags", "List of build tags considered true.") |
| flags.StringVar(&env.installSuffix, "installsuffix", "", "Standard library under GOROOT/pkg") |
| flags.BoolVar(&env.verbose, "v", false, "Whether subprocess command lines should be printed") |
| flags.BoolVar(&env.shouldPreserveWorkDir, "work", false, "if true, the temporary work directory will be preserved") |
| return env |
| } |
| |
| // checkFlags checks whether env flags were set to valid values. checkFlags |
| // should be called after parsing flags. |
| func (e *env) checkFlags() error { |
| if e.sdk == "" { |
| return errors.New("-sdk was not set") |
| } |
| return nil |
| } |
| |
| // workDir returns a path to a temporary work directory. The same directory |
| // is returned on multiple calls. The caller is responsible for cleaning |
| // up the work directory by calling cleanup. |
| func (e *env) workDir() (path string, cleanup func(), err error) { |
| if e.workDirPath != "" { |
| return e.workDirPath, func() {}, nil |
| } |
| // Keep the stem "rules_go_work" in sync with reproducible_binary_test.go. |
| e.workDirPath, err = ioutil.TempDir("", "rules_go_work-") |
| if err != nil { |
| return "", func() {}, err |
| } |
| if e.verbose { |
| log.Printf("WORK=%s\n", e.workDirPath) |
| } |
| if e.shouldPreserveWorkDir { |
| cleanup = func() {} |
| } else { |
| cleanup = func() { os.RemoveAll(e.workDirPath) } |
| } |
| return e.workDirPath, cleanup, nil |
| } |
| |
| // goTool returns a slice containing the path to an executable at |
| // $GOROOT/pkg/$GOOS_$GOARCH/$tool and additional arguments. |
| func (e *env) goTool(tool string, args ...string) []string { |
| platform := fmt.Sprintf("%s_%s", runtime.GOOS, runtime.GOARCH) |
| toolPath := filepath.Join(e.sdk, "pkg", "tool", platform, tool) |
| if runtime.GOOS == "windows" { |
| toolPath += ".exe" |
| } |
| return append([]string{toolPath}, args...) |
| } |
| |
| // goCmd returns a slice containing the path to the go executable |
| // and additional arguments. |
| func (e *env) goCmd(cmd string, args ...string) []string { |
| exe := filepath.Join(e.sdk, "bin", "go") |
| if runtime.GOOS == "windows" { |
| exe += ".exe" |
| } |
| return append([]string{exe, cmd}, args...) |
| } |
| |
| // runCommand executes a subprocess that inherits stdout, stderr, and the |
| // environment from this process. |
| func (e *env) runCommand(args []string) error { |
| cmd := exec.Command(args[0], args[1:]...) |
| // Redirecting stdout to stderr. This mirrors behavior in the go command: |
| // https://go.googlesource.com/go/+/refs/tags/go1.15.2/src/cmd/go/internal/work/exec.go#1958 |
| buf := &bytes.Buffer{} |
| cmd.Stdout = buf |
| cmd.Stderr = buf |
| err := runAndLogCommand(cmd, e.verbose) |
| os.Stderr.Write(relativizePaths(buf.Bytes())) |
| return err |
| } |
| |
| // runCommandToFile executes a subprocess and writes stdout/stderr to the given |
| // writers. |
| func (e *env) runCommandToFile(out, err io.Writer, args []string) error { |
| cmd := exec.Command(args[0], args[1:]...) |
| cmd.Stdout = out |
| cmd.Stderr = err |
| return runAndLogCommand(cmd, e.verbose) |
| } |
| |
| func absEnv(envNameList []string, argList []string) error { |
| for _, envName := range envNameList { |
| splitedEnv := strings.Fields(os.Getenv(envName)) |
| absArgs(splitedEnv, argList) |
| if err := os.Setenv(envName, strings.Join(splitedEnv, " ")); err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func runAndLogCommand(cmd *exec.Cmd, verbose bool) error { |
| if verbose { |
| fmt.Fprintln(os.Stderr, formatCommand(cmd)) |
| } |
| cleanup := passLongArgsInResponseFiles(cmd) |
| defer cleanup() |
| if err := cmd.Run(); err != nil { |
| return fmt.Errorf("error running subcommand %s: %v", cmd.Path, err) |
| } |
| return nil |
| } |
| |
| // expandParamsFiles looks for arguments in args of the form |
| // "-param=filename". When it finds these arguments it reads the file "filename" |
| // and replaces the argument with its content. |
| // It returns the expanded arguments as well as a bool that is true if any param |
| // files have been passed. |
| func expandParamsFiles(args []string) ([]string, bool, error) { |
| var paramsIndices []int |
| for i, arg := range args { |
| if strings.HasPrefix(arg, "-param=") { |
| paramsIndices = append(paramsIndices, i) |
| } |
| } |
| if len(paramsIndices) == 0 { |
| return args, false, nil |
| } |
| var expandedArgs []string |
| last := 0 |
| for _, pi := range paramsIndices { |
| expandedArgs = append(expandedArgs, args[last:pi]...) |
| last = pi + 1 |
| |
| fileName := args[pi][len("-param="):] |
| fileArgs, err := readParamsFile(fileName) |
| if err != nil { |
| return nil, true, err |
| } |
| expandedArgs = append(expandedArgs, fileArgs...) |
| } |
| expandedArgs = append(expandedArgs, args[last:]...) |
| return expandedArgs, true, nil |
| } |
| |
| // readParamsFiles parses a Bazel params file in "shell" format. The file |
| // should contain one argument per line. Arguments may be quoted with single |
| // quotes. All characters within quoted strings are interpreted literally |
| // including newlines and excepting single quotes. Characters outside quoted |
| // strings may be escaped with a backslash. |
| func readParamsFile(name string) ([]string, error) { |
| data, err := ioutil.ReadFile(name) |
| if err != nil { |
| return nil, err |
| } |
| |
| var args []string |
| var arg []byte |
| quote := false |
| escape := false |
| for p := 0; p < len(data); p++ { |
| b := data[p] |
| switch { |
| case escape: |
| arg = append(arg, b) |
| escape = false |
| |
| case b == '\'': |
| quote = !quote |
| |
| case !quote && b == '\\': |
| escape = true |
| |
| case !quote && b == '\n': |
| args = append(args, string(arg)) |
| arg = arg[:0] |
| |
| default: |
| arg = append(arg, b) |
| } |
| } |
| if quote { |
| return nil, fmt.Errorf("unterminated quote") |
| } |
| if escape { |
| return nil, fmt.Errorf("unterminated escape") |
| } |
| if len(arg) > 0 { |
| args = append(args, string(arg)) |
| } |
| return args, nil |
| } |
| |
| // writeParamsFile formats a list of arguments in Bazel's "shell" format and writes |
| // it to a file. |
| func writeParamsFile(path string, args []string) error { |
| buf := new(bytes.Buffer) |
| for _, arg := range args { |
| if !strings.ContainsAny(arg, "'\n\\") { |
| fmt.Fprintln(buf, arg) |
| continue |
| } |
| buf.WriteByte('\'') |
| for _, r := range arg { |
| if r == '\'' { |
| buf.WriteString(`'\''`) |
| } else { |
| buf.WriteRune(r) |
| } |
| } |
| buf.WriteString("'\n") |
| } |
| return ioutil.WriteFile(path, buf.Bytes(), 0666) |
| } |
| |
| // splitArgs splits a list of command line arguments into two parts: arguments |
| // that should be interpreted by the builder (before "--"), and arguments |
| // that should be passed through to the underlying tool (after "--"). |
| func splitArgs(args []string) (builderArgs []string, toolArgs []string) { |
| for i, arg := range args { |
| if arg == "--" { |
| return args[:i], args[i+1:] |
| } |
| } |
| return args, nil |
| } |
| |
| // abs returns the absolute representation of path. Some tools/APIs require |
| // absolute paths to work correctly. Most notably, golang on Windows cannot |
| // handle relative paths to files whose absolute path is > ~250 chars, while |
| // it can handle absolute paths. See http://goo.gl/eqeWjm. |
| // |
| // Note that strings that begin with "__BAZEL_" are not absolutized. These are |
| // used on macOS for paths that the compiler wrapper (wrapped_clang) is |
| // supposed to know about. |
| func abs(path string) string { |
| if strings.HasPrefix(path, "__BAZEL_") { |
| return path |
| } |
| |
| if abs, err := filepath.Abs(path); err != nil { |
| return path |
| } else { |
| return abs |
| } |
| } |
| |
| // absArgs applies abs to strings that appear in args. Only paths that are |
| // part of options named by flags are modified. |
| func absArgs(args []string, flags []string) { |
| absNext := false |
| for i := range args { |
| if absNext { |
| args[i] = abs(args[i]) |
| absNext = false |
| continue |
| } |
| for _, f := range flags { |
| if !strings.HasPrefix(args[i], f) { |
| continue |
| } |
| possibleValue := args[i][len(f):] |
| if len(possibleValue) == 0 { |
| absNext = true |
| break |
| } |
| separator := "" |
| if possibleValue[0] == '=' { |
| possibleValue = possibleValue[1:] |
| separator = "=" |
| } |
| args[i] = fmt.Sprintf("%s%s%s", f, separator, abs(possibleValue)) |
| break |
| } |
| } |
| } |
| |
| // relativizePaths converts absolute paths found in the given output string to |
| // relative, if they are within the working directory. |
| func relativizePaths(output []byte) []byte { |
| dir, err := os.Getwd() |
| if dir == "" || err != nil { |
| return output |
| } |
| dirBytes := make([]byte, len(dir), len(dir)+1) |
| copy(dirBytes, dir) |
| if bytes.HasSuffix(dirBytes, []byte{filepath.Separator}) { |
| return bytes.ReplaceAll(output, dirBytes, nil) |
| } |
| |
| // This is the common case. |
| // Replace "$CWD/" with "" and "$CWD" with "." |
| dirBytes = append(dirBytes, filepath.Separator) |
| output = bytes.ReplaceAll(output, dirBytes, nil) |
| dirBytes = dirBytes[:len(dirBytes)-1] |
| return bytes.ReplaceAll(output, dirBytes, []byte{'.'}) |
| } |
| |
| // formatCommand formats cmd as a string that can be pasted into a shell. |
| // Spaces in environment variables and arguments are escaped as needed. |
| func formatCommand(cmd *exec.Cmd) string { |
| quoteIfNeeded := func(s string) string { |
| if strings.IndexByte(s, ' ') < 0 { |
| return s |
| } |
| return strconv.Quote(s) |
| } |
| quoteEnvIfNeeded := func(s string) string { |
| eq := strings.IndexByte(s, '=') |
| if eq < 0 { |
| return s |
| } |
| key, value := s[:eq], s[eq+1:] |
| if strings.IndexByte(value, ' ') < 0 { |
| return s |
| } |
| return fmt.Sprintf("%s=%s", key, strconv.Quote(value)) |
| } |
| var w bytes.Buffer |
| environ := cmd.Env |
| if environ == nil { |
| environ = os.Environ() |
| } |
| for _, e := range environ { |
| fmt.Fprintf(&w, "%s \\\n", quoteEnvIfNeeded(e)) |
| } |
| |
| sep := "" |
| for _, arg := range cmd.Args { |
| fmt.Fprintf(&w, "%s%s", sep, quoteIfNeeded(arg)) |
| sep = " " |
| } |
| return w.String() |
| } |
| |
| // passLongArgsInResponseFiles modifies cmd such that, for |
| // certain programs, long arguments are passed in "response files", a |
| // file on disk with the arguments, with one arg per line. An actual |
| // argument starting with '@' means that the rest of the argument is |
| // a filename of arguments to expand. |
| // |
| // See https://github.com/golang/go/issues/18468 (Windows) and |
| // https://github.com/golang/go/issues/37768 (Darwin). |
| func passLongArgsInResponseFiles(cmd *exec.Cmd) (cleanup func()) { |
| cleanup = func() {} // no cleanup by default |
| var argLen int |
| for _, arg := range cmd.Args { |
| argLen += len(arg) |
| } |
| // If we're not approaching 32KB of args, just pass args normally. |
| // (use 30KB instead to be conservative; not sure how accounting is done) |
| if !useResponseFile(cmd.Path, argLen) { |
| return |
| } |
| tf, err := ioutil.TempFile("", "args") |
| if err != nil { |
| log.Fatalf("error writing long arguments to response file: %v", err) |
| } |
| cleanup = func() { os.Remove(tf.Name()) } |
| var buf bytes.Buffer |
| for _, arg := range cmd.Args[1:] { |
| fmt.Fprintf(&buf, "%s\n", arg) |
| } |
| if _, err := tf.Write(buf.Bytes()); err != nil { |
| tf.Close() |
| cleanup() |
| log.Fatalf("error writing long arguments to response file: %v", err) |
| } |
| if err := tf.Close(); err != nil { |
| cleanup() |
| log.Fatalf("error writing long arguments to response file: %v", err) |
| } |
| cmd.Args = []string{cmd.Args[0], "@" + tf.Name()} |
| return cleanup |
| } |
| |
| func useResponseFile(path string, argLen int) bool { |
| // Unless the program uses objabi.Flagparse, which understands |
| // response files, don't use response files. |
| // TODO: do we need more commands? asm? cgo? For now, no. |
| prog := strings.TrimSuffix(filepath.Base(path), ".exe") |
| switch prog { |
| case "compile", "link": |
| default: |
| return false |
| } |
| // Windows has a limit of 32 KB arguments. To be conservative and not |
| // worry about whether that includes spaces or not, just use 30 KB. |
| // Darwin's limit is less clear. The OS claims 256KB, but we've seen |
| // failures with arglen as small as 50KB. |
| if argLen > (30 << 10) { |
| return true |
| } |
| return false |
| } |