exec.go - platform/build/kati - Git at Google

 // Copyright 2015 Google Inc. All rights reserved
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
 // You may obtain a copy of the License at
 //
 //      http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing, software
 // distributed under the License is distributed on an "AS IS" BASIS,
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.

 package kati

 import (
 	"bytes"
 	"fmt"
 	"io"
 	"path/filepath"
 	"strings"
 	"sync"
 	"time"
 )

 // Executor manages execution of makefile rules.
 type Executor struct {
 	rules         map[string]*rule
 	implicitRules []*rule
 	suffixRules   map[string][]*rule
 	firstRule     *rule
 	shell         string
 	vars          Vars
 	varsLock      sync.Mutex
 	// target -> Job, nil means the target is currently being processed.
 	done map[string]*job

 	wm *workerManager

 	currentOutput string
 	currentInputs []string
 	currentStem   string

 	trace          []string
 	buildCnt       int
 	alreadyDoneCnt int
 	noRuleCnt      int
 	upToDateCnt    int
 	runCommandCnt  int
 }

 type autoVar struct{ ex *Executor }

 func (v autoVar) Flavor() string  { return "undefined" }
 func (v autoVar) Origin() string  { return "automatic" }
 func (v autoVar) IsDefined() bool { return true }
 func (v autoVar) Append(*Evaluator, string) (Var, error) {
 	return nil, fmt.Errorf("cannot append to autovar")
 }
 func (v autoVar) AppendVar(*Evaluator, Value) (Var, error) {
 	return nil, fmt.Errorf("cannot append to autovar")
 }
 func (v autoVar) serialize() serializableVar {
 	return serializableVar{Type: ""}
 }
 func (v autoVar) dump(d *dumpbuf) {
 	d.err = fmt.Errorf("cannot dump auto var: %v", v)
 }

 type autoAtVar struct{ autoVar }

 func (v autoAtVar) Eval(w io.Writer, ev *Evaluator) error {
 	fmt.Fprint(w, v.ex.currentOutput)
 	return nil
 }
 func (v autoAtVar) String() string { return "$*" }

 type autoLessVar struct{ autoVar }

 func (v autoLessVar) Eval(w io.Writer, ev *Evaluator) error {
 	if len(v.ex.currentInputs) > 0 {
 		fmt.Fprint(w, v.ex.currentInputs[0])
 	}
 	return nil
 }
 func (v autoLessVar) String() string { return "$<" }

 type autoHatVar struct{ autoVar }

 func (v autoHatVar) Eval(w io.Writer, ev *Evaluator) error {
 	var uniqueInputs []string
 	seen := make(map[string]bool)
 	for _, input := range v.ex.currentInputs {
 		if !seen[input] {
 			seen[input] = true
 			uniqueInputs = append(uniqueInputs, input)
 		}
 	}
 	fmt.Fprint(w, strings.Join(uniqueInputs, " "))
 	return nil
 }
 func (v autoHatVar) String() string { return "$^" }

 type autoPlusVar struct{ autoVar }

 func (v autoPlusVar) Eval(w io.Writer, ev *Evaluator) error {
 	fmt.Fprint(w, strings.Join(v.ex.currentInputs, " "))
 	return nil
 }
 func (v autoPlusVar) String() string { return "$+" }

 type autoStarVar struct{ autoVar }

 func (v autoStarVar) Eval(w io.Writer, ev *Evaluator) error {
 	// TODO: Use currentStem. See auto_stem_var.mk
 	fmt.Fprint(w, stripExt(v.ex.currentOutput))
 	return nil
 }
 func (v autoStarVar) String() string { return "$*" }

 type autoSuffixDVar struct {
 	autoVar
 	v Var
 }

 func (v autoSuffixDVar) Eval(w io.Writer, ev *Evaluator) error {
 	var buf bytes.Buffer
 	err := v.v.Eval(&buf, ev)
 	if err != nil {
 		return err
 	}
 	ws := newWordScanner(buf.Bytes())
 	sw := ssvWriter{w: w}
 	for ws.Scan() {
 		sw.WriteString(filepath.Dir(string(ws.Bytes())))
 	}
 	return nil
 }

 func (v autoSuffixDVar) String() string { return v.v.String() + "D" }

 type autoSuffixFVar struct {
 	autoVar
 	v Var
 }

 func (v autoSuffixFVar) Eval(w io.Writer, ev *Evaluator) error {
 	var buf bytes.Buffer
 	err := v.v.Eval(&buf, ev)
 	if err != nil {
 		return err
 	}
 	ws := newWordScanner(buf.Bytes())
 	sw := ssvWriter{w: w}
 	for ws.Scan() {
 		sw.WriteString(filepath.Base(string(ws.Bytes())))
 	}
 	return nil
 }

 func (v autoSuffixFVar) String() string { return v.v.String() + "F" }

 func (ex *Executor) makeJobs(n *DepNode, neededBy *job) error {
 	output := n.Output
 	if neededBy != nil {
 		logf("MakeJob: %s for %s", output, neededBy.n.Output)
 	}
 	ex.buildCnt++
 	if ex.buildCnt%100 == 0 {
 		ex.reportStats()
 	}

 	j, present := ex.done[output]

 	if present {
 		if j == nil {
 			if !n.IsPhony {
 				fmt.Printf("Circular %s <- %s dependency dropped.\n", neededBy.n.Output, n.Output)
 			}
 			if neededBy != nil {
 				neededBy.numDeps--
 			}
 		} else {
 			logf("%s already done: %d", j.n.Output, j.outputTs)
 			if neededBy != nil {
 				ex.wm.ReportNewDep(j, neededBy)
 			}
 		}
 		return nil
 	}

 	j = &job{
 		n:       n,
 		ex:      ex,
 		numDeps: len(n.Deps) + len(n.OrderOnlys),
 		depsTs:  int64(-1),
 	}
 	if neededBy != nil {
 		j.parents = append(j.parents, neededBy)
 	}

 	ex.done[output] = nil
 	// We iterate n.Deps twice. In the first run, we may modify
 	// numDeps. There will be a race if we do so after the first
 	// ex.makeJobs(d, j).
 	var deps []*DepNode
 	for _, d := range n.Deps {
 		deps = append(deps, d)
 	}
 	for _, d := range n.OrderOnlys {
 		if exists(d.Output) {
 			j.numDeps--
 			continue
 		}
 		deps = append(deps, d)
 	}
 	logf("new: %s (%d)", j.n.Output, j.numDeps)

 	for _, d := range deps {
 		ex.trace = append(ex.trace, d.Output)
 		err := ex.makeJobs(d, j)
 		ex.trace = ex.trace[0 : len(ex.trace)-1]
 		if err != nil {
 			return err
 		}
 	}

 	ex.done[output] = j
 	return ex.wm.PostJob(j)
 }

 func (ex *Executor) reportStats() {
 	if !LogFlag && !PeriodicStatsFlag {
 		return
 	}

 	logStats("build=%d alreadyDone=%d noRule=%d, upToDate=%d runCommand=%d",
 		ex.buildCnt, ex.alreadyDoneCnt, ex.noRuleCnt, ex.upToDateCnt, ex.runCommandCnt)
 	if len(ex.trace) > 1 {
 		logStats("trace=%q", ex.trace)
 	}
 }

 // ExecutorOpt is an option for Executor.
 type ExecutorOpt struct {
 	NumJobs  int
 	ParaPath string
 }

 // NewExecutor creates new Executor.
 func NewExecutor(vars Vars, opt *ExecutorOpt) (*Executor, error) {
 	if opt == nil {
 		opt = &ExecutorOpt{NumJobs: 1}
 	}
 	if opt.NumJobs < 1 {
 		opt.NumJobs = 1
 	}
 	wm, err := newWorkerManager(opt.NumJobs, opt.ParaPath)
 	if err != nil {
 		return nil, err
 	}
 	ex := &Executor{
 		rules:       make(map[string]*rule),
 		suffixRules: make(map[string][]*rule),
 		done:        make(map[string]*job),
 		vars:        vars,
 		wm:          wm,
 	}
 	// TODO: We should move this to somewhere around evalCmd so that
 	// we can handle SHELL in target specific variables.
 	ev := NewEvaluator(ex.vars)
 	ex.shell, err = ev.EvaluateVar("SHELL")
 	if err != nil {
 		ex.shell = "/bin/sh"
 	}
 	for k, v := range map[string]Var{
 		"@": autoAtVar{autoVar: autoVar{ex: ex}},
 		"<": autoLessVar{autoVar: autoVar{ex: ex}},
 		"^": autoHatVar{autoVar: autoVar{ex: ex}},
 		"+": autoPlusVar{autoVar: autoVar{ex: ex}},
 		"*": autoStarVar{autoVar: autoVar{ex: ex}},
 	} {
 		ex.vars[k] = v
 		ex.vars[k+"D"] = autoSuffixDVar{v: v}
 		ex.vars[k+"F"] = autoSuffixFVar{v: v}
 	}
 	return ex, nil
 }

 // Exec executes to build roots.
 func (ex *Executor) Exec(roots []*DepNode) error {
 	startTime := time.Now()
 	for _, root := range roots {
 		err := ex.makeJobs(root, nil)
 		if err != nil {
 			break
 		}
 	}
 	err := ex.wm.Wait()
 	logStats("exec time: %q", time.Since(startTime))
 	return err
 }

 func (ex *Executor) createRunners(n *DepNode, avoidIO bool) ([]runner, bool, error) {
 	var runners []runner
 	if len(n.Cmds) == 0 {
 		return runners, false, nil
 	}

 	var restores []func()
 	defer func() {
 		for i := len(restores) - 1; i >= 0; i-- {
 			restores[i]()
 		}
 	}()

 	ex.varsLock.Lock()
 	restores = append(restores, func() { ex.varsLock.Unlock() })
 	// For automatic variables.
 	ex.currentOutput = n.Output
 	ex.currentInputs = n.ActualInputs
 	for k, v := range n.TargetSpecificVars {
 		restores = append(restores, ex.vars.save(k))
 		ex.vars[k] = v
 		logf("tsv: %s=%s", k, v)
 	}

 	ev := NewEvaluator(ex.vars)
 	ev.avoidIO = avoidIO
 	ev.filename = n.Filename
 	ev.lineno = n.Lineno
 	logf("Building: %s cmds:%q", n.Output, n.Cmds)
 	r := runner{
 		output: n.Output,
 		echo:   true,
 		shell:  ex.shell,
 	}
 	for _, cmd := range n.Cmds {
 		rr, err := evalCmd(ev, r, cmd)
 		if err != nil {
 			return nil, false, err
 		}
 		for _, r := range rr {
 			if len(r.cmd) != 0 {
 				runners = append(runners, r)
 			}
 		}
 	}
 	return runners, ev.hasIO, nil
 }

 func evalCommands(nodes []*DepNode, vars Vars) error {
 	ioCnt := 0
 	ex, err := NewExecutor(vars, nil)
 	if err != nil {
 		return err
 	}
 	for i, n := range nodes {
 		runners, hasIO, err := ex.createRunners(n, true)
 		if err != nil {
 			return err
 		}
 		if hasIO {
 			ioCnt++
 			if ioCnt%100 == 0 {
 				logStats("%d/%d rules have IO", ioCnt, i+1)
 			}
 			continue
 		}

 		n.Cmds = []string{}
 		n.TargetSpecificVars = make(Vars)
 		for _, r := range runners {
 			cmd := r.cmd
 			// TODO: Do not preserve the effect of dryRunFlag.
 			if r.echo {
 				cmd = "@" + cmd
 			}
 			if r.ignoreError {
 				cmd = "-" + cmd
 			}
 			n.Cmds = append(n.Cmds, cmd)
 		}
 	}

 	err = ex.wm.Wait()
 	logStats("%d/%d rules have IO", ioCnt, len(nodes))
 	return err
 }
	// Copyright 2015 Google Inc. All rights reserved
	//
	// Licensed under the Apache License, Version 2.0 (the "License");
	// you may not use this file except in compliance with the License.
	// You may obtain a copy of the License at
	//
	// http://www.apache.org/licenses/LICENSE-2.0
	//
	// Unless required by applicable law or agreed to in writing, software
	// distributed under the License is distributed on an "AS IS" BASIS,
	// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
	// See the License for the specific language governing permissions and
	// limitations under the License.

	package kati

	import (
	"bytes"
	"fmt"
	"io"
	"path/filepath"
	"strings"
	"sync"
	"time"
	)

	// Executor manages execution of makefile rules.
	type Executor struct {
	rules map[string]*rule
	implicitRules []*rule
	suffixRules map[string][]*rule
	firstRule *rule
	shell string
	vars Vars
	varsLock sync.Mutex
	// target -> Job, nil means the target is currently being processed.
	done map[string]*job

	wm *workerManager

	currentOutput string
	currentInputs []string
	currentStem string

	trace []string
	buildCnt int
	alreadyDoneCnt int
	noRuleCnt int
	upToDateCnt int
	runCommandCnt int
	}

	type autoVar struct{ ex *Executor }

	func (v autoVar) Flavor() string { return "undefined" }
	func (v autoVar) Origin() string { return "automatic" }
	func (v autoVar) IsDefined() bool { return true }
	func (v autoVar) Append(*Evaluator, string) (Var, error) {
	return nil, fmt.Errorf("cannot append to autovar")
	}
	func (v autoVar) AppendVar(*Evaluator, Value) (Var, error) {
	return nil, fmt.Errorf("cannot append to autovar")
	}
	func (v autoVar) serialize() serializableVar {
	return serializableVar{Type: ""}
	}
	func (v autoVar) dump(d *dumpbuf) {
	d.err = fmt.Errorf("cannot dump auto var: %v", v)
	}

	type autoAtVar struct{ autoVar }

	func (v autoAtVar) Eval(w io.Writer, ev *Evaluator) error {
	fmt.Fprint(w, v.ex.currentOutput)
	return nil
	}
	func (v autoAtVar) String() string { return "$*" }

	type autoLessVar struct{ autoVar }

	func (v autoLessVar) Eval(w io.Writer, ev *Evaluator) error {
	if len(v.ex.currentInputs) > 0 {
	fmt.Fprint(w, v.ex.currentInputs[0])
	}
	return nil
	}
	func (v autoLessVar) String() string { return "$<" }

	type autoHatVar struct{ autoVar }

	func (v autoHatVar) Eval(w io.Writer, ev *Evaluator) error {
	var uniqueInputs []string
	seen := make(map[string]bool)
	for _, input := range v.ex.currentInputs {
	if !seen[input] {
	seen[input] = true
	uniqueInputs = append(uniqueInputs, input)
	}
	}
	fmt.Fprint(w, strings.Join(uniqueInputs, " "))
	return nil
	}
	func (v autoHatVar) String() string { return "$^" }

	type autoPlusVar struct{ autoVar }

	func (v autoPlusVar) Eval(w io.Writer, ev *Evaluator) error {
	fmt.Fprint(w, strings.Join(v.ex.currentInputs, " "))
	return nil
	}
	func (v autoPlusVar) String() string { return "$+" }

	type autoStarVar struct{ autoVar }

	func (v autoStarVar) Eval(w io.Writer, ev *Evaluator) error {
	// TODO: Use currentStem. See auto_stem_var.mk
	fmt.Fprint(w, stripExt(v.ex.currentOutput))
	return nil
	}
	func (v autoStarVar) String() string { return "$*" }

	type autoSuffixDVar struct {
	autoVar
	v Var
	}

	func (v autoSuffixDVar) Eval(w io.Writer, ev *Evaluator) error {
	var buf bytes.Buffer
	err := v.v.Eval(&buf, ev)
	if err != nil {
	return err
	}
	ws := newWordScanner(buf.Bytes())
	sw := ssvWriter{w: w}
	for ws.Scan() {
	sw.WriteString(filepath.Dir(string(ws.Bytes())))
	}
	return nil
	}

	func (v autoSuffixDVar) String() string { return v.v.String() + "D" }

	type autoSuffixFVar struct {
	autoVar
	v Var
	}

	func (v autoSuffixFVar) Eval(w io.Writer, ev *Evaluator) error {
	var buf bytes.Buffer
	err := v.v.Eval(&buf, ev)
	if err != nil {
	return err
	}
	ws := newWordScanner(buf.Bytes())
	sw := ssvWriter{w: w}
	for ws.Scan() {
	sw.WriteString(filepath.Base(string(ws.Bytes())))
	}
	return nil
	}

	func (v autoSuffixFVar) String() string { return v.v.String() + "F" }

	func (ex Executor) makeJobs(n DepNode, neededBy *job) error {
	output := n.Output
	if neededBy != nil {
	logf("MakeJob: %s for %s", output, neededBy.n.Output)
	}
	ex.buildCnt++
	if ex.buildCnt%100 == 0 {
	ex.reportStats()
	}

	j, present := ex.done[output]

	if present {
	if j == nil {
	if !n.IsPhony {
	fmt.Printf("Circular %s <- %s dependency dropped.\n", neededBy.n.Output, n.Output)
	}
	if neededBy != nil {
	neededBy.numDeps--
	}
	} else {
	logf("%s already done: %d", j.n.Output, j.outputTs)
	if neededBy != nil {
	ex.wm.ReportNewDep(j, neededBy)
	}
	}
	return nil
	}

	j = &job{
	n: n,
	ex: ex,
	numDeps: len(n.Deps) + len(n.OrderOnlys),
	depsTs: int64(-1),
	}
	if neededBy != nil {
	j.parents = append(j.parents, neededBy)
	}

	ex.done[output] = nil
	// We iterate n.Deps twice. In the first run, we may modify
	// numDeps. There will be a race if we do so after the first
	// ex.makeJobs(d, j).
	var deps []*DepNode
	for _, d := range n.Deps {
	deps = append(deps, d)
	}
	for _, d := range n.OrderOnlys {
	if exists(d.Output) {
	j.numDeps--
	continue
	}
	deps = append(deps, d)
	}
	logf("new: %s (%d)", j.n.Output, j.numDeps)

	for _, d := range deps {
	ex.trace = append(ex.trace, d.Output)
	err := ex.makeJobs(d, j)
	ex.trace = ex.trace[0 : len(ex.trace)-1]
	if err != nil {
	return err
	}
	}

	ex.done[output] = j
	return ex.wm.PostJob(j)
	}

	func (ex *Executor) reportStats() {
	if !LogFlag && !PeriodicStatsFlag {
	return
	}

	logStats("build=%d alreadyDone=%d noRule=%d, upToDate=%d runCommand=%d",
	ex.buildCnt, ex.alreadyDoneCnt, ex.noRuleCnt, ex.upToDateCnt, ex.runCommandCnt)
	if len(ex.trace) > 1 {
	logStats("trace=%q", ex.trace)
	}
	}

	// ExecutorOpt is an option for Executor.
	type ExecutorOpt struct {
	NumJobs int
	ParaPath string
	}

	// NewExecutor creates new Executor.
	func NewExecutor(vars Vars, opt ExecutorOpt) (Executor, error) {
	if opt == nil {
	opt = &ExecutorOpt{NumJobs: 1}
	}
	if opt.NumJobs < 1 {
	opt.NumJobs = 1
	}
	wm, err := newWorkerManager(opt.NumJobs, opt.ParaPath)
	if err != nil {
	return nil, err
	}
	ex := &Executor{
	rules: make(map[string]*rule),
	suffixRules: make(map[string][]*rule),
	done: make(map[string]*job),
	vars: vars,
	wm: wm,
	}
	// TODO: We should move this to somewhere around evalCmd so that
	// we can handle SHELL in target specific variables.
	ev := NewEvaluator(ex.vars)
	ex.shell, err = ev.EvaluateVar("SHELL")
	if err != nil {
	ex.shell = "/bin/sh"
	}
	for k, v := range map[string]Var{
	"@": autoAtVar{autoVar: autoVar{ex: ex}},
	"<": autoLessVar{autoVar: autoVar{ex: ex}},
	"^": autoHatVar{autoVar: autoVar{ex: ex}},
	"+": autoPlusVar{autoVar: autoVar{ex: ex}},
	"*": autoStarVar{autoVar: autoVar{ex: ex}},
	} {
	ex.vars[k] = v
	ex.vars[k+"D"] = autoSuffixDVar{v: v}
	ex.vars[k+"F"] = autoSuffixFVar{v: v}
	}
	return ex, nil
	}

	// Exec executes to build roots.
	func (ex Executor) Exec(roots []DepNode) error {
	startTime := time.Now()
	for _, root := range roots {
	err := ex.makeJobs(root, nil)
	if err != nil {
	break
	}
	}
	err := ex.wm.Wait()
	logStats("exec time: %q", time.Since(startTime))
	return err
	}

	func (ex Executor) createRunners(n DepNode, avoidIO bool) ([]runner, bool, error) {
	var runners []runner
	if len(n.Cmds) == 0 {
	return runners, false, nil
	}

	var restores []func()
	defer func() {
	for i := len(restores) - 1; i >= 0; i-- {
	restores[i]()
	}
	}()

	ex.varsLock.Lock()
	restores = append(restores, func() { ex.varsLock.Unlock() })
	// For automatic variables.
	ex.currentOutput = n.Output
	ex.currentInputs = n.ActualInputs
	for k, v := range n.TargetSpecificVars {
	restores = append(restores, ex.vars.save(k))
	ex.vars[k] = v
	logf("tsv: %s=%s", k, v)
	}

	ev := NewEvaluator(ex.vars)
	ev.avoidIO = avoidIO
	ev.filename = n.Filename
	ev.lineno = n.Lineno
	logf("Building: %s cmds:%q", n.Output, n.Cmds)
	r := runner{
	output: n.Output,
	echo: true,
	shell: ex.shell,
	}
	for _, cmd := range n.Cmds {
	rr, err := evalCmd(ev, r, cmd)
	if err != nil {
	return nil, false, err
	}
	for _, r := range rr {
	if len(r.cmd) != 0 {
	runners = append(runners, r)
	}
	}
	}
	return runners, ev.hasIO, nil
	}

	func evalCommands(nodes []*DepNode, vars Vars) error {
	ioCnt := 0
	ex, err := NewExecutor(vars, nil)
	if err != nil {
	return err
	}
	for i, n := range nodes {
	runners, hasIO, err := ex.createRunners(n, true)
	if err != nil {
	return err
	}
	if hasIO {
	ioCnt++
	if ioCnt%100 == 0 {
	logStats("%d/%d rules have IO", ioCnt, i+1)
	}
	continue
	}

	n.Cmds = []string{}
	n.TargetSpecificVars = make(Vars)
	for _, r := range runners {
	cmd := r.cmd
	// TODO: Do not preserve the effect of dryRunFlag.
	if r.echo {
	cmd = "@" + cmd
	}
	if r.ignoreError {
	cmd = "-" + cmd
	}
	n.Cmds = append(n.Cmds, cmd)
	}
	}

	err = ex.wm.Wait()
	logStats("%d/%d rules have IO", ioCnt, len(nodes))
	return err
	}