| // Copyright 2015 Google Inc. All rights reserved |
| // |
| // Licensed under the Apache License, Version 2.0 (the "License"); |
| // you may not use this file except in compliance with the License. |
| // You may obtain a copy of the License at |
| // |
| // http://www.apache.org/licenses/LICENSE-2.0 |
| // |
| // Unless required by applicable law or agreed to in writing, software |
| // distributed under the License is distributed on an "AS IS" BASIS, |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| // See the License for the specific language governing permissions and |
| // limitations under the License. |
| |
| package kati |
| |
| import ( |
| "bytes" |
| "crypto/sha1" |
| "fmt" |
| "os" |
| "path/filepath" |
| "strconv" |
| "strings" |
| "sync" |
| |
| "github.com/golang/glog" |
| ) |
| |
| type fileState int |
| |
| const ( |
| fileExists fileState = iota |
| fileNotExists |
| fileInconsistent // Modified during kati is running. |
| ) |
| |
| type accessedMakefile struct { |
| Filename string |
| Hash [sha1.Size]byte |
| State fileState |
| } |
| |
| type accessCache struct { |
| mu sync.Mutex |
| m map[string]*accessedMakefile |
| } |
| |
| func newAccessCache() *accessCache { |
| return &accessCache{ |
| m: make(map[string]*accessedMakefile), |
| } |
| } |
| |
| func (ac *accessCache) update(fn string, hash [sha1.Size]byte, st fileState) string { |
| if ac == nil { |
| return "" |
| } |
| ac.mu.Lock() |
| defer ac.mu.Unlock() |
| rm, present := ac.m[fn] |
| if present { |
| switch rm.State { |
| case fileExists: |
| if st != fileExists { |
| return fmt.Sprintf("%s was removed after the previous read", fn) |
| } else if !bytes.Equal(hash[:], rm.Hash[:]) { |
| ac.m[fn].State = fileInconsistent |
| return fmt.Sprintf("%s was modified after the previous read", fn) |
| } |
| return "" |
| case fileNotExists: |
| if st != fileNotExists { |
| ac.m[fn].State = fileInconsistent |
| return fmt.Sprintf("%s was created after the previous read", fn) |
| } |
| case fileInconsistent: |
| return "" |
| } |
| return "" |
| } |
| ac.m[fn] = &accessedMakefile{ |
| Filename: fn, |
| Hash: hash, |
| State: st, |
| } |
| return "" |
| } |
| |
| func (ac *accessCache) Slice() []*accessedMakefile { |
| if ac == nil { |
| return nil |
| } |
| ac.mu.Lock() |
| defer ac.mu.Unlock() |
| r := []*accessedMakefile{} |
| for _, v := range ac.m { |
| r = append(r, v) |
| } |
| return r |
| } |
| |
| type evalResult struct { |
| vars Vars |
| rules []*rule |
| ruleVars map[string]Vars |
| accessedMks []*accessedMakefile |
| exports map[string]bool |
| vpaths searchPaths |
| } |
| |
| type srcpos struct { |
| filename string |
| lineno int |
| } |
| |
| func (p srcpos) String() string { |
| return fmt.Sprintf("%s:%d", p.filename, p.lineno) |
| } |
| |
| // EvalError is an error in kati evaluation. |
| type EvalError struct { |
| Filename string |
| Lineno int |
| Err error |
| } |
| |
| func (e EvalError) Error() string { |
| return fmt.Sprintf("%s:%d: %v", e.Filename, e.Lineno, e.Err) |
| } |
| |
| func (p srcpos) errorf(f string, args ...interface{}) error { |
| return EvalError{ |
| Filename: p.filename, |
| Lineno: p.lineno, |
| Err: fmt.Errorf(f, args...), |
| } |
| } |
| |
| func (p srcpos) error(err error) error { |
| if _, ok := err.(EvalError); ok { |
| return err |
| } |
| return EvalError{ |
| Filename: p.filename, |
| Lineno: p.lineno, |
| Err: err, |
| } |
| } |
| |
| // Evaluator manages makefile evaluation. |
| type Evaluator struct { |
| paramVars []tmpval // $1 => paramVars[1] |
| outVars Vars |
| outRules []*rule |
| outRuleVars map[string]Vars |
| vars Vars |
| lastRule *rule |
| currentScope Vars |
| cache *accessCache |
| exports map[string]bool |
| vpaths []vpath |
| |
| avoidIO bool |
| hasIO bool |
| // delayedOutputs are commands which should run at ninja-time |
| // (i.e., info, warning, and error). |
| delayedOutputs []string |
| |
| srcpos |
| } |
| |
| // NewEvaluator creates new Evaluator. |
| func NewEvaluator(vars map[string]Var) *Evaluator { |
| return &Evaluator{ |
| outVars: make(Vars), |
| vars: vars, |
| outRuleVars: make(map[string]Vars), |
| exports: make(map[string]bool), |
| } |
| } |
| |
| func (ev *Evaluator) args(buf *evalBuffer, args ...Value) ([][]byte, error) { |
| pos := make([]int, 0, len(args)) |
| for _, arg := range args { |
| buf.resetSep() |
| err := arg.Eval(buf, ev) |
| if err != nil { |
| return nil, err |
| } |
| pos = append(pos, buf.Len()) |
| } |
| v := buf.Bytes() |
| buf.args = buf.args[:0] |
| s := 0 |
| for _, p := range pos { |
| buf.args = append(buf.args, v[s:p]) |
| s = p |
| } |
| return buf.args, nil |
| } |
| |
| func (ev *Evaluator) evalAssign(ast *assignAST) error { |
| ev.lastRule = nil |
| lhs, rhs, err := ev.evalAssignAST(ast) |
| if err != nil { |
| return err |
| } |
| if glog.V(1) { |
| glog.Infof("ASSIGN: %s=%q (flavor:%q)", lhs, rhs, rhs.Flavor()) |
| } |
| if lhs == "" { |
| return ast.errorf("*** empty variable name.") |
| } |
| ev.outVars.Assign(lhs, rhs) |
| return nil |
| } |
| |
| func (ev *Evaluator) evalAssignAST(ast *assignAST) (string, Var, error) { |
| ev.srcpos = ast.srcpos |
| |
| var lhs string |
| switch v := ast.lhs.(type) { |
| case literal: |
| lhs = string(v) |
| case tmpval: |
| lhs = string(v) |
| default: |
| buf := newEbuf() |
| err := v.Eval(buf, ev) |
| if err != nil { |
| return "", nil, err |
| } |
| lhs = string(trimSpaceBytes(buf.Bytes())) |
| buf.release() |
| } |
| rhs, err := ast.evalRHS(ev, lhs) |
| if err != nil { |
| return "", nil, err |
| } |
| return lhs, rhs, nil |
| } |
| |
| func (ev *Evaluator) setTargetSpecificVar(assign *assignAST, output string) error { |
| vars, present := ev.outRuleVars[output] |
| if !present { |
| vars = make(Vars) |
| ev.outRuleVars[output] = vars |
| } |
| ev.currentScope = vars |
| lhs, rhs, err := ev.evalAssignAST(assign) |
| if err != nil { |
| return err |
| } |
| if glog.V(1) { |
| glog.Infof("rule outputs:%q assign:%q%s%q (flavor:%q)", output, lhs, assign.op, rhs, rhs.Flavor()) |
| } |
| vars.Assign(lhs, &targetSpecificVar{v: rhs, op: assign.op}) |
| ev.currentScope = nil |
| return nil |
| } |
| |
| func (ev *Evaluator) evalMaybeRule(ast *maybeRuleAST) error { |
| ev.lastRule = nil |
| ev.srcpos = ast.srcpos |
| |
| if glog.V(1) { |
| glog.Infof("maybe rule %s: %q assign:%v", ev.srcpos, ast.expr, ast.assign) |
| } |
| |
| abuf := newEbuf() |
| aexpr := toExpr(ast.expr) |
| var rhs expr |
| semi := ast.semi |
| for i, v := range aexpr { |
| var hashFound bool |
| var buf evalBuffer |
| buf.resetSep() |
| switch v.(type) { |
| case literal, tmpval: |
| s := v.String() |
| i := strings.Index(s, "#") |
| if i >= 0 { |
| hashFound = true |
| v = tmpval(trimRightSpaceBytes([]byte(s[:i]))) |
| } |
| } |
| err := v.Eval(&buf, ev) |
| if err != nil { |
| return err |
| } |
| b := buf.Bytes() |
| if ast.isRule { |
| abuf.Write(b) |
| continue |
| } |
| eq := findLiteralChar(b, '=', 0, skipVar) |
| if eq >= 0 { |
| abuf.Write(b[:eq+1]) |
| if eq+1 < len(b) { |
| rhs = append(rhs, tmpval(trimLeftSpaceBytes(b[eq+1:]))) |
| } |
| if i+1 < len(aexpr) { |
| rhs = append(rhs, aexpr[i+1:]...) |
| } |
| if ast.semi != nil { |
| rhs = append(rhs, literal(';')) |
| sexpr, _, err := parseExpr(ast.semi, nil, parseOp{}) |
| if err != nil { |
| return err |
| } |
| rhs = append(rhs, toExpr(sexpr)...) |
| semi = nil |
| } |
| break |
| } |
| abuf.Write(b) |
| if hashFound { |
| break |
| } |
| } |
| |
| line := abuf.Bytes() |
| r := &rule{srcpos: ast.srcpos} |
| if glog.V(1) { |
| glog.Infof("rule? %s: %q assign:%v rhs:%s", r.srcpos, line, ast.assign, rhs) |
| } |
| assign, err := r.parse(line, ast.assign, rhs) |
| if err != nil { |
| ws := newWordScanner(line) |
| if ws.Scan() { |
| if string(ws.Bytes()) == "override" { |
| warnNoPrefix(ast.srcpos, "invalid `override' directive") |
| return nil |
| } |
| } |
| return ast.error(err) |
| } |
| abuf.release() |
| if glog.V(1) { |
| glog.Infof("rule %q assign:%v rhs:%v=> outputs:%q, inputs:%q", ast.expr, ast.assign, rhs, r.outputs, r.inputs) |
| } |
| |
| // TODO: Pretty print. |
| // glog.V(1).Infof("RULE: %s=%s (%d commands)", lhs, rhs, len(cmds)) |
| |
| if assign != nil { |
| glog.V(1).Infof("target specific var: %#v", assign) |
| for _, output := range r.outputs { |
| ev.setTargetSpecificVar(assign, output) |
| } |
| for _, output := range r.outputPatterns { |
| ev.setTargetSpecificVar(assign, output.String()) |
| } |
| return nil |
| } |
| |
| if semi != nil { |
| r.cmds = append(r.cmds, string(semi)) |
| } |
| if glog.V(1) { |
| glog.Infof("rule outputs:%q cmds:%q", r.outputs, r.cmds) |
| } |
| ev.lastRule = r |
| ev.outRules = append(ev.outRules, r) |
| return nil |
| } |
| |
| func (ev *Evaluator) evalCommand(ast *commandAST) error { |
| ev.srcpos = ast.srcpos |
| if ev.lastRule == nil || ev.lastRule.outputs == nil { |
| // This could still be an assignment statement. See |
| // assign_after_tab.mk. |
| if strings.IndexByte(ast.cmd, '=') >= 0 { |
| line := trimLeftSpace(ast.cmd) |
| mk, err := parseMakefileString(line, ast.srcpos) |
| if err != nil { |
| return ast.errorf("parse failed: %q: %v", line, err) |
| } |
| if len(mk.stmts) >= 1 && mk.stmts[len(mk.stmts)-1].(*assignAST) != nil { |
| for _, stmt := range mk.stmts { |
| err = ev.eval(stmt) |
| if err != nil { |
| return err |
| } |
| } |
| } |
| return nil |
| } |
| // Or, a comment is OK. |
| if strings.TrimSpace(ast.cmd)[0] == '#' { |
| return nil |
| } |
| return ast.errorf("*** commands commence before first target.") |
| } |
| ev.lastRule.cmds = append(ev.lastRule.cmds, ast.cmd) |
| if ev.lastRule.cmdLineno == 0 { |
| ev.lastRule.cmdLineno = ast.lineno |
| } |
| return nil |
| } |
| |
| func (ev *Evaluator) paramVar(name string) (Var, error) { |
| idx, err := strconv.ParseInt(name, 10, 32) |
| if err != nil { |
| return nil, fmt.Errorf("param: %s: %v", name, err) |
| } |
| i := int(idx) |
| if i < 0 || i >= len(ev.paramVars) { |
| return nil, fmt.Errorf("param: %s out of %d", name, len(ev.paramVars)) |
| } |
| return &automaticVar{value: []byte(ev.paramVars[i])}, nil |
| } |
| |
| // LookupVar looks up named variable. |
| func (ev *Evaluator) LookupVar(name string) Var { |
| if ev.currentScope != nil { |
| v := ev.currentScope.Lookup(name) |
| if v.IsDefined() { |
| return v |
| } |
| } |
| v := ev.outVars.Lookup(name) |
| if v.IsDefined() { |
| return v |
| } |
| v, err := ev.paramVar(name) |
| if err == nil { |
| return v |
| } |
| return ev.vars.Lookup(name) |
| } |
| |
| func (ev *Evaluator) lookupVarInCurrentScope(name string) Var { |
| if ev.currentScope != nil { |
| v := ev.currentScope.Lookup(name) |
| return v |
| } |
| v := ev.outVars.Lookup(name) |
| if v.IsDefined() { |
| return v |
| } |
| v, err := ev.paramVar(name) |
| if err == nil { |
| return v |
| } |
| return ev.vars.Lookup(name) |
| } |
| |
| // EvaluateVar evaluates variable named name. |
| // Only for a few special uses such as getting SHELL and handling |
| // export/unexport. |
| func (ev *Evaluator) EvaluateVar(name string) (string, error) { |
| var buf evalBuffer |
| buf.resetSep() |
| err := ev.LookupVar(name).Eval(&buf, ev) |
| if err != nil { |
| return "", err |
| } |
| return buf.String(), nil |
| } |
| |
| func (ev *Evaluator) evalIncludeFile(fname string, mk makefile) error { |
| te := traceEvent.begin("include", literal(fname), traceEventMain) |
| defer func() { |
| traceEvent.end(te) |
| }() |
| var err error |
| makefileList := ev.outVars.Lookup("MAKEFILE_LIST") |
| makefileList, err = makefileList.Append(ev, mk.filename) |
| if err != nil { |
| return err |
| } |
| ev.outVars.Assign("MAKEFILE_LIST", makefileList) |
| |
| for _, stmt := range mk.stmts { |
| err = ev.eval(stmt) |
| if err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func (ev *Evaluator) evalInclude(ast *includeAST) error { |
| ev.lastRule = nil |
| ev.srcpos = ast.srcpos |
| |
| glog.Infof("%s include %q", ev.srcpos, ast.expr) |
| v, _, err := parseExpr([]byte(ast.expr), nil, parseOp{}) |
| if err != nil { |
| return ast.errorf("parse failed: %q: %v", ast.expr, err) |
| } |
| var buf evalBuffer |
| buf.resetSep() |
| err = v.Eval(&buf, ev) |
| if err != nil { |
| return ast.errorf("%v", err) |
| } |
| pats := splitSpaces(buf.String()) |
| buf.Reset() |
| |
| var files []string |
| for _, pat := range pats { |
| if strings.Contains(pat, "*") || strings.Contains(pat, "?") { |
| matched, err := filepath.Glob(pat) |
| if err != nil { |
| return ast.errorf("glob error: %s: %v", pat, err) |
| } |
| files = append(files, matched...) |
| } else { |
| files = append(files, pat) |
| } |
| } |
| |
| for _, fn := range files { |
| fn = trimLeadingCurdir(fn) |
| if IgnoreOptionalInclude != "" && ast.op == "-include" && matchPattern(fn, IgnoreOptionalInclude) { |
| continue |
| } |
| mk, hash, err := makefileCache.parse(fn) |
| if os.IsNotExist(err) { |
| if ast.op == "include" { |
| return ev.errorf("%v\nNOTE: kati does not support generating missing makefiles", err) |
| } |
| msg := ev.cache.update(fn, hash, fileNotExists) |
| if msg != "" { |
| warn(ev.srcpos, "%s", msg) |
| } |
| continue |
| } |
| msg := ev.cache.update(fn, hash, fileExists) |
| if msg != "" { |
| warn(ev.srcpos, "%s", msg) |
| } |
| err = ev.evalIncludeFile(fn, mk) |
| if err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func (ev *Evaluator) evalIf(iast *ifAST) error { |
| var isTrue bool |
| switch iast.op { |
| case "ifdef", "ifndef": |
| expr := iast.lhs |
| buf := newEbuf() |
| err := expr.Eval(buf, ev) |
| if err != nil { |
| return iast.errorf("%v\n expr:%s", err, expr) |
| } |
| v := ev.LookupVar(buf.String()) |
| buf.Reset() |
| err = v.Eval(buf, ev) |
| if err != nil { |
| return iast.errorf("%v\n expr:%s=>%s", err, expr, v) |
| } |
| value := buf.String() |
| val := buf.Len() |
| buf.release() |
| isTrue = (val > 0) == (iast.op == "ifdef") |
| if glog.V(1) { |
| glog.Infof("%s lhs=%q value=%q => %t", iast.op, iast.lhs, value, isTrue) |
| } |
| case "ifeq", "ifneq": |
| lexpr := iast.lhs |
| rexpr := iast.rhs |
| buf := newEbuf() |
| params, err := ev.args(buf, lexpr, rexpr) |
| if err != nil { |
| return iast.errorf("%v\n (%s,%s)", err, lexpr, rexpr) |
| } |
| lhs := string(params[0]) |
| rhs := string(params[1]) |
| buf.release() |
| isTrue = (lhs == rhs) == (iast.op == "ifeq") |
| if glog.V(1) { |
| glog.Infof("%s lhs=%q %q rhs=%q %q => %t", iast.op, iast.lhs, lhs, iast.rhs, rhs, isTrue) |
| } |
| default: |
| return iast.errorf("unknown if statement: %q", iast.op) |
| } |
| |
| var stmts []ast |
| if isTrue { |
| stmts = iast.trueStmts |
| } else { |
| stmts = iast.falseStmts |
| } |
| for _, stmt := range stmts { |
| err := ev.eval(stmt) |
| if err != nil { |
| return err |
| } |
| } |
| return nil |
| } |
| |
| func (ev *Evaluator) evalExport(ast *exportAST) error { |
| ev.lastRule = nil |
| ev.srcpos = ast.srcpos |
| |
| v, _, err := parseExpr(ast.expr, nil, parseOp{}) |
| if err != nil { |
| return ast.errorf("failed to parse: %q: %v", string(ast.expr), err) |
| } |
| var buf evalBuffer |
| buf.resetSep() |
| err = v.Eval(&buf, ev) |
| if err != nil { |
| return ast.errorf("%v\n expr:%s", err, v) |
| } |
| if ast.hasEqual { |
| ev.exports[string(trimSpaceBytes(buf.Bytes()))] = ast.export |
| } else { |
| for _, n := range splitSpacesBytes(buf.Bytes()) { |
| ev.exports[string(n)] = ast.export |
| } |
| } |
| return nil |
| } |
| |
| func (ev *Evaluator) evalVpath(ast *vpathAST) error { |
| ev.lastRule = nil |
| ev.srcpos = ast.srcpos |
| |
| var ebuf evalBuffer |
| ebuf.resetSep() |
| err := ast.expr.Eval(&ebuf, ev) |
| if err != nil { |
| return ast.errorf("%v\n expr:%s", err, ast.expr) |
| } |
| ws := newWordScanner(ebuf.Bytes()) |
| if !ws.Scan() { |
| ev.vpaths = nil |
| return nil |
| } |
| pat := string(ws.Bytes()) |
| if !ws.Scan() { |
| vpaths := ev.vpaths |
| ev.vpaths = nil |
| for _, v := range vpaths { |
| if v.pattern == pat { |
| continue |
| } |
| ev.vpaths = append(ev.vpaths, v) |
| } |
| return nil |
| } |
| // The search path, DIRECTORIES, is a list of directories to be |
| // searched, separated by colons (semi-colons on MS-DOS and |
| // MS-Windows) or blanks, just like the search path used in the |
| // `VPATH' variable. |
| var dirs []string |
| for { |
| for _, dir := range bytes.Split(ws.Bytes(), []byte{':'}) { |
| dirs = append(dirs, string(dir)) |
| } |
| if !ws.Scan() { |
| break |
| } |
| } |
| ev.vpaths = append(ev.vpaths, vpath{ |
| pattern: pat, |
| dirs: dirs, |
| }) |
| return nil |
| } |
| |
| func (ev *Evaluator) eval(stmt ast) error { |
| return stmt.eval(ev) |
| } |
| |
| func eval(mk makefile, vars Vars, useCache bool) (er *evalResult, err error) { |
| ev := NewEvaluator(vars) |
| if useCache { |
| ev.cache = newAccessCache() |
| } |
| |
| makefileList := vars.Lookup("MAKEFILE_LIST") |
| if !makefileList.IsDefined() { |
| makefileList = &simpleVar{value: []string{""}, origin: "file"} |
| } |
| makefileList, err = makefileList.Append(ev, mk.filename) |
| if err != nil { |
| return nil, err |
| } |
| ev.outVars.Assign("MAKEFILE_LIST", makefileList) |
| |
| for _, stmt := range mk.stmts { |
| err = ev.eval(stmt) |
| if err != nil { |
| return nil, err |
| } |
| } |
| |
| vpaths := searchPaths{ |
| vpaths: ev.vpaths, |
| } |
| v, found := ev.outVars["VPATH"] |
| if found { |
| wb := newWbuf() |
| err := v.Eval(wb, ev) |
| if err != nil { |
| return nil, err |
| } |
| // In the 'VPATH' variable, directory names are separated |
| // by colons or blanks. (on windows, semi-colons) |
| for _, word := range wb.words { |
| for _, dir := range bytes.Split(word, []byte{':'}) { |
| vpaths.dirs = append(vpaths.dirs, string(dir)) |
| } |
| } |
| } |
| glog.Infof("vpaths: %#v", vpaths) |
| |
| return &evalResult{ |
| vars: ev.outVars, |
| rules: ev.outRules, |
| ruleVars: ev.outRuleVars, |
| accessedMks: ev.cache.Slice(), |
| exports: ev.exports, |
| vpaths: vpaths, |
| }, nil |
| } |
