Merge pull request #213 from danw/bpglob_bootstrap
Run globs during earlier bootstrap phases
diff --git a/blueprint_impl.bash b/blueprint_impl.bash
index 514ba31..6f5abba 100644
--- a/blueprint_impl.bash
+++ b/blueprint_impl.bash
@@ -28,6 +28,8 @@
BUILDDIR="${BUILDDIR}/.minibootstrap" build_go minibp github.com/google/blueprint/bootstrap/minibp
+BUILDDIR="${BUILDDIR}/.minibootstrap" build_go bpglob github.com/google/blueprint/bootstrap/bpglob
+
# Build the bootstrap build.ninja
"${NINJA}" -w dupbuild=err -f "${BUILDDIR}/.minibootstrap/build.ninja"
diff --git a/bootstrap.bash b/bootstrap.bash
index 24f16c2..c02fe81 100755
--- a/bootstrap.bash
+++ b/bootstrap.bash
@@ -107,6 +107,10 @@
echo "builddir = $NINJA_BUILDDIR" >> $BUILDDIR/.minibootstrap/build.ninja
echo "include $BLUEPRINTDIR/bootstrap/build.ninja" >> $BUILDDIR/.minibootstrap/build.ninja
+if [ ! -f "$BUILDDIR/.minibootstrap/build-globs.ninja" ]; then
+ touch "$BUILDDIR/.minibootstrap/build-globs.ninja"
+fi
+
echo "BLUEPRINT_BOOTSTRAP_VERSION=2" > $BUILDDIR/.blueprint.bootstrap
echo "SRCDIR=\"${SRCDIR}\"" >> $BUILDDIR/.blueprint.bootstrap
echo "BLUEPRINTDIR=\"${BLUEPRINTDIR}\"" >> $BUILDDIR/.blueprint.bootstrap
diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go
index e9a8f01..92b8a32 100644
--- a/bootstrap/bootstrap.go
+++ b/bootstrap/bootstrap.go
@@ -17,6 +17,8 @@
import (
"fmt"
"go/build"
+ "io/ioutil"
+ "os"
"path/filepath"
"runtime"
"strings"
@@ -118,14 +120,14 @@
generateBuildNinja = pctx.StaticRule("build.ninja",
blueprint.RuleParams{
- Command: "$builder $extra -b $buildDir -n $ninjaBuildDir -d $out.d -o $out $in",
+ Command: "$builder $extra -b $buildDir -n $ninjaBuildDir -d $out.d -globFile $globFile -o $out $in",
CommandDeps: []string{"$builder"},
Description: "$builder $out",
Deps: blueprint.DepsGCC,
Depfile: "$out.d",
Restat: true,
},
- "builder", "extra", "generator")
+ "builder", "extra", "generator", "globFile")
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
phony = pctx.StaticRule("phony",
@@ -668,32 +670,33 @@
topLevelBlueprints := filepath.Join("$srcDir",
filepath.Base(s.config.topLevelBlueprintsFile))
- mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
- primaryBuilderNinjaFile := filepath.Join(bootstrapDir, "build.ninja")
-
ctx.SetNinjaBuildDir(pctx, "${ninjaBuildDir}")
- // Build the main build.ninja
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: generateBuildNinja,
- Outputs: []string{mainNinjaFile},
- Inputs: []string{topLevelBlueprints},
- Args: map[string]string{
- "builder": primaryBuilderFile,
- "extra": primaryBuilderExtraFlags,
- },
- })
+ if s.config.stage == StagePrimary {
+ mainNinjaFile := filepath.Join("$buildDir", "build.ninja")
+ primaryBuilderNinjaGlobFile := filepath.Join(BuildDir, bootstrapSubDir, "build-globs.ninja")
- // Add a way to rebuild the primary build.ninja so that globs works
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: generateBuildNinja,
- Outputs: []string{primaryBuilderNinjaFile},
- Inputs: []string{topLevelBlueprints},
- Args: map[string]string{
- "builder": minibpFile,
- "extra": extraSharedFlagString,
- },
- })
+ if _, err := os.Stat(primaryBuilderNinjaGlobFile); os.IsNotExist(err) {
+ err = ioutil.WriteFile(primaryBuilderNinjaGlobFile, nil, 0666)
+ if err != nil {
+ ctx.Errorf("Failed to create empty ninja file: %s", err)
+ }
+ }
+
+ ctx.AddSubninja(primaryBuilderNinjaGlobFile)
+
+ // Build the main build.ninja
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: generateBuildNinja,
+ Outputs: []string{mainNinjaFile},
+ Inputs: []string{topLevelBlueprints},
+ Args: map[string]string{
+ "builder": primaryBuilderFile,
+ "extra": primaryBuilderExtraFlags,
+ "globFile": primaryBuilderNinjaGlobFile,
+ },
+ })
+ }
if s.config.stage == StageMain {
if primaryBuilderName == "minibp" {
diff --git a/bootstrap/build.ninja b/bootstrap/build.ninja
index b338843..5787c72 100644
--- a/bootstrap/build.ninja
+++ b/bootstrap/build.ninja
@@ -7,8 +7,11 @@
ninja_required_version = 1.7.0
+myGlobs = ${bootstrapBuildDir}/.minibootstrap/build-globs.ninja
+subninja ${myGlobs}
+
rule build.ninja
- command = ${builder} ${extraArgs} -b ${bootstrapBuildDir} -n ${builddir} -d ${out}.d -o ${out} ${in}
+ command = ${builder} ${extraArgs} -b ${bootstrapBuildDir} -n ${builddir} -d ${out}.d -globFile ${myGlobs} -o ${out} ${in}
deps = gcc
depfile = ${out}.d
description = ${builder} ${out}
diff --git a/bootstrap/cleanup.go b/bootstrap/cleanup.go
index a4a18c7..4a8ce25 100644
--- a/bootstrap/cleanup.go
+++ b/bootstrap/cleanup.go
@@ -30,9 +30,9 @@
// removeAbandonedFilesUnder removes any files that appear in the Ninja log, and
// are prefixed with one of the `under` entries, but that are not currently
-// build targets.
+// build targets, or in `exempt`
func removeAbandonedFilesUnder(ctx *blueprint.Context, config *Config,
- srcDir string, under []string) error {
+ srcDir string, under, exempt []string) error {
if len(under) == 0 {
return nil
@@ -57,6 +57,10 @@
replacedTarget := replacer.Replace(target)
targets[filepath.Clean(replacedTarget)] = true
}
+ for _, target := range exempt {
+ replacedTarget := replacer.Replace(target)
+ targets[filepath.Clean(replacedTarget)] = true
+ }
filePaths, err := parseNinjaLog(ninjaBuildDir, under)
if err != nil {
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 04eb535..ac3fd00 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -32,6 +32,7 @@
var (
outFile string
+ globFile string
depFile string
docFile string
cpuprofile string
@@ -48,6 +49,7 @@
func init() {
flag.StringVar(&outFile, "o", "build.ninja", "the Ninja file to output")
+ flag.StringVar(&globFile, "globFile", "build-globs.ninja", "the Ninja file of globs to output")
flag.StringVar(&BuildDir, "b", ".", "the build output directory")
flag.StringVar(&NinjaBuildDir, "n", "", "the ninja builddir directory")
flag.StringVar(&depFile, "d", "", "the dependency file to output")
@@ -179,6 +181,18 @@
fatalf("error writing %s: %s", outFile, err)
}
+ if globFile != "" {
+ buffer, errs := generateGlobNinjaFile(ctx.Globs)
+ if len(errs) > 0 {
+ fatalErrors(errs)
+ }
+
+ err = ioutil.WriteFile(globFile, buffer, outFilePermissions)
+ if err != nil {
+ fatalf("error writing %s: %s", outFile, err)
+ }
+ }
+
if depFile != "" {
err := deptools.WriteDepFile(depFile, outFile, deps)
if err != nil {
@@ -187,8 +201,8 @@
}
if c, ok := config.(ConfigRemoveAbandonedFilesUnder); ok {
- under := c.RemoveAbandonedFilesUnder()
- err := removeAbandonedFilesUnder(ctx, bootstrapConfig, SrcDir, under)
+ under, except := c.RemoveAbandonedFilesUnder()
+ err := removeAbandonedFilesUnder(ctx, bootstrapConfig, SrcDir, under, except)
if err != nil {
fatalf("error removing abandoned files: %s", err)
}
diff --git a/bootstrap/config.go b/bootstrap/config.go
index 5785ea7..790ac4b 100644
--- a/bootstrap/config.go
+++ b/bootstrap/config.go
@@ -69,10 +69,11 @@
}
type ConfigRemoveAbandonedFilesUnder interface {
- // RemoveAbandonedFilesUnder should return a slice of path prefixes that
- // will be cleaned of files that are no longer active targets, but are
- // listed in the .ninja_log.
- RemoveAbandonedFilesUnder() []string
+ // RemoveAbandonedFilesUnder should return two slices:
+ // - a slice of path prefixes that will be cleaned of files that are no
+ // longer active targets, but are listed in the .ninja_log.
+ // - a slice of paths that are exempt from cleaning
+ RemoveAbandonedFilesUnder() (under, except []string)
}
type ConfigBlueprintToolLocation interface {
diff --git a/bootstrap/doc.go b/bootstrap/doc.go
index 3c7108e..69a1784 100644
--- a/bootstrap/doc.go
+++ b/bootstrap/doc.go
@@ -120,23 +120,27 @@
// - Runs .bootstrap/build.ninja to build and run the primary builder
// - Runs build.ninja to build your code
//
-// Microfactory takes care of building an up to date version of `minibp` under
-// the .minibootstrap/ directory.
+// Microfactory takes care of building an up to date version of `minibp` and
+// `bpglob` under the .minibootstrap/ directory.
//
// During <builddir>/.minibootstrap/build.ninja, the following actions are
// taken, if necessary:
//
// - Run minibp to generate .bootstrap/build.ninja (Primary stage)
+// - Includes .minibootstrap/build-globs.ninja, which defines rules to
+// run bpglob during incremental builds. These outputs are listed in
+// the dependency file output by minibp.
//
// During the <builddir>/.bootstrap/build.ninja, the following actions are
// taken, if necessary:
//
-// - Rebuild .bootstrap/build.ninja, usually due to globs changing --
-// other dependencies will trigger it to be built during minibootstrap
// - Build the primary builder, anything marked `default: true`, and
// any dependencies.
// - Run the primary builder to generate build.ninja
// - Run the primary builder to extract documentation
+// - Includes .bootstrap/build-globs.ninja, which defines rules to run
+// bpglob during incremental builds. These outputs are listed in the
+// dependency file output by the primary builder.
//
// Then the main stage is at <builddir>/build.ninja, and will contain all the
// rules generated by the primary builder. In addition, the bootstrap code
diff --git a/bootstrap/glob.go b/bootstrap/glob.go
index 160ef58..9841611 100644
--- a/bootstrap/glob.go
+++ b/bootstrap/glob.go
@@ -15,6 +15,7 @@
package bootstrap
import (
+ "bytes"
"fmt"
"path/filepath"
"strings"
@@ -40,7 +41,7 @@
// in a build failure with a "missing and no known rule to make it" error.
var (
- globCmd = filepath.Join("$BinDir", "bpglob")
+ globCmd = filepath.Join(miniBootstrapDir, "bpglob")
// globRule rule traverses directories to produce a list of files that match $glob
// and writes it to $out if it has changed, and writes the directories to $out.d
@@ -111,6 +112,7 @@
// primary builder if the results change.
type globSingleton struct {
globLister func() []blueprint.GlobPath
+ writeRule bool
}
func globSingletonFactory(ctx *blueprint.Context) func() blueprint.Singleton {
@@ -124,15 +126,52 @@
func (s *globSingleton) GenerateBuildActions(ctx blueprint.SingletonContext) {
for _, g := range s.globLister() {
fileListFile := filepath.Join(BuildDir, ".glob", g.Name)
- depFile := fileListFile + ".d"
- fileList := strings.Join(g.Files, "\n") + "\n"
- pathtools.WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
- deptools.WriteDepFile(depFile, fileListFile, g.Deps)
+ if s.writeRule {
+ depFile := fileListFile + ".d"
- GlobFile(ctx, g.Pattern, g.Excludes, fileListFile, depFile)
+ fileList := strings.Join(g.Files, "\n") + "\n"
+ pathtools.WriteFileIfChanged(fileListFile, []byte(fileList), 0666)
+ deptools.WriteDepFile(depFile, fileListFile, g.Deps)
- // Make build.ninja depend on the fileListFile
- ctx.AddNinjaFileDeps(fileListFile)
+ GlobFile(ctx, g.Pattern, g.Excludes, fileListFile, depFile)
+ } else {
+ // Make build.ninja depend on the fileListFile
+ ctx.AddNinjaFileDeps(fileListFile)
+ }
}
}
+
+func generateGlobNinjaFile(globLister func() []blueprint.GlobPath) ([]byte, []error) {
+ ctx := blueprint.NewContext()
+ ctx.RegisterSingletonType("glob", func() blueprint.Singleton {
+ return &globSingleton{
+ globLister: globLister,
+ writeRule: true,
+ }
+ })
+
+ extraDeps, errs := ctx.ResolveDependencies(nil)
+ if len(extraDeps) > 0 {
+ return nil, []error{fmt.Errorf("shouldn't have extra deps")}
+ }
+ if len(errs) > 0 {
+ return nil, errs
+ }
+
+ extraDeps, errs = ctx.PrepareBuildActions(nil)
+ if len(extraDeps) > 0 {
+ return nil, []error{fmt.Errorf("shouldn't have extra deps")}
+ }
+ if len(errs) > 0 {
+ return nil, errs
+ }
+
+ buf := bytes.NewBuffer(nil)
+ err := ctx.WriteBuildFile(buf)
+ if err != nil {
+ return nil, []error{err}
+ }
+
+ return buf.Bytes(), nil
+}
diff --git a/bootstrap/minibp/main.go b/bootstrap/minibp/main.go
index 72ed9f6..1714739 100644
--- a/bootstrap/minibp/main.go
+++ b/bootstrap/minibp/main.go
@@ -37,8 +37,12 @@
return c.generatingPrimaryBuilder
}
-func (c Config) RemoveAbandonedFilesUnder() []string {
- return []string{filepath.Join(bootstrap.BuildDir, ".bootstrap")}
+func (c Config) RemoveAbandonedFilesUnder() (under, exempt []string) {
+ if c.generatingPrimaryBuilder {
+ under = []string{filepath.Join(bootstrap.BuildDir, ".bootstrap")}
+ exempt = []string{filepath.Join(bootstrap.BuildDir, ".bootstrap", "build.ninja")}
+ }
+ return
}
func main() {
diff --git a/context.go b/context.go
index 919377c..69f26f6 100644
--- a/context.go
+++ b/context.go
@@ -102,6 +102,8 @@
requiredNinjaMinor int // For the ninja_required_version variable
requiredNinjaMicro int // For the ninja_required_version variable
+ subninjas []string
+
// set lazily by sortedModuleGroups
cachedSortedModuleGroups []*moduleGroup
@@ -2939,6 +2941,11 @@
return err
}
+ err = c.writeSubninjas(nw)
+ if err != nil {
+ return err
+ }
+
// TODO: Group the globals by package.
err = c.writeGlobalVariables(nw)
@@ -3048,6 +3055,13 @@
return nw.BlankLine()
}
+func (c *Context) writeSubninjas(nw *ninjaWriter) error {
+ for _, subninja := range c.subninjas {
+ nw.Subninja(subninja)
+ }
+ return nw.BlankLine()
+}
+
func (c *Context) writeBuildDir(nw *ninjaWriter) error {
if c.ninjaBuildDir != nil {
err := nw.Assign("builddir", c.ninjaBuildDir.Value(c.pkgNames))
diff --git a/ninja_writer.go b/ninja_writer.go
index a61667d..5902986 100644
--- a/ninja_writer.go
+++ b/ninja_writer.go
@@ -193,6 +193,12 @@
return wrapper.Flush()
}
+func (n *ninjaWriter) Subninja(file string) error {
+ n.justDidBlankLine = false
+ _, err := fmt.Fprintf(n.writer, "subninja %s\n", file)
+ return err
+}
+
func (n *ninjaWriter) BlankLine() (err error) {
// We don't output multiple blank lines in a row.
if !n.justDidBlankLine {
diff --git a/ninja_writer_test.go b/ninja_writer_test.go
index 44e4ff8..cc880e5 100644
--- a/ninja_writer_test.go
+++ b/ninja_writer_test.go
@@ -74,6 +74,12 @@
},
{
input: func(w *ninjaWriter) {
+ ck(w.Subninja("build.ninja"))
+ },
+ output: "subninja build.ninja\n",
+ },
+ {
+ input: func(w *ninjaWriter) {
ck(w.BlankLine())
},
output: "\n",
diff --git a/singleton_ctx.go b/singleton_ctx.go
index bbfce00..333811d 100644
--- a/singleton_ctx.go
+++ b/singleton_ctx.go
@@ -47,6 +47,10 @@
// set at most one time for a single build, later calls are ignored.
SetNinjaBuildDir(pctx PackageContext, value string)
+ // AddSubninja adds a ninja file to include with subninja. This should likely
+ // only ever be used inside bootstrap to handle glob rules.
+ AddSubninja(file string)
+
// Eval takes a string with embedded ninja variables, and returns a string
// with all of the variables recursively expanded. Any variables references
// are expanded in the scope of the PackageContext.
@@ -203,6 +207,10 @@
s.context.setNinjaBuildDir(ninjaValue)
}
+func (s *singletonContext) AddSubninja(file string) {
+ s.context.subninjas = append(s.context.subninjas, file)
+}
+
func (s *singletonContext) VisitAllModules(visit func(Module)) {
s.context.VisitAllModules(visit)
}
