Enhance bootstrap stage selection
This simplifies the bootstrap process while making it more flexible by
moving the stage selection into a go binary(choosestage). It will now be
possible to have more than two build stages.
Now each stage has a ninja template(main.ninja.in) and a timestamp
file(main.ninja.in.timestamp). The timestamp file may be updated by any
build stage that wishes to regenerate the ninja template. If the
choosestage binaries sees that the timestamp is newer than the template,
it will choose the prior stage.
The main stage no longer writes to the source tree to update the
build.ninja.in file. This was a problem for read-only source trees.
Instead, the choosestage binary first checks to see if that file is
newer than the last bootstrap.ninja.in, copies it in place, and starts
the boostrap stage.
The bootstrap stage regenerates it's own ninja template, but that
required a loop through the main stage to actually run it. The
choosestage binary now detects if the template has changed for the
current stage, and will restart the stage.
One change is that if dependencies do get messed up, instead of silently
failing, there's a higher chance that the bootstrap step will just
continue looping, doing nothing. This can happen if the main stage
has a dependency that triggers the bootstrap stage, but the bootstrap
stage doesn't see anything required to rebuild the main ninja file. A
side effect of this requirement is that changes to test code will now
rebuild the main ninja file.
Change-Id: I9965cfba79dc0dbbd3af05f5944f7653054455a2
diff --git a/.travis.yml b/.travis.yml
index 9268df0..95529e2 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -5,10 +5,9 @@
script:
- go test ./...
- - cp build.ninja.in build.ninja.in.orig
- mkdir stage
- cd stage
- ../bootstrap.bash
- ninja
- - diff -us ../build.ninja.in ../build.ninja.in.orig
+ - diff -us ../build.ninja.in .bootstrap/bootstrap.ninja.in
- ../tests/test.sh
diff --git a/Blueprints b/Blueprints
index 46b5c84..3845ad0 100644
--- a/Blueprints
+++ b/Blueprints
@@ -123,3 +123,8 @@
name = "gotestmain",
srcs = ["gotestmain/gotestmain.go"],
)
+
+bootstrap_go_binary(
+ name = "choosestage",
+ srcs = ["choosestage/choosestage.go"],
+)
diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go
index 66a4036..3023b08 100644
--- a/bootstrap/bootstrap.go
+++ b/bootstrap/bootstrap.go
@@ -28,9 +28,10 @@
var (
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
- gcCmd = pctx.StaticVariable("gcCmd", "$goToolDir/${goChar}g")
- linkCmd = pctx.StaticVariable("linkCmd", "$goToolDir/${goChar}l")
- goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
+ gcCmd = pctx.StaticVariable("gcCmd", "$goToolDir/${goChar}g")
+ linkCmd = pctx.StaticVariable("linkCmd", "$goToolDir/${goChar}l")
+ goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
+ chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage"))
gc = pctx.StaticRule("gc",
blueprint.RuleParams{
@@ -70,11 +71,25 @@
bootstrap = pctx.StaticRule("bootstrap",
blueprint.RuleParams{
- Command: "echo \"Choosing $$(basename $in) for next stage\" && $bootstrapCmd -i $in",
+ Command: "$bootstrapCmd -i $in",
Description: "bootstrap $in",
Generator: true,
})
+ chooseStage = pctx.StaticRule("chooseStage",
+ blueprint.RuleParams{
+ Command: "$chooseStageCmd --current $current --bootstrap $bootstrapManifest -o $out $in",
+ Description: "choosing next stage",
+ },
+ "current", "generator")
+
+ touch = pctx.StaticRule("touch",
+ blueprint.RuleParams{
+ Command: "touch $out",
+ Description: "touch $out",
+ },
+ "depfile", "generator")
+
// Work around a Ninja issue. See https://github.com/martine/ninja/pull/634
phony = pctx.StaticRule("phony",
blueprint.RuleParams{
@@ -186,7 +201,8 @@
// the circular dependence that occurs when the builder requires a new Ninja
// file to be built, but building a new ninja file requires the builder to
// be built.
- if g.config.generatingBootstrapper {
+ switch g.config.stage {
+ case StageBootstrap:
var deps []string
if g.config.runGoTests {
@@ -197,7 +213,7 @@
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
g.properties.Srcs, deps)
- } else {
+ case StageMain:
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
}
@@ -251,7 +267,8 @@
// the circular dependence that occurs when the builder requires a new Ninja
// file to be built, but building a new ninja file requires the builder to
// be built.
- if g.config.generatingBootstrapper {
+ switch g.config.stage {
+ case StageBootstrap:
var deps []string
if g.config.runGoTests {
@@ -287,7 +304,7 @@
Outputs: []string{binaryFile},
Inputs: []string{aoutFile},
})
- } else {
+ case StageMain:
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
}
@@ -506,18 +523,34 @@
}
// Get the filename of the top-level Blueprints file to pass to minibp.
- // This comes stored in a global variable that's set by Main.
topLevelBlueprints := filepath.Join("$srcDir",
filepath.Base(s.config.topLevelBlueprintsFile))
+ rebootstrapDeps = append(rebootstrapDeps, topLevelBlueprints)
+
mainNinjaFile := filepath.Join(bootstrapDir, "main.ninja.in")
- mainNinjaDepFile := mainNinjaFile + ".d"
+ mainNinjaTimestampFile := mainNinjaFile + ".timestamp"
+ mainNinjaTimestampDepFile := mainNinjaTimestampFile + ".d"
bootstrapNinjaFile := filepath.Join(bootstrapDir, "bootstrap.ninja.in")
docsFile := filepath.Join(docsDir, primaryBuilderName+".html")
rebootstrapDeps = append(rebootstrapDeps, docsFile)
- if s.config.generatingBootstrapper {
+ // If the tests change, be sure to re-run them. These need to be
+ // dependencies for the ninja file so that it's updated after these
+ // run. Otherwise we'd never leave the bootstrap stage, since the
+ // timestamp file would be newer than the ninja file.
+ ctx.VisitAllModulesIf(isGoTestProducer,
+ func(module blueprint.Module) {
+ testModule := module.(goTestProducer)
+ target := testModule.GoTestTarget()
+ if target != "" {
+ rebootstrapDeps = append(rebootstrapDeps, target)
+ }
+ })
+
+ switch s.config.stage {
+ case StageBootstrap:
// We're generating a bootstrapper Ninja file, so we need to set things
// up to rebuild the build.ninja file using the primary builder.
@@ -525,6 +558,64 @@
// otherwise the cleanup process will remove files from the other build.
ctx.SetBuildDir(pctx, bootstrapDir)
+ // Rebuild the bootstrap Ninja file using the minibp that we just built.
+ // If this produces a difference, choosestage will retrigger this stage.
+ minibp := ctx.Rule(pctx, "minibp",
+ blueprint.RuleParams{
+ Command: fmt.Sprintf("%s $runTests -m $bootstrapManifest "+
+ "-d $out.d -o $out $in", minibpFile),
+ Description: "minibp $out",
+ Generator: true,
+ Depfile: "$out.d",
+ },
+ "runTests")
+
+ args := map[string]string{}
+
+ if s.config.runGoTests {
+ args["runTests"] = "-t"
+ }
+
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: minibp,
+ Outputs: []string{bootstrapNinjaFile},
+ Inputs: []string{topLevelBlueprints},
+ // $bootstrapManifest is here so that when it is updated, we
+ // force a rebuild of bootstrap.ninja.in. chooseStage should
+ // have already copied the new version over, but kept the old
+ // timestamps to force this regeneration.
+ Implicits: []string{"$bootstrapManifest", minibpFile},
+ Args: args,
+ })
+
+ // We generate the depfile here that includes the dependencies for all
+ // the Blueprints files that contribute to generating the big build
+ // manifest (build.ninja file). This depfile will be used by the non-
+ // bootstrap build manifest to determine whether it should touch the
+ // timestamp file to trigger a re-bootstrap.
+ bigbp := ctx.Rule(pctx, "bigbp",
+ blueprint.RuleParams{
+ Command: fmt.Sprintf("%s %s -m $bootstrapManifest "+
+ "--timestamp $timestamp --timestampdep $timestampdep "+
+ "-d $outfile.d -o $outfile $in", primaryBuilderFile,
+ primaryBuilderExtraFlags),
+ Description: fmt.Sprintf("%s $outfile", primaryBuilderName),
+ Depfile: "$outfile.d",
+ },
+ "timestamp", "timestampdep", "outfile")
+
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: bigbp,
+ Outputs: []string{mainNinjaFile, mainNinjaTimestampFile},
+ Inputs: []string{topLevelBlueprints},
+ Implicits: rebootstrapDeps,
+ Args: map[string]string{
+ "timestamp": mainNinjaTimestampFile,
+ "timestampdep": mainNinjaTimestampDepFile,
+ "outfile": mainNinjaFile,
+ },
+ })
+
// Generate build system docs for the primary builder. Generating docs reads the source
// files used to build the primary builder, but that dependency will be picked up through
// the dependency on the primary builder itself. There are no dependencies on the
@@ -543,36 +634,10 @@
Implicits: []string{primaryBuilderFile},
})
- // We generate the depfile here that includes the dependencies for all
- // the Blueprints files that contribute to generating the big build
- // manifest (build.ninja file). This depfile will be used by the non-
- // bootstrap build manifest to determine whether it should trigger a re-
- // bootstrap. Because the re-bootstrap rule's output is "build.ninja"
- // we need to force the depfile to have that as its "make target"
- // (recall that depfiles use a subset of the Makefile syntax).
- bigbp := ctx.Rule(pctx, "bigbp",
- blueprint.RuleParams{
- Command: fmt.Sprintf("%s %s -d %s -m $bootstrapManifest "+
- "-o $out $in", primaryBuilderFile,
- primaryBuilderExtraFlags, mainNinjaDepFile),
- Description: fmt.Sprintf("%s $out", primaryBuilderName),
- Depfile: mainNinjaDepFile,
- })
-
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: bigbp,
- Outputs: []string{mainNinjaFile},
- Inputs: []string{topLevelBlueprints},
- Implicits: rebootstrapDeps,
- })
-
// When the current build.ninja file is a bootstrapper, we always want
// to have it replace itself with a non-bootstrapper build.ninja. To
// accomplish that we depend on a file that should never exist and
// "build" it using Ninja's built-in phony rule.
- //
- // We also need to add an implicit dependency on bootstrapNinjaFile so
- // that it gets generated as part of the bootstrap process.
notAFile := filepath.Join(bootstrapDir, "notAFile")
ctx.Build(pctx, blueprint.BuildParams{
Rule: blueprint.Phony,
@@ -580,103 +645,54 @@
})
ctx.Build(pctx, blueprint.BuildParams{
- Rule: bootstrap,
- Outputs: []string{"build.ninja"},
- Inputs: []string{mainNinjaFile},
- Implicits: []string{"$bootstrapCmd", notAFile, bootstrapNinjaFile},
- })
-
- // Rebuild the bootstrap Ninja file using the minibp that we just built.
- // The checkFile tells minibp to compare the new bootstrap file to the
- // current one. If the files are the same then minibp sets the new
- // file's mtime to match that of the current one. If they're different
- // then the new file will have a newer timestamp than the current one
- // and it will trigger a reboostrap by the non-boostrap build manifest.
- minibp := ctx.Rule(pctx, "minibp",
- blueprint.RuleParams{
- Command: fmt.Sprintf("%s $runTests -c $checkFile -m $bootstrapManifest "+
- "-d $out.d -o $out $in", minibpFile),
- Description: "minibp $out",
- Generator: true,
- Depfile: "$out.d",
+ Rule: chooseStage,
+ Outputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")},
+ Inputs: []string{bootstrapNinjaFile, mainNinjaFile},
+ Implicits: []string{"$chooseStageCmd", "$bootstrapManifest", notAFile},
+ Args: map[string]string{
+ "current": bootstrapNinjaFile,
},
- "checkFile", "runTests")
-
- args := map[string]string{
- "checkFile": "$bootstrapManifest",
- }
-
- if s.config.runGoTests {
- args["runTests"] = "-t"
- }
-
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: minibp,
- Outputs: []string{bootstrapNinjaFile},
- Inputs: []string{topLevelBlueprints},
- Implicits: []string{minibpFile},
- Args: args,
})
- } else {
- ctx.VisitAllModulesIf(isGoTestProducer,
- func(module blueprint.Module) {
- testModule := module.(goTestProducer)
- target := testModule.GoTestTarget()
- if target != "" {
- rebootstrapDeps = append(rebootstrapDeps, target)
- }
- })
+ case StageMain:
// We're generating a non-bootstrapper Ninja file, so we need to set it
- // up to depend on the bootstrapper Ninja file. The build.ninja target
- // also has an implicit dependency on the primary builder and all other
- // bootstrap go binaries, which will have phony dependencies on all of
- // their sources. This will cause any changes to a bootstrap binary's
- // sources to trigger a re-bootstrap operation, which will rebuild the
- // binary.
+ // up to re-bootstrap if necessary. We do this by making build.ninja.in
+ // depend on the various Ninja files, the source build.ninja.in, and
+ // on the timestamp files.
//
- // On top of that we need to use the depfile generated by the bigbp
- // rule. We do this by depending on that file and then setting up a
- // phony rule to generate it that uses the depfile.
- buildNinjaDeps := []string{"$bootstrapCmd", mainNinjaFile}
- buildNinjaDeps = append(buildNinjaDeps, rebootstrapDeps...)
-
+ // The timestamp files themselves are set up with the same dependencies
+ // as their Ninja files, including their own depfile. If any of the
+ // dependencies need to be updated, we'll touch the timestamp file,
+ // which will tell choosestage to switch to the stage that rebuilds
+ // that Ninja file.
ctx.Build(pctx, blueprint.BuildParams{
- Rule: bootstrap,
- Outputs: []string{"build.ninja"},
- Inputs: []string{"$bootstrapManifest"},
- Implicits: buildNinjaDeps,
- })
-
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: phony,
- Outputs: []string{mainNinjaFile},
- Inputs: []string{topLevelBlueprints},
+ Rule: touch,
+ Outputs: []string{mainNinjaTimestampFile},
+ Implicits: rebootstrapDeps,
Args: map[string]string{
- "depfile": mainNinjaDepFile,
- },
- })
-
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: phony,
- Outputs: []string{docsFile},
- Implicits: []string{primaryBuilderFile},
- })
-
- // If the bootstrap Ninja invocation caused a new bootstrapNinjaFile to be
- // generated then that means we need to rebootstrap using it instead of
- // the current bootstrap manifest. We enable the Ninja "generator"
- // behavior so that Ninja doesn't invoke this build just because it's
- // missing a command line log entry for the bootstrap manifest.
- ctx.Build(pctx, blueprint.BuildParams{
- Rule: cp,
- Outputs: []string{"$bootstrapManifest"},
- Inputs: []string{bootstrapNinjaFile},
- Args: map[string]string{
+ "depfile": mainNinjaTimestampDepFile,
"generator": "true",
},
})
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: chooseStage,
+ Outputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")},
+ Inputs: []string{bootstrapNinjaFile, mainNinjaFile},
+ Implicits: []string{"$chooseStageCmd", "$bootstrapManifest", mainNinjaTimestampFile},
+ Args: map[string]string{
+ "current": mainNinjaFile,
+ "generator": "true",
+ },
+ })
+
+ // Create this phony rule so that upgrades don't delete these during
+ // cleanup
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: blueprint.Phony,
+ Outputs: []string{mainNinjaFile, docsFile},
+ })
+
if primaryBuilderName == "minibp" {
// This is a standalone Blueprint build, so we copy the minibp
// binary to the "bin" directory to make it easier to find.
@@ -688,6 +704,13 @@
})
}
}
+
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: bootstrap,
+ Outputs: []string{"build.ninja"},
+ Inputs: []string{filepath.Join(bootstrapDir, "build.ninja.in")},
+ Implicits: []string{"$bootstrapCmd"},
+ })
}
// packageRoot returns the module-specific package root directory path. This
diff --git a/bootstrap/cleanup.go b/bootstrap/cleanup.go
index f3cb634..ba9fd48 100644
--- a/bootstrap/cleanup.go
+++ b/bootstrap/cleanup.go
@@ -33,7 +33,7 @@
srcDir, manifestFile string) error {
buildDir := "."
- if config.generatingBootstrapper {
+ if config.stage == StageBootstrap {
buildDir = bootstrapDir
}
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 932cfd7..4fb9c51 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -29,19 +29,21 @@
)
var (
- outFile string
- depFile string
- checkFile string
- manifestFile string
- docFile string
- cpuprofile string
- runGoTests bool
+ outFile string
+ depFile string
+ timestampFile string
+ timestampDepFile string
+ manifestFile string
+ docFile string
+ cpuprofile string
+ runGoTests bool
)
func init() {
flag.StringVar(&outFile, "o", "build.ninja.in", "the Ninja file to output")
flag.StringVar(&depFile, "d", "", "the dependency file to output")
- flag.StringVar(&checkFile, "c", "", "the existing file to check against")
+ flag.StringVar(×tampFile, "timestamp", "", "file to write before the output file")
+ flag.StringVar(×tampDepFile, "timestampdep", "", "the dependency file for the timestamp file")
flag.StringVar(&manifestFile, "m", "", "the bootstrap manifest file")
flag.StringVar(&docFile, "docs", "", "build documentation file to output")
flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
@@ -69,13 +71,15 @@
fatalf("no Blueprints file specified")
}
- generatingBootstrapper := false
+ stage := StageMain
if c, ok := config.(ConfigInterface); ok {
- generatingBootstrapper = c.GeneratingBootstrapper()
+ if c.GeneratingBootstrapper() {
+ stage = StageBootstrap
+ }
}
bootstrapConfig := &Config{
- generatingBootstrapper: generatingBootstrapper,
+ stage: stage,
topLevelBlueprintsFile: flag.Arg(0),
runGoTests: runGoTests,
}
@@ -118,48 +122,34 @@
}
const outFilePermissions = 0666
+ if timestampFile != "" {
+ err := ioutil.WriteFile(timestampFile, []byte{}, outFilePermissions)
+ if err != nil {
+ fatalf("error writing %s: %s", timestampFile, err)
+ }
+
+ if timestampDepFile != "" {
+ err := deptools.WriteDepFile(timestampDepFile, timestampFile, deps)
+ if err != nil {
+ fatalf("error writing depfile: %s", err)
+ }
+ }
+ }
+
err = ioutil.WriteFile(outFile, buf.Bytes(), outFilePermissions)
if err != nil {
fatalf("error writing %s: %s", outFile, err)
}
- if checkFile != "" {
- checkData, err := ioutil.ReadFile(checkFile)
- if err != nil {
- fatalf("error reading %s: %s", checkFile, err)
- }
-
- matches := buf.Len() == len(checkData)
- if matches {
- for i, value := range buf.Bytes() {
- if value != checkData[i] {
- matches = false
- break
- }
- }
- }
-
- if matches {
- // The new file content matches the check-file content, so we set
- // the new file's mtime and atime to match that of the check-file.
- checkFileInfo, err := os.Stat(checkFile)
- if err != nil {
- fatalf("error stat'ing %s: %s", checkFile, err)
- }
-
- time := checkFileInfo.ModTime()
- err = os.Chtimes(outFile, time, time)
- if err != nil {
- fatalf("error setting timestamps for %s: %s", outFile, err)
- }
- }
- }
-
if depFile != "" {
err := deptools.WriteDepFile(depFile, outFile, deps)
if err != nil {
fatalf("error writing depfile: %s", err)
}
+ err = deptools.WriteDepFile(depFile+".timestamp", outFile+".timestamp", deps)
+ if err != nil {
+ fatalf("error writing depfile: %s", err)
+ }
}
srcDir := filepath.Dir(bootstrapConfig.topLevelBlueprintsFile)
diff --git a/bootstrap/config.go b/bootstrap/config.go
index ce96318..ab3b160 100644
--- a/bootstrap/config.go
+++ b/bootstrap/config.go
@@ -38,11 +38,15 @@
GeneratingBootstrapper() bool
}
+type Stage int
+
+const (
+ StageBootstrap Stage = iota
+ StageMain
+)
+
type Config struct {
- // generatingBootstrapper should be true if this build invocation is
- // creating a build.ninja.in file to be used in a build bootstrapping
- // sequence.
- generatingBootstrapper bool
+ stage Stage
topLevelBlueprintsFile string
diff --git a/build.ninja.in b/build.ninja.in
index 77661e9..183e954 100644
--- a/build.ninja.in
+++ b/build.ninja.in
@@ -13,6 +13,8 @@
g.bootstrap.bootstrapManifest = @@BootstrapManifest@@
+g.bootstrap.chooseStageCmd = .bootstrap/bin/choosestage
+
g.bootstrap.goRoot = @@GoRoot@@
g.bootstrap.goOS = @@GoOS@@
@@ -32,10 +34,14 @@
builddir = .bootstrap
rule g.bootstrap.bootstrap
- command = echo "Choosing $$(basename ${in}) for next stage" && ${g.bootstrap.bootstrapCmd} -i ${in}
+ command = ${g.bootstrap.bootstrapCmd} -i ${in}
description = bootstrap ${in}
generator = true
+rule g.bootstrap.chooseStage
+ command = ${g.bootstrap.chooseStageCmd} --current ${current} --bootstrap ${g.bootstrap.bootstrapManifest} -o ${out} ${in}
+ description = choosing next stage
+
rule g.bootstrap.cp
command = cp ${in} ${out}
description = cp ${out}
@@ -221,6 +227,26 @@
default .bootstrap/bin/bpmodify
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# Module: choosestage
+# Variant:
+# Type: bootstrap_go_binary
+# Factory: github.com/google/blueprint/bootstrap.func·002
+# Defined: Blueprints:127:1
+
+build .bootstrap/choosestage/obj/choosestage.a: g.bootstrap.gc $
+ ${g.bootstrap.srcDir}/choosestage/choosestage.go | $
+ ${g.bootstrap.gcCmd}
+ pkgPath = choosestage
+default .bootstrap/choosestage/obj/choosestage.a
+
+build .bootstrap/choosestage/obj/a.out: g.bootstrap.link $
+ .bootstrap/choosestage/obj/choosestage.a | ${g.bootstrap.linkCmd}
+default .bootstrap/choosestage/obj/a.out
+build .bootstrap/bin/choosestage: g.bootstrap.cp $
+ .bootstrap/choosestage/obj/a.out
+default .bootstrap/bin/choosestage
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Module: gotestmain
# Variant:
# Type: bootstrap_go_binary
@@ -271,37 +297,49 @@
# Singleton: bootstrap
# Factory: github.com/google/blueprint/bootstrap.func·007
-rule s.bootstrap.bigbpDocs
- command = .bootstrap/bin/minibp -p --docs ${out} ${g.bootstrap.srcDir}/Blueprints
- description = minibp docs ${out}
-
-rule s.bootstrap.bigbp
- command = .bootstrap/bin/minibp -p -d .bootstrap/main.ninja.in.d -m ${g.bootstrap.bootstrapManifest} -o ${out} ${in}
- depfile = .bootstrap/main.ninja.in.d
- description = minibp ${out}
-
rule s.bootstrap.minibp
- command = .bootstrap/bin/minibp ${runTests} -c ${checkFile} -m ${g.bootstrap.bootstrapManifest} -d ${out}.d -o ${out} ${in}
+ command = .bootstrap/bin/minibp ${runTests} -m ${g.bootstrap.bootstrapManifest} -d ${out}.d -o ${out} ${in}
depfile = ${out}.d
description = minibp ${out}
generator = true
+rule s.bootstrap.bigbp
+ command = .bootstrap/bin/minibp -p -m ${g.bootstrap.bootstrapManifest} --timestamp ${timestamp} --timestampdep ${timestampdep} -d ${outfile}.d -o ${outfile} ${in}
+ depfile = ${outfile}.d
+ description = minibp ${outfile}
+
+rule s.bootstrap.bigbpDocs
+ command = .bootstrap/bin/minibp -p --docs ${out} ${g.bootstrap.srcDir}/Blueprints
+ description = minibp docs ${out}
+
+build .bootstrap/bootstrap.ninja.in: s.bootstrap.minibp $
+ ${g.bootstrap.srcDir}/Blueprints | ${g.bootstrap.bootstrapManifest} $
+ .bootstrap/bin/minibp
+default .bootstrap/bootstrap.ninja.in
+build .bootstrap/main.ninja.in .bootstrap/main.ninja.in.timestamp: $
+ s.bootstrap.bigbp ${g.bootstrap.srcDir}/Blueprints | $
+ .bootstrap/bin/bpfmt .bootstrap/bin/bpmodify $
+ .bootstrap/bin/choosestage .bootstrap/bin/gotestmain $
+ .bootstrap/bin/minibp ${g.bootstrap.srcDir}/Blueprints $
+ .bootstrap/docs/minibp.html
+ outfile = .bootstrap/main.ninja.in
+ timestamp = .bootstrap/main.ninja.in.timestamp
+ timestampdep = .bootstrap/main.ninja.in.timestamp.d
+default .bootstrap/main.ninja.in .bootstrap/main.ninja.in.timestamp
+
build .bootstrap/docs/minibp.html: s.bootstrap.bigbpDocs | $
.bootstrap/bin/minibp
default .bootstrap/docs/minibp.html
-build .bootstrap/main.ninja.in: s.bootstrap.bigbp $
- ${g.bootstrap.srcDir}/Blueprints | .bootstrap/bin/bpfmt $
- .bootstrap/bin/bpmodify .bootstrap/bin/gotestmain $
- .bootstrap/bin/minibp .bootstrap/docs/minibp.html
-default .bootstrap/main.ninja.in
build .bootstrap/notAFile: phony
default .bootstrap/notAFile
-build build.ninja: g.bootstrap.bootstrap .bootstrap/main.ninja.in | $
- ${g.bootstrap.bootstrapCmd} .bootstrap/notAFile $
- .bootstrap/bootstrap.ninja.in
+build .bootstrap/build.ninja.in: g.bootstrap.chooseStage $
+ .bootstrap/bootstrap.ninja.in .bootstrap/main.ninja.in | $
+ ${g.bootstrap.chooseStageCmd} ${g.bootstrap.bootstrapManifest} $
+ .bootstrap/notAFile
+ current = .bootstrap/bootstrap.ninja.in
+default .bootstrap/build.ninja.in
+
+build build.ninja: g.bootstrap.bootstrap .bootstrap/build.ninja.in | $
+ ${g.bootstrap.bootstrapCmd}
default build.ninja
-build .bootstrap/bootstrap.ninja.in: s.bootstrap.minibp $
- ${g.bootstrap.srcDir}/Blueprints | .bootstrap/bin/minibp
- checkFile = ${g.bootstrap.bootstrapManifest}
-default .bootstrap/bootstrap.ninja.in
diff --git a/choosestage/choosestage.go b/choosestage/choosestage.go
new file mode 100644
index 0000000..e1445e0
--- /dev/null
+++ b/choosestage/choosestage.go
@@ -0,0 +1,194 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Choose which ninja file (stage) to run next
+//
+// In the common case, this program takes a list of ninja files, compares their
+// mtimes against their $file.timestamp mtimes, and picks the last up to date
+// ninja file to output. That stage is expected to rebuild the next file in the
+// list and call this program again. If none of the ninja files are considered
+// dirty, the last stage is output.
+//
+// One exception is if the current stage's ninja file was rewritten, it will be
+// run again.
+//
+// Another exception is if the source bootstrap file has been updated more
+// recently than the first stage, the source file will be copied to the first
+// stage, and output. This would be expected with a new source drop via git.
+// The timestamp of the first file is not updated so that it can be regenerated
+// with any local changes.
+
+package choosestage
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+)
+
+var (
+ outputFile string
+ currentFile string
+ bootstrapFile string
+ verbose bool
+)
+
+func init() {
+ flag.StringVar(&outputFile, "o", "", "Output file")
+ flag.StringVar(¤tFile, "current", "", "Current stage's file")
+ flag.StringVar(&bootstrapFile, "bootstrap", "", "Bootstrap file checked into source")
+ flag.BoolVar(&verbose, "v", false, "Verbose mode")
+}
+
+func compareFiles(a, b string) (bool, error) {
+ aData, err := ioutil.ReadFile(a)
+ if err != nil {
+ return false, err
+ }
+
+ bData, err := ioutil.ReadFile(b)
+ if err != nil {
+ return false, err
+ }
+
+ return bytes.Equal(aData, bData), nil
+}
+
+// If the source bootstrap reference file is newer, then we may have gotten
+// other source updates too. So we need to restart everything with the file
+// that was checked in instead of the bootstrap that we last built.
+func copyBootstrapIfNecessary(bootstrapFile, filename string) (bool, error) {
+ if bootstrapFile == "" {
+ return false, nil
+ }
+
+ bootstrapStat, err := os.Stat(bootstrapFile)
+ if err != nil {
+ return false, err
+ }
+
+ fileStat, err := os.Stat(filename)
+ if err != nil {
+ return false, err
+ }
+
+ time := fileStat.ModTime()
+ if !bootstrapStat.ModTime().After(time) {
+ return false, nil
+ }
+
+ fmt.Printf("Newer source version of %s. Copying to %s\n", filepath.Base(bootstrapFile), filepath.Base(filename))
+ if verbose {
+ fmt.Printf("Source: %s\nBuilt: %s\n", bootstrapStat.ModTime(), time)
+ }
+
+ data, err := ioutil.ReadFile(bootstrapFile)
+ if err != nil {
+ return false, err
+ }
+
+ err = ioutil.WriteFile(filename, data, 0666)
+ if err != nil {
+ return false, err
+ }
+
+ // Restore timestamp to force regeneration of the bootstrap.ninja.in
+ err = os.Chtimes(filename, time, time)
+ return true, err
+}
+
+func main() {
+ flag.Parse()
+
+ if flag.NArg() == 0 {
+ fmt.Fprintf(os.Stderr, "Must specify at least one ninja file\n")
+ os.Exit(1)
+ }
+
+ if outputFile == "" {
+ fmt.Fprintf(os.Stderr, "Must specify an output file\n")
+ os.Exit(1)
+ }
+
+ gotoFile := flag.Arg(0)
+ if copied, err := copyBootstrapIfNecessary(bootstrapFile, flag.Arg(0)); err != nil {
+ fmt.Fprintf(os.Stderr, "Failed to copy bootstrap ninja file: %s\n", err)
+ os.Exit(1)
+ } else if !copied {
+ for _, fileName := range flag.Args() {
+ timestampName := fileName + ".timestamp"
+
+ // If we're currently running this stage, and the build.ninja.in
+ // file differs from the current stage file, then it has been rebuilt.
+ // Restart the stage.
+ if currentFile == fileName {
+ if _, err := os.Stat(outputFile); !os.IsNotExist(err) {
+ if ok, err := compareFiles(fileName, outputFile); err != nil {
+ fmt.Fprintf(os.Stderr, "Failure when comparing files: %s\n", err)
+ os.Exit(1)
+ } else if !ok {
+ fmt.Printf("Stage %s has changed, restarting\n", filepath.Base(fileName))
+ gotoFile = fileName
+ break
+ }
+ }
+ }
+
+ fileStat, err := os.Stat(fileName)
+ if err != nil {
+ // Regenerate this stage on error
+ break
+ }
+
+ timestampStat, err := os.Stat(timestampName)
+ if err != nil {
+ // This file may not exist. There's no point for
+ // the first stage to have one, as it should be
+ // a subset of the second stage dependencies,
+ // and both will return to the first stage.
+ continue
+ }
+
+ if verbose {
+ fmt.Printf("For %s:\n file: %s\n time: %s\n", fileName, fileStat.ModTime(), timestampStat.ModTime())
+ }
+
+ // If the timestamp file has a later modification time, that
+ // means that this stage needs to be regenerated. Break, so
+ // that we run the last found stage.
+ if timestampStat.ModTime().After(fileStat.ModTime()) {
+ break
+ }
+
+ gotoFile = fileName
+ }
+ }
+
+ fmt.Printf("Choosing %s for next stage\n", filepath.Base(gotoFile))
+
+ data, err := ioutil.ReadFile(gotoFile)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Can't read file: %s", err)
+ os.Exit(1)
+ }
+
+ err = ioutil.WriteFile(outputFile, data, 0666)
+ if err != nil {
+ fmt.Fprintf(os.Stderr, "Can't write file: %s", err)
+ os.Exit(1)
+ }
+}
diff --git a/tests/expected_all b/tests/expected_all
index 380b0cc..a92f0b3 100644
--- a/tests/expected_all
+++ b/tests/expected_all
@@ -1,2 +1,2 @@
-Choosing build.ninja.in for next stage
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
diff --git a/tests/expected_manifest b/tests/expected_manifest
index 380b0cc..ab67d4b 100644
--- a/tests/expected_manifest
+++ b/tests/expected_manifest
@@ -1,2 +1,3 @@
-Choosing build.ninja.in for next stage
+Newer source version of build.ninja.in. Copying to bootstrap.ninja.in
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
diff --git a/tests/expected_rebuild_test b/tests/expected_rebuild_test
index 162d1db..a92f0b3 100644
--- a/tests/expected_rebuild_test
+++ b/tests/expected_rebuild_test
@@ -1,2 +1,2 @@
-Choosing src.build.ninja.in for next stage
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
diff --git a/tests/expected_regen b/tests/expected_regen
index 162d1db..4f7adaa 100644
--- a/tests/expected_regen
+++ b/tests/expected_regen
@@ -1,2 +1,5 @@
-Choosing src.build.ninja.in for next stage
+Newer source version of src.build.ninja.in. Copying to bootstrap.ninja.in
+Choosing bootstrap.ninja.in for next stage
+Stage bootstrap.ninja.in has changed, restarting
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
diff --git a/tests/expected_start2 b/tests/expected_start2
index dc55ac3..d0b3900 100644
--- a/tests/expected_start2
+++ b/tests/expected_start2
@@ -1 +1,3 @@
+Stage bootstrap.ninja.in has changed, restarting
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
diff --git a/tests/expected_start_add_tests b/tests/expected_start_add_tests
index dc55ac3..d0b3900 100644
--- a/tests/expected_start_add_tests
+++ b/tests/expected_start_add_tests
@@ -1 +1,3 @@
+Stage bootstrap.ninja.in has changed, restarting
+Choosing bootstrap.ninja.in for next stage
Choosing main.ninja.in for next stage
