Implement plugins for bootstrap go modules
Now that we have multi-stage bootstrapping, we can make the primary
builder build more dynamic. Add the concept of plugins that will be
linked and loaded into bootstrap_go_binary or bootstrap_go_package
modules. It's expected that the plugin's init() functions will do
whatever registration is necessary.
Example Blueprint definition:
bootstrap_go_binary {
name: "builder",
...
}
bootstrap_go_package {
name: "plugin1",
pluginFor: ["builder"],
}
A package may specify more than one plugin if it will be inserted into
more than one go module.
Change-Id: I109835f444196b66fc4018c3fa36ba0875823184
diff --git a/Blueprints b/Blueprints
index 173017a..70d57db 100644
--- a/Blueprints
+++ b/Blueprints
@@ -128,3 +128,8 @@
name = "choosestage",
srcs = ["choosestage/choosestage.go"],
)
+
+bootstrap_go_binary{
+ name = "loadplugins",
+ srcs = ["loadplugins/loadplugins.go"],
+}
diff --git a/bootstrap/bootstrap.go b/bootstrap/bootstrap.go
index 3cc5672..c9f1b68 100644
--- a/bootstrap/bootstrap.go
+++ b/bootstrap/bootstrap.go
@@ -29,8 +29,9 @@
var (
pctx = blueprint.NewPackageContext("github.com/google/blueprint/bootstrap")
- goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
- chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage"))
+ goTestMainCmd = pctx.StaticVariable("goTestMainCmd", filepath.Join(bootstrapDir, "bin", "gotestmain"))
+ chooseStageCmd = pctx.StaticVariable("chooseStageCmd", filepath.Join(bootstrapDir, "bin", "choosestage"))
+ pluginGenSrcCmd = pctx.StaticVariable("pluginGenSrcCmd", filepath.Join(bootstrapDir, "bin", "loadplugins"))
compile = pctx.StaticRule("compile",
blueprint.RuleParams{
@@ -54,6 +55,13 @@
},
"pkg")
+ pluginGenSrc = pctx.StaticRule("pluginGenSrc",
+ blueprint.RuleParams{
+ Command: "$pluginGenSrcCmd -o $out -p $pkg $plugins",
+ Description: "create $out",
+ },
+ "pkg", "plugins")
+
test = pctx.StaticRule("test",
blueprint.RuleParams{
Command: "(cd $pkgSrcDir && $$OLDPWD/$in -test.short) && touch $out",
@@ -121,6 +129,14 @@
})
}
+func pluginDeps(ctx blueprint.BottomUpMutatorContext) {
+ if pkg, ok := ctx.Module().(*goPackage); ok {
+ for _, plugin := range pkg.properties.PluginFor {
+ ctx.AddReverseDependency(ctx.Module(), plugin)
+ }
+ }
+}
+
type goPackageProducer interface {
GoPkgRoot() string
GoPackageTarget() string
@@ -141,6 +157,20 @@
return ok
}
+type goPluginProvider interface {
+ GoPkgPath() string
+ IsPluginFor(string) bool
+}
+
+func isGoPluginFor(name string) func(blueprint.Module) bool {
+ return func(module blueprint.Module) bool {
+ if plugin, ok := module.(goPluginProvider); ok {
+ return plugin.IsPluginFor(name)
+ }
+ return false
+ }
+}
+
func isBootstrapModule(module blueprint.Module) bool {
_, isPackage := module.(*goPackage)
_, isBinary := module.(*goBinary)
@@ -155,9 +185,10 @@
// A goPackage is a module for building Go packages.
type goPackage struct {
properties struct {
- PkgPath string
- Srcs []string
- TestSrcs []string
+ PkgPath string
+ Srcs []string
+ TestSrcs []string
+ PluginFor []string
}
// The root dir in which the package .a file is located. The full .a file
@@ -189,6 +220,10 @@
}
}
+func (g *goPackage) GoPkgPath() string {
+ return g.properties.PkgPath
+}
+
func (g *goPackage) GoPkgRoot() string {
return g.pkgRoot
}
@@ -209,8 +244,22 @@
g.buildStage = buildStage
}
+func (g *goPackage) IsPluginFor(name string) bool {
+ for _, plugin := range g.properties.PluginFor {
+ if plugin == name {
+ return true
+ }
+ }
+ return false
+}
+
func (g *goPackage) GenerateBuildActions(ctx blueprint.ModuleContext) {
- name := ctx.ModuleName()
+ var (
+ name = ctx.ModuleName()
+ hasPlugins = false
+ pluginSrc = ""
+ genSrcs = []string{}
+ )
if g.properties.PkgPath == "" {
ctx.ModuleErrorf("module %s did not specify a valid pkgPath", name)
@@ -225,6 +274,13 @@
filepath.FromSlash(g.properties.PkgPath)+".a")
}
+ ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
+ func(module blueprint.Module) { hasPlugins = true })
+ if hasPlugins {
+ pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
+ genSrcs = append(genSrcs, pluginSrc)
+ }
+
// We only actually want to build the builder modules if we're running as
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
// the circular dependence that occurs when the builder requires a new Ninja
@@ -233,19 +289,23 @@
if g.config.stage == g.BuildStage() {
var deps []string
+ if hasPlugins && !buildGoPluginLoader(ctx, g.properties.PkgPath, pluginSrc, g.config.stage) {
+ return
+ }
+
if g.config.runGoTests {
deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
- g.properties.PkgPath, g.properties.Srcs,
+ g.properties.PkgPath, g.properties.Srcs, genSrcs,
g.properties.TestSrcs)
}
buildGoPackage(ctx, g.pkgRoot, g.properties.PkgPath, g.archiveFile,
- g.properties.Srcs, deps)
+ g.properties.Srcs, genSrcs, deps)
} else if g.config.stage != StageBootstrap {
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
- phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
+ phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil, nil)
}
- phonyGoTarget(ctx, g.archiveFile, g.properties.Srcs, nil)
+ phonyGoTarget(ctx, g.archiveFile, g.properties.Srcs, genSrcs, nil)
}
}
@@ -296,12 +356,22 @@
archiveFile = filepath.Join(objDir, name+".a")
aoutFile = filepath.Join(objDir, "a.out")
binaryFile = filepath.Join("$BinDir", name)
+ hasPlugins = false
+ pluginSrc = ""
+ genSrcs = []string{}
)
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
g.testArchiveFile = filepath.Join(testRoot(ctx), name+".a")
}
+ ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
+ func(module blueprint.Module) { hasPlugins = true })
+ if hasPlugins {
+ pluginSrc = filepath.Join(moduleGenSrcDir(ctx), "plugin.go")
+ genSrcs = append(genSrcs, pluginSrc)
+ }
+
// We only actually want to build the builder modules if we're running as
// minibp (i.e. we're generating a bootstrap Ninja file). This is to break
// the circular dependence that occurs when the builder requires a new Ninja
@@ -310,12 +380,16 @@
if g.config.stage == g.BuildStage() {
var deps []string
- if g.config.runGoTests {
- deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
- name, g.properties.Srcs, g.properties.TestSrcs)
+ if hasPlugins && !buildGoPluginLoader(ctx, "main", pluginSrc, g.config.stage) {
+ return
}
- buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, deps)
+ if g.config.runGoTests {
+ deps = buildGoTest(ctx, testRoot(ctx), g.testArchiveFile,
+ name, g.properties.Srcs, genSrcs, g.properties.TestSrcs)
+ }
+
+ buildGoPackage(ctx, objDir, name, archiveFile, g.properties.Srcs, genSrcs, deps)
var libDirFlags []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@@ -345,19 +419,49 @@
})
} else if g.config.stage != StageBootstrap {
if len(g.properties.TestSrcs) > 0 && g.config.runGoTests {
- phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil)
+ phonyGoTarget(ctx, g.testArchiveFile, g.properties.TestSrcs, nil, nil)
}
intermediates := []string{aoutFile, archiveFile}
- phonyGoTarget(ctx, binaryFile, g.properties.Srcs, intermediates)
+ phonyGoTarget(ctx, binaryFile, g.properties.Srcs, genSrcs, intermediates)
}
}
+func buildGoPluginLoader(ctx blueprint.ModuleContext, pkgPath, pluginSrc string, stage Stage) bool {
+ ret := true
+ name := ctx.ModuleName()
+
+ var pluginPaths []string
+ ctx.VisitDepsDepthFirstIf(isGoPluginFor(name),
+ func(module blueprint.Module) {
+ plugin := module.(goPluginProvider)
+ pluginPaths = append(pluginPaths, plugin.GoPkgPath())
+ if stage == StageBootstrap {
+ ctx.OtherModuleErrorf(module, "plugin %q may not be included in core module %q",
+ ctx.OtherModuleName(module), name)
+ ret = false
+ }
+ })
+
+ ctx.Build(pctx, blueprint.BuildParams{
+ Rule: pluginGenSrc,
+ Outputs: []string{pluginSrc},
+ Implicits: []string{"$pluginGenSrcCmd"},
+ Args: map[string]string{
+ "pkg": pkgPath,
+ "plugins": strings.Join(pluginPaths, " "),
+ },
+ })
+
+ return ret
+}
+
func buildGoPackage(ctx blueprint.ModuleContext, pkgRoot string,
- pkgPath string, archiveFile string, srcs []string, orderDeps []string) {
+ pkgPath string, archiveFile string, srcs []string, genSrcs []string, orderDeps []string) {
srcDir := moduleSrcDir(ctx)
srcFiles := pathtools.PrefixPaths(srcs, srcDir)
+ srcFiles = append(srcFiles, genSrcs...)
var incFlags []string
deps := []string{"$compileCmd"}
@@ -388,9 +492,8 @@
})
}
-func buildGoTest(ctx blueprint.ModuleContext, testRoot string,
- testPkgArchive string, pkgPath string, srcs []string,
- testSrcs []string) []string {
+func buildGoTest(ctx blueprint.ModuleContext, testRoot, testPkgArchive,
+ pkgPath string, srcs, genSrcs, testSrcs []string) []string {
if len(testSrcs) == 0 {
return nil
@@ -405,7 +508,7 @@
testPassed := filepath.Join(testRoot, "test.passed")
buildGoPackage(ctx, testRoot, pkgPath, testPkgArchive,
- append(srcs, testSrcs...), nil)
+ append(srcs, testSrcs...), genSrcs, nil)
ctx.Build(pctx, blueprint.BuildParams{
Rule: goTestMain,
@@ -460,7 +563,7 @@
}
func phonyGoTarget(ctx blueprint.ModuleContext, target string, srcs []string,
- intermediates []string) {
+ gensrcs []string, intermediates []string) {
var depTargets []string
ctx.VisitDepsDepthFirstIf(isGoPackageProducer,
@@ -472,6 +575,7 @@
moduleDir := ctx.ModuleDir()
srcs = pathtools.PrefixPaths(srcs, filepath.Join("$srcDir", moduleDir))
+ srcs = append(srcs, gensrcs...)
ctx.Build(pctx, blueprint.BuildParams{
Rule: phony,
@@ -884,3 +988,8 @@
func moduleObjDir(ctx blueprint.ModuleContext) string {
return filepath.Join(bootstrapDir, ctx.ModuleName(), "obj")
}
+
+// moduleGenSrcDir returns the module-specific generated sources path.
+func moduleGenSrcDir(ctx blueprint.ModuleContext) string {
+ return filepath.Join(bootstrapDir, ctx.ModuleName(), "gen")
+}
diff --git a/bootstrap/command.go b/bootstrap/command.go
index 63f07ec..cf88666 100644
--- a/bootstrap/command.go
+++ b/bootstrap/command.go
@@ -90,6 +90,7 @@
runGoTests: runGoTests,
}
+ ctx.RegisterBottomUpMutator("bootstrap_plugin_deps", pluginDeps)
ctx.RegisterModuleType("bootstrap_go_package", newGoPackageModuleFactory(bootstrapConfig))
ctx.RegisterModuleType("bootstrap_core_go_binary", newGoBinaryModuleFactory(bootstrapConfig, StageBootstrap))
ctx.RegisterModuleType("bootstrap_go_binary", newGoBinaryModuleFactory(bootstrapConfig, StagePrimary))
diff --git a/build.ninja.in b/build.ninja.in
index feaad7a..49d808a 100644
--- a/build.ninja.in
+++ b/build.ninja.in
@@ -54,7 +54,7 @@
# Module: blueprint
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:1:1
build $
@@ -80,7 +80,7 @@
# Module: blueprint-bootstrap
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:70:1
build $
@@ -107,7 +107,7 @@
# Module: blueprint-bootstrap-bpdoc
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:89:1
build $
@@ -127,7 +127,7 @@
# Module: blueprint-deptools
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:46:1
build $
@@ -142,7 +142,7 @@
# Module: blueprint-parser
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:31:1
build $
@@ -159,7 +159,7 @@
# Module: blueprint-pathtools
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:52:1
build $
@@ -174,7 +174,7 @@
# Module: blueprint-proptools
# Variant:
# Type: bootstrap_go_package
-# Factory: github.com/google/blueprint/bootstrap.func·002
+# Factory: github.com/google/blueprint/bootstrap.func·003
# Defined: Blueprints:64:1
build $
@@ -189,7 +189,7 @@
# Module: choosestage
# Variant:
# Type: bootstrap_core_go_binary
-# Factory: github.com/google/blueprint/bootstrap.func·003
+# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:127:1
build ${g.bootstrap.buildDir}/.bootstrap/choosestage/obj/choosestage.a: $
@@ -211,7 +211,7 @@
# Module: gotestmain
# Variant:
# Type: bootstrap_core_go_binary
-# Factory: github.com/google/blueprint/bootstrap.func·003
+# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:122:1
build ${g.bootstrap.buildDir}/.bootstrap/gotestmain/obj/gotestmain.a: $
@@ -233,7 +233,7 @@
# Module: minibp
# Variant:
# Type: bootstrap_core_go_binary
-# Factory: github.com/google/blueprint/bootstrap.func·003
+# Factory: github.com/google/blueprint/bootstrap.func·005
# Defined: Blueprints:101:1
build ${g.bootstrap.buildDir}/.bootstrap/minibp/obj/minibp.a: $
@@ -262,7 +262,7 @@
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# Singleton: bootstrap
-# Factory: github.com/google/blueprint/bootstrap.func·008
+# Factory: github.com/google/blueprint/bootstrap.func·012
rule s.bootstrap.primarybp
command = ${g.bootstrap.BinDir}/minibp --build-primary ${runTests} -m ${g.bootstrap.bootstrapManifest} --timestamp ${timestamp} --timestampdep ${timestampdep} -b ${g.bootstrap.buildDir} -d ${outfile}.d -o ${outfile} ${in}
diff --git a/context.go b/context.go
index c5e8b15..3892f4e 100644
--- a/context.go
+++ b/context.go
@@ -1077,6 +1077,11 @@
return errs
}
+ errs = c.runMutators(config)
+ if len(errs) > 0 {
+ return errs
+ }
+
c.dependenciesReady = true
return nil
}
@@ -1151,6 +1156,22 @@
return
}
+// findMatchingVariant searches the moduleGroup for a module with the same variant as module,
+// and returns the matching module, or nil if one is not found.
+func (c *Context) findMatchingVariant(module *moduleInfo, group *moduleGroup) *moduleInfo {
+ if len(group.modules) == 1 {
+ return group.modules[0]
+ } else {
+ for _, m := range group.modules {
+ if m.variant.equal(module.dependencyVariant) {
+ return m
+ }
+ }
+ }
+
+ return nil
+}
+
func (c *Context) addDependency(module *moduleInfo, depName string) []error {
depsPos := module.propertyPos["deps"]
@@ -1176,16 +1197,9 @@
}
}
- if len(depInfo.modules) == 1 {
- module.directDeps = append(module.directDeps, depInfo.modules[0])
+ if m := c.findMatchingVariant(module, depInfo); m != nil {
+ module.directDeps = append(module.directDeps, m)
return nil
- } else {
- for _, m := range depInfo.modules {
- if m.variant.equal(module.dependencyVariant) {
- module.directDeps = append(module.directDeps, m)
- return nil
- }
- }
}
return []error{&Error{
@@ -1196,6 +1210,36 @@
}}
}
+func (c *Context) addReverseDependency(module *moduleInfo, destName string) []error {
+ if destName == module.properties.Name {
+ return []error{&Error{
+ Err: fmt.Errorf("%q depends on itself", destName),
+ Pos: module.pos,
+ }}
+ }
+
+ destInfo, ok := c.moduleGroups[destName]
+ if !ok {
+ return []error{&Error{
+ Err: fmt.Errorf("%q has a reverse dependency on undefined module %q",
+ module.properties.Name, destName),
+ Pos: module.pos,
+ }}
+ }
+
+ if m := c.findMatchingVariant(module, destInfo); m != nil {
+ m.directDeps = append(m.directDeps, module)
+ return nil
+ }
+
+ return []error{&Error{
+ Err: fmt.Errorf("reverse dependency %q of %q missing variant %q",
+ destName, module.properties.Name,
+ c.prettyPrintVariant(module.dependencyVariant)),
+ Pos: module.pos,
+ }}
+}
+
func (c *Context) addVariationDependency(module *moduleInfo, variations []Variation,
depName string, far bool) []error {
@@ -1435,11 +1479,6 @@
}
}
- errs = c.runMutators(config)
- if len(errs) > 0 {
- return nil, errs
- }
-
liveGlobals := newLiveTracker(config)
c.initSpecialVariables()
diff --git a/loadplugins/loadplugins.go b/loadplugins/loadplugins.go
new file mode 100644
index 0000000..3c7e1e3
--- /dev/null
+++ b/loadplugins/loadplugins.go
@@ -0,0 +1,67 @@
+// Copyright 2015 Google Inc. All rights reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "bytes"
+ "flag"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "text/template"
+)
+
+var (
+ output = flag.String("o", "", "output filename")
+ pkg = flag.String("p", "main", "package name")
+)
+
+func main() {
+ flag.Parse()
+
+ if flag.NArg() == 0 {
+ fmt.Fprintln(os.Stderr, "error: must pass at least one input")
+ os.Exit(1)
+ }
+
+ buf := &bytes.Buffer{}
+
+ err := pluginTmpl.Execute(buf, struct {
+ Package string
+ Plugins []string
+ }{
+ filepath.Base(*pkg),
+ flag.Args(),
+ })
+ if err != nil {
+ panic(err)
+ }
+
+ err = ioutil.WriteFile(*output, buf.Bytes(), 0666)
+ if err != nil {
+ panic(err)
+ }
+}
+
+var pluginTmpl = template.Must(template.New("pluginloader").Parse(`
+package {{.Package}}
+
+import (
+{{range .Plugins}}
+ _ "{{.}}"
+{{end}}
+)
+`))
diff --git a/module_ctx.go b/module_ctx.go
index 76d3977..6254596 100644
--- a/module_ctx.go
+++ b/module_ctx.go
@@ -359,8 +359,7 @@
type mutatorContext struct {
baseModuleContext
- name string
- dependenciesModified bool
+ name string
}
type baseMutatorContext interface {
@@ -389,6 +388,7 @@
baseMutatorContext
AddDependency(module Module, name string)
+ AddReverseDependency(module Module, name string)
CreateVariations(...string) []Module
CreateLocalVariations(...string) []Module
SetDependencyVariation(string)
@@ -464,8 +464,7 @@
return mctx.module.logicModule
}
-// Add a dependency to the given module. The depender can be a specific variant
-// of a module, but the dependee must be a module that has no variations.
+// Add a dependency to the given module.
// Does not affect the ordering of the current mutator pass, but will be ordered
// correctly for all future mutator passes.
func (mctx *mutatorContext) AddDependency(module Module, depName string) {
@@ -473,7 +472,16 @@
if len(errs) > 0 {
mctx.errs = append(mctx.errs, errs...)
}
- mctx.dependenciesModified = true
+}
+
+// Add a dependency from the destination to the given module.
+// Does not affect the ordering of the current mutator pass, but will be ordered
+// correctly for all future mutator passes.
+func (mctx *mutatorContext) AddReverseDependency(module Module, destName string) {
+ errs := mctx.context.addReverseDependency(mctx.context.moduleInfo[module], destName)
+ if len(errs) > 0 {
+ mctx.errs = append(mctx.errs, errs...)
+ }
}
func (mctx *mutatorContext) VisitDirectDeps(visit func(Module)) {
