[specs][starlark] Add a tool for generating starlark that generates a spec
These tools are likely to be short-lived. I'll add tests for them later since
they're need to unblock others for the migration, and they aren't at-risk of
breaking a production system.
Example usage:
bb log <build-id> "load spec" textproto | go run ./cmd/specs to_starlark -recipe fuchsia | yapf
Bug: IN-1366 #comment
Change-Id: Ia4f53dbcd8815000d1dc9d8a7b0e6fc173085586
diff --git a/cmd/specs/generators/fuchsia_py.go b/cmd/specs/generators/fuchsia_py.go
new file mode 100644
index 0000000..128d1b6
--- /dev/null
+++ b/cmd/specs/generators/fuchsia_py.go
@@ -0,0 +1,54 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package generators
+
+import (
+ "errors"
+ "fmt"
+ "io"
+ "io/ioutil"
+ "strings"
+
+ recipes "fuchsia.googlesource.com/infra/infra/fxicfg/starlark/protos/recipes"
+ "fuchsia.googlesource.com/infra/infra/starlark/starlarkgen"
+ "github.com/golang/protobuf/proto"
+)
+
+// FuchsiaPySpecGenerator produces Starlark code that generates a spec for the fuchsia.py
+// recipe.
+type FuchsiaPySpecGenerator struct {
+ w io.Writer
+}
+
+func NewFuchsiaPySpecGenerator(output io.Writer) *FuchsiaPySpecGenerator {
+ return &FuchsiaPySpecGenerator{output}
+}
+
+func (g *FuchsiaPySpecGenerator) Generate(in io.Reader) error {
+ spec, err := g.parseTextproto(in)
+ if err != nil {
+ return fmt.Errorf("failed to parse input: %v", err)
+ }
+ output := starlarkgen.Generate(spec)
+ fmt.Fprintf(g.w, output)
+ return nil
+}
+
+// parse parses a spec for the Fuchsia recipe from the given io.Reader.
+func (g *FuchsiaPySpecGenerator) parseTextproto(r io.Reader) (*recipes.Fuchsia, error) {
+ bytes, err := ioutil.ReadAll(r)
+ if err != nil {
+ return nil, err
+ }
+ textproto := strings.TrimSpace(string(bytes))
+ if textproto == "" {
+ return nil, errors.New("input is empty")
+ }
+ spec := new(recipes.Fuchsia)
+ if err := proto.UnmarshalText(string(textproto), spec); err != nil {
+ return nil, err
+ }
+ return spec, nil
+}
diff --git a/cmd/specs/main.go b/cmd/specs/main.go
new file mode 100644
index 0000000..2d125a5
--- /dev/null
+++ b/cmd/specs/main.go
@@ -0,0 +1,26 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package main implements a tool for working with spec files. Some subcommands are useful
+// for migrating a recipe to a new proto API and addding new spec files. Others are useful
+// for recipe execution with spec files.
+package main
+
+import (
+ "context"
+ "flag"
+ "os"
+
+ "github.com/google/subcommands"
+)
+
+func main() {
+ subcommands.Register(subcommands.HelpCommand(), "")
+ subcommands.Register(subcommands.CommandsCommand(), "")
+ subcommands.Register(subcommands.FlagsCommand(), "")
+ subcommands.Register(&ToStarlarkCommand{}, "")
+
+ flag.Parse()
+ os.Exit(int(subcommands.Execute(context.Background())))
+}
diff --git a/cmd/specs/to_starlark.go b/cmd/specs/to_starlark.go
new file mode 100644
index 0000000..e65419c
--- /dev/null
+++ b/cmd/specs/to_starlark.go
@@ -0,0 +1,69 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package main
+
+import (
+ "context"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+
+ "log"
+ "os"
+
+ "fuchsia.googlesource.com/infra/infra/cmd/specs/generators"
+ "github.com/google/subcommands"
+)
+
+// StarlarkCodeGenerator transforms a text protobuf message read from the given io.Reader
+// into the starlark code that would generate that same message when executed by fxicfg.
+// The output is written to the given io.Writer.
+//
+// This relies on the generated Golang protobuf API being checked in to the fxcicfg pkg.
+type StarlarkCodeGenerator interface {
+ Generate(io.Reader) error
+}
+
+// FuchsiaPyID is the ID used to select the Fuchsia spec generator.
+const FuchsiaPyID = "fuchsia"
+
+// ToStarlarkCommand generates starlark code from a spec.
+type ToStarlarkCommand struct {
+ // The selected StarlarkCodeGenerator.
+ recipe string
+}
+
+func (*ToStarlarkCommand) Name() string { return "to_starlark" }
+func (*ToStarlarkCommand) Usage() string { return "cat /path/to/spec | to_starlark" }
+func (*ToStarlarkCommand) Synopsis() string {
+ return "Generates the fxicfg starlark code that would produce the input spec"
+}
+
+func (cmd *ToStarlarkCommand) SetFlags(f *flag.FlagSet) {
+ f.StringVar(&cmd.recipe, "recipe", FuchsiaPyID, fmt.Sprintf("The recipe whose api is being generated. Options: %v", []string{
+ FuchsiaPyID,
+ }))
+}
+
+func (cmd *ToStarlarkCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...interface{}) subcommands.ExitStatus {
+ generator, err := cmd.selectGenerator(cmd.recipe)
+ if err != nil {
+ log.Fatalf("invalid generator: %v", cmd.recipe)
+ }
+ if err := generator.Generate(os.Stdin); err != nil {
+ log.Fatalf("failed to generate spec: %v", err)
+ }
+ return subcommands.ExitFailure
+}
+
+func (cmd *ToStarlarkCommand) selectGenerator(key string) (StarlarkCodeGenerator, error) {
+ switch key {
+ case FuchsiaPyID:
+ return generators.NewFuchsiaPySpecGenerator(os.Stdout), nil
+ default:
+ return nil, errors.New("generator not found")
+ }
+}
diff --git a/starlark/starlarkfmt/starlarkfmt.go b/starlark/starlarkfmt/starlarkfmt.go
new file mode 100644
index 0000000..499df8d
--- /dev/null
+++ b/starlark/starlarkfmt/starlarkfmt.go
@@ -0,0 +1,50 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package starlarkfmt formats Go values as starlark code.
+// TODO(kjharland): Use the starlark libraries for this?
+package starlarkfmt
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+)
+
+func Format(v reflect.Value) string {
+ switch v.Kind() {
+ case reflect.String:
+ return String(v)
+ case reflect.Slice:
+ return Slice(v)
+ case reflect.Bool:
+ return Bool(v)
+ case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+ reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+ reflect.Float32, reflect.Float64:
+ return Num(v)
+ default:
+ panic(fmt.Sprintf("unimplemented: %v", v.Kind()))
+ }
+}
+
+func Num(v reflect.Value) string {
+ return fmt.Sprintf("%d", v.Int())
+}
+
+func String(v reflect.Value) string {
+ return fmt.Sprintf("\"%s\"", v.String())
+}
+
+func Slice(v reflect.Value) string {
+ output := "["
+ for i := 0; i < v.Len(); i++ {
+ output += fmt.Sprintf("%s,", Format(v.Index(i)))
+ }
+ return output + "]"
+}
+
+func Bool(v reflect.Value) string {
+ return strings.Title(fmt.Sprintf("%t", v.Interface()))
+}
diff --git a/starlark/starlarkgen/ast.go b/starlark/starlarkgen/ast.go
new file mode 100644
index 0000000..31eeca1
--- /dev/null
+++ b/starlark/starlarkgen/ast.go
@@ -0,0 +1,48 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package starlarkgen
+
+import (
+ "bytes"
+ "fmt"
+ "reflect"
+
+ "fuchsia.googlesource.com/infra/infra/starlark/starlarkfmt"
+)
+
+type Node interface {
+ String() string
+}
+
+type ConstructorCall struct {
+ Package string
+ Typename string
+ Kwargs []Node
+}
+
+func (c ConstructorCall) String() string {
+ args := new(bytes.Buffer)
+ for _, arg := range c.Kwargs {
+ args.WriteString(arg.String() + ",")
+ }
+ return fmt.Sprintf("%s.%s(%s)", c.Package, c.Typename, args)
+}
+
+type Kwarg struct {
+ Keyword string
+ Value Node
+}
+
+func (kwarg Kwarg) String() string {
+ return fmt.Sprintf("%s=%s", kwarg.Keyword, kwarg.Value)
+}
+
+type Primitive struct {
+ Value reflect.Value
+}
+
+func (a Primitive) String() string {
+ return starlarkfmt.Format(a.Value)
+}
diff --git a/starlark/starlarkgen/generator.go b/starlark/starlarkgen/generator.go
new file mode 100644
index 0000000..29c209c
--- /dev/null
+++ b/starlark/starlarkgen/generator.go
@@ -0,0 +1,74 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package starlarkgen
+
+import (
+ "reflect"
+ "strings"
+
+ "github.com/golang/protobuf/descriptor"
+ protoc "github.com/golang/protobuf/protoc-gen-go/descriptor"
+)
+
+// Generate produces the starlark code needed to generate the input protobuf message.
+func Generate(msg descriptor.Message) string {
+ ast := genMessage(reflect.ValueOf(msg))
+ return ast.String()
+}
+
+func genMessage(val reflect.Value) Node {
+ // Don't output anything for nil or zero values.
+ if isNothing(val) {
+ return nil
+ }
+
+ // Must read these values before calling val.Elem(). Otherwise the value it reflects
+ // is no longer a descriptor.Message.
+ msg := val.Interface().(descriptor.Message)
+ filedesc, desc := descriptor.ForMessage(msg)
+ node := &ConstructorCall{
+ Package: filedesc.GetPackage(),
+ Typename: desc.GetName(),
+ }
+ val = val.Elem()
+ for _, field := range desc.GetField() {
+ fieldValue := val.FieldByName(strings.Title(field.GetJsonName()))
+ fieldAst := genField(field, fieldValue)
+ // Skip fields that have nil pointer values.
+ if fieldAst == nil {
+ continue
+ }
+ // Skip fields that are set to zero-values.
+ if primitive, ok := fieldAst.(Primitive); ok && isZero(primitive.Value) {
+ continue
+ }
+ node.Kwargs = append(node.Kwargs, Kwarg{
+ Keyword: field.GetName(),
+ Value: fieldAst,
+ })
+ }
+ return node
+}
+
+func genField(desc *protoc.FieldDescriptorProto, val reflect.Value) Node {
+ if desc.GetTypeName() == "" {
+ // This field is a not a proto message (and the instance is not a Go struct). Just
+ // write the raw value.
+ return Primitive{val}
+ }
+
+ // This field is a proto.Message, which has a corresponding Go struct type. Generate
+ // it and its children recursively.
+ return genMessage(val)
+}
+
+func isZero(val reflect.Value) bool {
+ zero := reflect.Zero(val.Type()).Interface()
+ return reflect.DeepEqual(zero, val.Interface())
+}
+
+func isNothing(val reflect.Value) bool {
+ return isZero(val) || val.IsNil()
+}
