[roller-configurator] Support jiri pkgs and projects
Add logic to resolve details about Jiri projects and packages based
solely on project/package names, as was as validation (e.g. checking that the referenced projects/packages actually exist in Jiri manifests).
Bug: b/42051371
Change-Id: Iea88be48bc97c383102bd7d38572dcc183b7d0dc
Reviewed-on: https://fuchsia-review.googlesource.com/c/infra/infra/+/964153
Reviewed-by: Carver Forbes <carverforbes@google.com>
Commit-Queue: Auto-Submit <auto-submit@fuchsia-infra.iam.gserviceaccount.com>
Fuchsia-Auto-Submit: Oliver Newman <olivernewman@google.com>
diff --git a/cmd/roller-configurator/jiri/manifest.go b/cmd/roller-configurator/jiri/manifest.go
new file mode 100644
index 0000000..363fb36
--- /dev/null
+++ b/cmd/roller-configurator/jiri/manifest.go
@@ -0,0 +1,61 @@
+// Copyright 2023 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package jiri contains jiri manifest schema definitions copied from Jiri's
+// source code.
+//
+// Only fields relevant to roller configuration are included, others are
+// omitted.
+package jiri
+
+import (
+ "encoding/xml"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+)
+
+type Manifest struct {
+ Projects []Project `xml:"projects>project"`
+ Packages []Package `xml:"packages>package"`
+ XMLName struct{} `xml:"manifest"`
+}
+
+// Project represents a jiri project.
+type Project struct {
+ // Name is the project name.
+ Name string `xml:"name,attr,omitempty"`
+
+ // Remote is the project remote.
+ Remote string `xml:"remote,attr,omitempty"`
+
+ // RemoteBranch is the name of the remote branch to track.
+ RemoteBranch string `xml:"remotebranch,attr,omitempty"`
+}
+
+// Package struct represents the <package> tag in manifest files.
+type Package struct {
+ // Name represents the remote cipd path of the package.
+ Name string `xml:"name,attr"`
+
+ // Version represents the version tag of the cipd package.
+ Version string `xml:"version,attr"`
+}
+
+// LoadManifest reads a Jiri manifest from disk.
+func LoadManifest(path string) (*Manifest, error) {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ return nil, err
+ }
+ var manifest Manifest
+ if err := xml.Unmarshal(b, &manifest); err != nil {
+ if errors.Is(err, io.EOF) {
+ return nil, fmt.Errorf("invalid XML in %s", path)
+ }
+ return nil, err
+ }
+ return &manifest, nil
+}
diff --git a/cmd/roller-configurator/proto/resolvers.go b/cmd/roller-configurator/proto/resolvers.go
index 1a94844..078aef8 100644
--- a/cmd/roller-configurator/proto/resolvers.go
+++ b/cmd/roller-configurator/proto/resolvers.go
@@ -6,18 +6,14 @@
import (
"context"
- "encoding/json"
-
- "google.golang.org/protobuf/encoding/protojson"
- "google.golang.org/protobuf/proto"
)
func (s *GitSubmodule) Resolve(ctx context.Context, repoRoot string) (map[string]any, error) {
- res, err := protoToMap(s)
+ res, err := ProtoToMap(s)
if err != nil {
return nil, err
}
- res["url"], err = s.url(ctx, repoRoot)
+ res["remote"], err = s.url(ctx, repoRoot)
if err != nil {
return nil, err
}
@@ -26,7 +22,7 @@
}
func (c *CIPDEnsureFile) Resolve(ctx context.Context, repoRoot string) (map[string]any, error) {
- res, err := protoToMap(c)
+ res, err := ProtoToMap(c)
if err != nil {
return nil, err
}
@@ -35,35 +31,28 @@
}
func (p *JiriProject) Resolve(ctx context.Context, repoRoot string) (map[string]any, error) {
- res, err := protoToMap(p)
+ me, err := p.manifestEntry(repoRoot)
if err != nil {
return nil, err
}
+ res, err := ProtoToMap(p)
+ if err != nil {
+ return nil, err
+ }
+ res["remote"] = me.Remote
+ res["remote_branch"] = me.RemoteBranch
res["type"] = "jiri_project"
return res, nil
}
func (p *JiriPackages) Resolve(ctx context.Context, repoRoot string) (map[string]any, error) {
- res, err := protoToMap(p)
+ if p.Ref == "" {
+ p.Ref = "latest"
+ }
+ res, err := ProtoToMap(p)
if err != nil {
return nil, err
}
res["type"] = "jiri_packages"
return res, nil
}
-
-func protoToMap(m proto.Message) (map[string]any, error) {
- opts := protojson.MarshalOptions{
- UseProtoNames: true,
- EmitUnpopulated: true,
- }
- b, err := opts.Marshal(m)
- if err != nil {
- return nil, err
- }
- res := make(map[string]any)
- if err := json.Unmarshal(b, &res); err != nil {
- return nil, err
- }
- return res, nil
-}
diff --git a/cmd/roller-configurator/proto/rollers_cfg.pb.go b/cmd/roller-configurator/proto/rollers_cfg.pb.go
index 7c88c2a..2c505ad 100644
--- a/cmd/roller-configurator/proto/rollers_cfg.pb.go
+++ b/cmd/roller-configurator/proto/rollers_cfg.pb.go
@@ -13,7 +13,6 @@
import (
protoreflect "google.golang.org/protobuf/reflect/protoreflect"
protoimpl "google.golang.org/protobuf/runtime/protoimpl"
- structpb "google.golang.org/protobuf/types/known/structpb"
reflect "reflect"
sync "sync"
)
@@ -378,8 +377,8 @@
sizeCache protoimpl.SizeCache
unknownFields protoimpl.UnknownFields
- // Mapping from Jiri manifest path to list of packages to roll within that
- PackagesByManifest map[string]*structpb.ListValue `protobuf:"bytes,1,rep,name=packages_by_manifest,json=packagesByManifest,proto3" json:"packages_by_manifest,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+ // Jiri manifests containing packages to roll.
+ Manifests []*JiriPackages_Manifest `protobuf:"bytes,1,rep,name=manifests,proto3" json:"manifests,omitempty"`
// CIPD ref to track. Default is "latest".
Ref string `protobuf:"bytes,2,opt,name=ref,proto3" json:"ref,omitempty"`
}
@@ -416,9 +415,9 @@
return file_rollers_cfg_proto_rawDescGZIP(), []int{5}
}
-func (x *JiriPackages) GetPackagesByManifest() map[string]*structpb.ListValue {
+func (x *JiriPackages) GetManifests() []*JiriPackages_Manifest {
if x != nil {
- return x.PackagesByManifest
+ return x.Manifests
}
return nil
}
@@ -430,62 +429,110 @@
return ""
}
+type JiriPackages_Manifest struct {
+ state protoimpl.MessageState
+ sizeCache protoimpl.SizeCache
+ unknownFields protoimpl.UnknownFields
+
+ Path string `protobuf:"bytes,1,opt,name=path,proto3" json:"path,omitempty"`
+ Packages []string `protobuf:"bytes,2,rep,name=packages,proto3" json:"packages,omitempty"`
+}
+
+func (x *JiriPackages_Manifest) Reset() {
+ *x = JiriPackages_Manifest{}
+ if protoimpl.UnsafeEnabled {
+ mi := &file_rollers_cfg_proto_msgTypes[6]
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ ms.StoreMessageInfo(mi)
+ }
+}
+
+func (x *JiriPackages_Manifest) String() string {
+ return protoimpl.X.MessageStringOf(x)
+}
+
+func (*JiriPackages_Manifest) ProtoMessage() {}
+
+func (x *JiriPackages_Manifest) ProtoReflect() protoreflect.Message {
+ mi := &file_rollers_cfg_proto_msgTypes[6]
+ if protoimpl.UnsafeEnabled && x != nil {
+ ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
+ if ms.LoadMessageInfo() == nil {
+ ms.StoreMessageInfo(mi)
+ }
+ return ms
+ }
+ return mi.MessageOf(x)
+}
+
+// Deprecated: Use JiriPackages_Manifest.ProtoReflect.Descriptor instead.
+func (*JiriPackages_Manifest) Descriptor() ([]byte, []int) {
+ return file_rollers_cfg_proto_rawDescGZIP(), []int{5, 0}
+}
+
+func (x *JiriPackages_Manifest) GetPath() string {
+ if x != nil {
+ return x.Path
+ }
+ return ""
+}
+
+func (x *JiriPackages_Manifest) GetPackages() []string {
+ if x != nil {
+ return x.Packages
+ }
+ return nil
+}
+
var File_rollers_cfg_proto protoreflect.FileDescriptor
var file_rollers_cfg_proto_rawDesc = []byte{
0x0a, 0x11, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x5f, 0x63, 0x66, 0x67, 0x2e, 0x70, 0x72,
- 0x6f, 0x74, 0x6f, 0x1a, 0x1c, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74,
- 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x73, 0x74, 0x72, 0x75, 0x63, 0x74, 0x2e, 0x70, 0x72, 0x6f, 0x74,
- 0x6f, 0x22, 0x2b, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x0a, 0x07, 0x72,
- 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x07, 0x2e, 0x52,
- 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x22, 0xa9,
- 0x02, 0x0a, 0x06, 0x52, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x2d, 0x0a, 0x09, 0x73, 0x75, 0x62,
- 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x47,
- 0x69, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x09, 0x73,
- 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x3b, 0x0a, 0x10, 0x63, 0x69, 0x70, 0x64,
- 0x5f, 0x65, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x43, 0x49, 0x50, 0x44, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x46,
- 0x69, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, 0x70, 0x64, 0x45, 0x6e, 0x73, 0x75, 0x72,
- 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x31, 0x0a, 0x0c, 0x6a, 0x69, 0x72, 0x69, 0x5f, 0x70, 0x72,
- 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x4a, 0x69,
- 0x72, 0x69, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x6a, 0x69, 0x72,
- 0x69, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x0d, 0x6a, 0x69, 0x72, 0x69,
- 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32,
- 0x0d, 0x2e, 0x4a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x48, 0x00,
- 0x52, 0x0c, 0x6a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x12, 0x1a,
- 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09,
- 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x6e, 0x6f,
- 0x74, 0x69, 0x66, 0x79, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x06, 0x20, 0x03, 0x28,
- 0x09, 0x52, 0x0c, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x45, 0x6d, 0x61, 0x69, 0x6c, 0x73, 0x42,
- 0x09, 0x0a, 0x07, 0x74, 0x6f, 0x5f, 0x72, 0x6f, 0x6c, 0x6c, 0x22, 0x22, 0x0a, 0x0c, 0x47, 0x69,
- 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61,
- 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x22, 0x36,
- 0x0a, 0x0e, 0x43, 0x49, 0x50, 0x44, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65,
- 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04,
- 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28,
- 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x43, 0x0a, 0x0b, 0x4a, 0x69, 0x72, 0x69, 0x50, 0x72,
- 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
- 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73,
- 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0xdc, 0x01, 0x0a, 0x0c,
- 0x4a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x12, 0x57, 0x0a, 0x14,
- 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x5f, 0x62, 0x79, 0x5f, 0x6d, 0x61, 0x6e, 0x69,
- 0x66, 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x4a, 0x69, 0x72,
- 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67,
- 0x65, 0x73, 0x42, 0x79, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x45, 0x6e, 0x74, 0x72,
- 0x79, 0x52, 0x12, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x42, 0x79, 0x4d, 0x61, 0x6e,
- 0x69, 0x66, 0x65, 0x73, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01,
- 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x1a, 0x61, 0x0a, 0x17, 0x50, 0x61, 0x63, 0x6b, 0x61,
- 0x67, 0x65, 0x73, 0x42, 0x79, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x45, 0x6e, 0x74,
- 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52,
- 0x03, 0x6b, 0x65, 0x79, 0x12, 0x30, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20,
- 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f,
- 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4c, 0x69, 0x73, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x52,
- 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x6f,
- 0x2e, 0x66, 0x75, 0x63, 0x68, 0x73, 0x69, 0x61, 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66,
- 0x72, 0x61, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2d, 0x63, 0x6f,
- 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x6f, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f,
- 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x6f, 0x74, 0x6f, 0x22, 0x2b, 0x0a, 0x06, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x12, 0x21, 0x0a,
+ 0x07, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x07,
+ 0x2e, 0x52, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x52, 0x07, 0x72, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x73,
+ 0x22, 0xa9, 0x02, 0x0a, 0x06, 0x52, 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x12, 0x2d, 0x0a, 0x09, 0x73,
+ 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d,
+ 0x2e, 0x47, 0x69, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x48, 0x00, 0x52,
+ 0x09, 0x73, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x3b, 0x0a, 0x10, 0x63, 0x69,
+ 0x70, 0x64, 0x5f, 0x65, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02,
+ 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x43, 0x49, 0x50, 0x44, 0x45, 0x6e, 0x73, 0x75, 0x72,
+ 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x00, 0x52, 0x0e, 0x63, 0x69, 0x70, 0x64, 0x45, 0x6e, 0x73,
+ 0x75, 0x72, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x31, 0x0a, 0x0c, 0x6a, 0x69, 0x72, 0x69, 0x5f,
+ 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e,
+ 0x4a, 0x69, 0x72, 0x69, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x48, 0x00, 0x52, 0x0b, 0x6a,
+ 0x69, 0x72, 0x69, 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x34, 0x0a, 0x0d, 0x6a, 0x69,
+ 0x72, 0x69, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28,
+ 0x0b, 0x32, 0x0d, 0x2e, 0x4a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73,
+ 0x48, 0x00, 0x52, 0x0c, 0x6a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73,
+ 0x12, 0x1a, 0x0a, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x18, 0x05, 0x20, 0x01,
+ 0x28, 0x09, 0x52, 0x08, 0x73, 0x63, 0x68, 0x65, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x23, 0x0a, 0x0d,
+ 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x5f, 0x65, 0x6d, 0x61, 0x69, 0x6c, 0x73, 0x18, 0x06, 0x20,
+ 0x03, 0x28, 0x09, 0x52, 0x0c, 0x6e, 0x6f, 0x74, 0x69, 0x66, 0x79, 0x45, 0x6d, 0x61, 0x69, 0x6c,
+ 0x73, 0x42, 0x09, 0x0a, 0x07, 0x74, 0x6f, 0x5f, 0x72, 0x6f, 0x6c, 0x6c, 0x22, 0x22, 0x0a, 0x0c,
+ 0x47, 0x69, 0x74, 0x53, 0x75, 0x62, 0x6d, 0x6f, 0x64, 0x75, 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04,
+ 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68,
+ 0x22, 0x36, 0x0a, 0x0e, 0x43, 0x49, 0x50, 0x44, 0x45, 0x6e, 0x73, 0x75, 0x72, 0x65, 0x46, 0x69,
+ 0x6c, 0x65, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20,
+ 0x01, 0x28, 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x22, 0x43, 0x0a, 0x0b, 0x4a, 0x69, 0x72, 0x69,
+ 0x50, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x61, 0x6e, 0x69, 0x66,
+ 0x65, 0x73, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x61, 0x6e, 0x69, 0x66,
+ 0x65, 0x73, 0x74, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x18, 0x02,
+ 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x70, 0x72, 0x6f, 0x6a, 0x65, 0x63, 0x74, 0x22, 0x92, 0x01,
+ 0x0a, 0x0c, 0x4a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x12, 0x34,
+ 0x0a, 0x09, 0x6d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28,
+ 0x0b, 0x32, 0x16, 0x2e, 0x4a, 0x69, 0x72, 0x69, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73,
+ 0x2e, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65, 0x73, 0x74, 0x52, 0x09, 0x6d, 0x61, 0x6e, 0x69, 0x66,
+ 0x65, 0x73, 0x74, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x72, 0x65, 0x66, 0x18, 0x02, 0x20, 0x01, 0x28,
+ 0x09, 0x52, 0x03, 0x72, 0x65, 0x66, 0x1a, 0x3a, 0x0a, 0x08, 0x4d, 0x61, 0x6e, 0x69, 0x66, 0x65,
+ 0x73, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09,
+ 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67,
+ 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67,
+ 0x65, 0x73, 0x42, 0x34, 0x5a, 0x32, 0x67, 0x6f, 0x2e, 0x66, 0x75, 0x63, 0x68, 0x73, 0x69, 0x61,
+ 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x69, 0x6e, 0x66, 0x72, 0x61, 0x2f, 0x63, 0x6d, 0x64, 0x2f, 0x72,
+ 0x6f, 0x6c, 0x6c, 0x65, 0x72, 0x2d, 0x63, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74,
+ 0x6f, 0x72, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
@@ -502,14 +549,13 @@
var file_rollers_cfg_proto_msgTypes = make([]protoimpl.MessageInfo, 7)
var file_rollers_cfg_proto_goTypes = []interface{}{
- (*Config)(nil), // 0: Config
- (*Roller)(nil), // 1: Roller
- (*GitSubmodule)(nil), // 2: GitSubmodule
- (*CIPDEnsureFile)(nil), // 3: CIPDEnsureFile
- (*JiriProject)(nil), // 4: JiriProject
- (*JiriPackages)(nil), // 5: JiriPackages
- nil, // 6: JiriPackages.PackagesByManifestEntry
- (*structpb.ListValue)(nil), // 7: google.protobuf.ListValue
+ (*Config)(nil), // 0: Config
+ (*Roller)(nil), // 1: Roller
+ (*GitSubmodule)(nil), // 2: GitSubmodule
+ (*CIPDEnsureFile)(nil), // 3: CIPDEnsureFile
+ (*JiriProject)(nil), // 4: JiriProject
+ (*JiriPackages)(nil), // 5: JiriPackages
+ (*JiriPackages_Manifest)(nil), // 6: JiriPackages.Manifest
}
var file_rollers_cfg_proto_depIdxs = []int32{
1, // 0: Config.rollers:type_name -> Roller
@@ -517,13 +563,12 @@
3, // 2: Roller.cipd_ensure_file:type_name -> CIPDEnsureFile
4, // 3: Roller.jiri_project:type_name -> JiriProject
5, // 4: Roller.jiri_packages:type_name -> JiriPackages
- 6, // 5: JiriPackages.packages_by_manifest:type_name -> JiriPackages.PackagesByManifestEntry
- 7, // 6: JiriPackages.PackagesByManifestEntry.value:type_name -> google.protobuf.ListValue
- 7, // [7:7] is the sub-list for method output_type
- 7, // [7:7] is the sub-list for method input_type
- 7, // [7:7] is the sub-list for extension type_name
- 7, // [7:7] is the sub-list for extension extendee
- 0, // [0:7] is the sub-list for field type_name
+ 6, // 5: JiriPackages.manifests:type_name -> JiriPackages.Manifest
+ 6, // [6:6] is the sub-list for method output_type
+ 6, // [6:6] is the sub-list for method input_type
+ 6, // [6:6] is the sub-list for extension type_name
+ 6, // [6:6] is the sub-list for extension extendee
+ 0, // [0:6] is the sub-list for field type_name
}
func init() { file_rollers_cfg_proto_init() }
@@ -604,6 +649,18 @@
return nil
}
}
+ file_rollers_cfg_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} {
+ switch v := v.(*JiriPackages_Manifest); i {
+ case 0:
+ return &v.state
+ case 1:
+ return &v.sizeCache
+ case 2:
+ return &v.unknownFields
+ default:
+ return nil
+ }
+ }
}
file_rollers_cfg_proto_msgTypes[1].OneofWrappers = []interface{}{
(*Roller_Submodule)(nil),
diff --git a/cmd/roller-configurator/proto/rollers_cfg.proto b/cmd/roller-configurator/proto/rollers_cfg.proto
index a23477f..24e9672 100644
--- a/cmd/roller-configurator/proto/rollers_cfg.proto
+++ b/cmd/roller-configurator/proto/rollers_cfg.proto
@@ -4,8 +4,6 @@
syntax = "proto3";
-import "google/protobuf/struct.proto";
-
option go_package = "go.fuchsia.dev/infra/cmd/roller-configurator/proto";
message Config {
@@ -62,8 +60,13 @@
// CIPD packages pinned in a Jiri manifest to roll.
message JiriPackages {
- // Mapping from Jiri manifest path to list of packages to roll within that
- map<string, google.protobuf.ListValue> packages_by_manifest = 1;
+ message Manifest {
+ string path = 1;
+ repeated string packages = 2;
+ }
+
+ // Jiri manifests containing packages to roll.
+ repeated Manifest manifests = 1;
// CIPD ref to track. Default is "latest".
string ref = 2;
diff --git a/cmd/roller-configurator/proto/util.go b/cmd/roller-configurator/proto/util.go
new file mode 100644
index 0000000..9eca643
--- /dev/null
+++ b/cmd/roller-configurator/proto/util.go
@@ -0,0 +1,29 @@
+// Copyright 2023 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package proto
+
+import (
+ "encoding/json"
+
+ "google.golang.org/protobuf/encoding/protojson"
+ "google.golang.org/protobuf/proto"
+)
+
+// ProtoToMap converts a proto message to a Go map.
+func ProtoToMap(m proto.Message) (map[string]any, error) {
+ opts := protojson.MarshalOptions{
+ UseProtoNames: true,
+ EmitUnpopulated: true,
+ }
+ b, err := opts.Marshal(m)
+ if err != nil {
+ return nil, err
+ }
+ res := make(map[string]any)
+ if err := json.Unmarshal(b, &res); err != nil {
+ return nil, err
+ }
+ return res, nil
+}
diff --git a/cmd/roller-configurator/proto/validators.go b/cmd/roller-configurator/proto/validators.go
index 790f0a0..26a1dad 100644
--- a/cmd/roller-configurator/proto/validators.go
+++ b/cmd/roller-configurator/proto/validators.go
@@ -6,11 +6,15 @@
import (
"context"
+ "errors"
"fmt"
+ "io/fs"
"os"
"os/exec"
"path/filepath"
"strings"
+
+ "go.fuchsia.dev/infra/cmd/roller-configurator/jiri"
)
func (s *GitSubmodule) url(ctx context.Context, repoRoot string) (string, error) {
@@ -59,21 +63,53 @@
}
func (p *JiriProject) Validate(ctx context.Context, repoRoot string) error {
- if _, err := os.Stat(filepath.Join(repoRoot, p.Manifest)); err != nil {
- return fmt.Errorf("no such file: %s", p.Manifest)
+ _, err := p.manifestEntry(repoRoot)
+ return err
+}
+
+func (p *JiriProject) manifestEntry(repoRoot string) (jiri.Project, error) {
+ manifest, err := jiri.LoadManifest(filepath.Join(repoRoot, p.Manifest))
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = fmt.Errorf("no such file: %s", p.Manifest)
+ }
+ return jiri.Project{}, err
}
- // TODO(olivernewman): Check that the project name refers to a valid entry
- // in the manifest.
- return nil
+ for _, proj := range manifest.Projects {
+ if proj.Name == p.Project {
+ return proj, nil
+ }
+ }
+ return jiri.Project{}, fmt.Errorf("no project %q in manifest %q", p.Project, p.Manifest)
}
func (p *JiriPackages) Validate(ctx context.Context, repoRoot string) error {
- for manifest := range p.PackagesByManifest {
- if _, err := os.Stat(filepath.Join(repoRoot, manifest)); err != nil {
- return fmt.Errorf("no such file: %s", manifest)
+ _, err := p.manifestEntries(repoRoot)
+ return err
+}
+
+func (p *JiriPackages) manifestEntries(repoRoot string) ([]jiri.Package, error) {
+ var entries []jiri.Package
+ for _, m := range p.Manifests {
+ manifest, err := jiri.LoadManifest(filepath.Join(repoRoot, m.Path))
+ if err != nil {
+ if errors.Is(err, fs.ErrNotExist) {
+ err = fmt.Errorf("no such file: %s", m.Path)
+ }
+ return nil, err
+ }
+ for _, pkg := range m.Packages {
+ var found bool
+ for _, entry := range manifest.Packages {
+ if entry.Name == pkg {
+ entries = append(entries, entry)
+ found = true
+ }
+ }
+ if !found {
+ return nil, fmt.Errorf("no package %q in manifest %q", pkg, m.Path)
+ }
}
}
- // TODO(olivernewman): Check that the package names refer to valid entries
- // in the manifest.
- return nil
+ return entries, nil
}
diff --git a/cmd/roller-configurator/resolve.go b/cmd/roller-configurator/resolve.go
index 6bfa281..1932405 100644
--- a/cmd/roller-configurator/resolve.go
+++ b/cmd/roller-configurator/resolve.go
@@ -87,11 +87,18 @@
log.Panicf("unknown to_roll type: %q", field.Name())
}
- resolved, err := toResolve.Resolve(ctx, repoRoot)
+ resolved, err := proto.ProtoToMap(roller)
if err != nil {
return err
}
-
+ delete(resolved, string(field.Name()))
+ addl, err := toResolve.Resolve(ctx, repoRoot)
+ if err != nil {
+ return err
+ }
+ for k, v := range addl {
+ resolved[k] = v
+ }
out = append(out, resolved)
}
enc := json.NewEncoder(output)
diff --git a/cmd/roller-configurator/resolve_test.go b/cmd/roller-configurator/resolve_test.go
index cb0f29a..68ed827 100644
--- a/cmd/roller-configurator/resolve_test.go
+++ b/cmd/roller-configurator/resolve_test.go
@@ -19,7 +19,23 @@
".gitmodules": `
[submodule "path/to/submodule"]
url = "https://example.com/lib"
- `,
+ `,
+ "path/to/jiri_manifest": `
+ <?xml version="1.0" encoding="UTF-8"?>
+ <manifest>
+ <projects>
+ <project name="foo-project"
+ remote="https://example.com/jiri-project"
+ anotherfield="bar"/>
+ </projects>
+ <packages>
+ <package name="foo/package1/${platform}"
+ anotherfield="baz"/>
+ <package name="foo/package2/${platform}"
+ anotherfield="baz"/>
+ </packages>
+ </manifest>
+ `,
}
func TestResolve(t *testing.T) {
@@ -42,13 +58,98 @@
submodule: {
path: "path/to/submodule"
}
+ schedule: "* * * * *"
}
]`,
want: `[
{
"type": "submodule",
"path": "path/to/submodule",
- "url": "https://example.com/lib"
+ "remote": "https://example.com/lib",
+ "schedule": "* * * * *",
+ "notify_emails": []
+ }
+ ]`,
+ },
+ {
+ name: "jiri project roller",
+ input: `rollers: [
+ {
+ jiri_project: {
+ manifest: "path/to/jiri_manifest"
+ project: "foo-project"
+ }
+ }
+ ]`,
+ want: `[
+ {
+ "type": "jiri_project",
+ "manifest": "path/to/jiri_manifest",
+ "project": "foo-project",
+ "remote": "https://example.com/jiri-project",
+ "remote_branch": "",
+ "schedule": "",
+ "notify_emails": []
+ }
+ ]`,
+ },
+ {
+ name: "jiri package roller",
+ input: `rollers: [
+ {
+ jiri_packages: {
+ ref: "foo"
+ manifests: [
+ {
+ path: "path/to/jiri_manifest"
+ packages: [
+ "foo/package1/${platform}"
+ ]
+ }
+ ]
+ }
+ },
+ {
+ jiri_packages: {
+ manifests: [
+ {
+ path: "path/to/jiri_manifest"
+ packages: [
+ "foo/package2/${platform}"
+ ]
+ }
+ ]
+ }
+ }
+ ]`,
+ want: `[
+ {
+ "type": "jiri_packages",
+ "manifests": [
+ {
+ "path": "path/to/jiri_manifest",
+ "packages": [
+ "foo/package1/${platform}"
+ ]
+ }
+ ],
+ "ref": "foo",
+ "schedule": "",
+ "notify_emails": []
+ },
+ {
+ "type": "jiri_packages",
+ "manifests": [
+ {
+ "path": "path/to/jiri_manifest",
+ "packages": [
+ "foo/package2/${platform}"
+ ]
+ }
+ ],
+ "ref": "latest",
+ "schedule": "",
+ "notify_emails": []
}
]`,
},
diff --git a/cmd/roller-configurator/validate_test.go b/cmd/roller-configurator/validate_test.go
index fa75322..b827581 100644
--- a/cmd/roller-configurator/validate_test.go
+++ b/cmd/roller-configurator/validate_test.go
@@ -11,7 +11,6 @@
"testing"
"go.fuchsia.dev/infra/cmd/roller-configurator/proto"
- "google.golang.org/protobuf/types/known/structpb"
)
var filesWithGitmodules = map[string]string{
@@ -21,6 +20,22 @@
`,
}
+var filesWithJiriManifest = map[string]string{
+ "path/to/jiri_manifest": `
+ <?xml version="1.0" encoding="UTF-8"?>
+ <manifest>
+ <projects>
+ <project name="foo-project"
+ remote="https://example.com/jiri-project"/>
+ </projects>
+ <packages>
+ <package name="package1"/>
+ <package name="package2"/>
+ </packages>
+ </manifest>
+ `,
+}
+
func TestValidate_valid(t *testing.T) {
t.Parallel()
@@ -70,8 +85,8 @@
Rollers: []*proto.Roller{
{
ToRoll: &proto.Roller_JiriProject{JiriProject: &proto.JiriProject{
- Manifest: "path/to/manifest",
- Project: "project-name",
+ Manifest: "path/to/jiri_manifest",
+ Project: "foo-project",
}},
NotifyEmails: []string{
"foo@example.com",
@@ -80,9 +95,7 @@
},
},
},
- files: map[string]string{
- "path/to/manifest": ``,
- },
+ files: filesWithJiriManifest,
},
{
name: "jiri packages",
@@ -90,11 +103,12 @@
Rollers: []*proto.Roller{
{
ToRoll: &proto.Roller_JiriPackages{JiriPackages: &proto.JiriPackages{
- PackagesByManifest: map[string]*structpb.ListValue{
- "path/to/manifest": {
- Values: []*structpb.Value{
- structpb.NewStringValue("package1"),
- structpb.NewStringValue("package2"),
+ Manifests: []*proto.JiriPackages_Manifest{
+ {
+ Path: "path/to/jiri_manifest",
+ Packages: []string{
+ "package1",
+ "package2",
},
},
},
@@ -102,9 +116,7 @@
},
},
},
- files: map[string]string{
- "path/to/manifest": ``,
- },
+ files: filesWithJiriManifest,
},
}
@@ -286,11 +298,12 @@
Rollers: []*proto.Roller{
{
ToRoll: &proto.Roller_JiriPackages{JiriPackages: &proto.JiriPackages{
- PackagesByManifest: map[string]*structpb.ListValue{
- "path/to/manifest": {
- Values: []*structpb.Value{
- structpb.NewStringValue("package1"),
- structpb.NewStringValue("package2"),
+ Manifests: []*proto.JiriPackages_Manifest{
+ {
+ Path: "path/to/manifest",
+ Packages: []string{
+ "package1",
+ "package2",
},
},
},
@@ -300,6 +313,43 @@
},
err: "no such file: path/to/manifest",
},
+ {
+ name: "invalid jiri project",
+ config: &proto.Config{
+ Rollers: []*proto.Roller{
+ {
+ ToRoll: &proto.Roller_JiriProject{JiriProject: &proto.JiriProject{
+ Manifest: "path/to/jiri_manifest",
+ Project: "not-a-project",
+ }},
+ },
+ },
+ },
+ files: filesWithJiriManifest,
+ err: `no project "not-a-project" in manifest "path/to/jiri_manifest"`,
+ },
+ {
+ name: "invalid jiri package",
+ config: &proto.Config{
+ Rollers: []*proto.Roller{
+ {
+ ToRoll: &proto.Roller_JiriPackages{JiriPackages: &proto.JiriPackages{
+ Manifests: []*proto.JiriPackages_Manifest{
+ {
+ Path: "path/to/jiri_manifest",
+ Packages: []string{
+ "bad-package1",
+ "bad-package2",
+ },
+ },
+ },
+ }},
+ },
+ },
+ },
+ files: filesWithJiriManifest,
+ err: `no package "bad-package1" in manifest "path/to/jiri_manifest"`,
+ },
}
for _, tc := range testCases {
