[artifacts] Refactor to enforce a new GCS layout
We need to use this package to upload test logs from the
testrunner, and to locate them when uploading data to
ResultStore. This change updates the artifacts package
to adhere to a specific layout, so that all CloudStorage
clients can consume this package (through some CLI or as
source) and write or read to and from Cloud Storage in
the exact same way. We will update our infra recipes to
use this library to upload to GCS.
IN-888 #comment
TEST=ran the commands in artifacts/main.go by hand
Change-Id: If9b9e5443d5eebb733c295104cdab52d3d34ad09
diff --git a/artifacts/artifacts.go b/artifacts/artifacts.go
index 995e178..ee43f21 100644
--- a/artifacts/artifacts.go
+++ b/artifacts/artifacts.go
@@ -6,100 +6,31 @@
import (
"context"
- "io"
"strings"
"cloud.google.com/go/storage"
-
- "google.golang.org/api/iterator"
)
-// ArtifactsClient provides access to Fuchsia build artifacts.
-type ArtifactsClient struct {
+// Client provides access to the artifacts produced by Fuchsia CI tasks.
+type Client struct {
client *storage.Client
}
-// NewClient creates a new ArtifactsClient.
-func NewClient(ctx context.Context) (*ArtifactsClient, error) {
+// NewClient creates a new Client.
+func NewClient(ctx context.Context) (*Client, error) {
client, err := storage.NewClient(ctx)
if err != nil {
return nil, err
}
- return &ArtifactsClient{client: client}, nil
+ return &Client{client: client}, nil
}
-// List lists all objects in the artifact directory for the given build. bucket is the
-// Cloud Storage bucket for the given build.
-func (c *ArtifactsClient) List(ctx context.Context, bucket, build string) ([]string, error) {
- dir := c.openDir(ctx, bucket, build)
- return dir.list(ctx)
-}
-
-// Open returns a reader for an object in the artifact directory for the given build.
-// bucket is the Cloud Storage bucket for the given build.
-func (c *ArtifactsClient) Open(ctx context.Context, bucket, build, path string) (io.Reader, error) {
- dir := c.openDir(ctx, bucket, build)
- return dir.open(ctx, path)
-}
-
-// Create returns a storage.Writer for an object in the artifact directory. bucket is the
-// Cloud Storage bucket for the given build. build is the string Buildbucket build ID.
-// path is the object path relative to the root of the build artifact directory.
-func (c *ArtifactsClient) Create(ctx context.Context, bucket, build, path string) *storage.Writer {
- dir := c.openDir(ctx, bucket, build)
- return dir.create(ctx, path)
-}
-
-// openDir returns a Handle to a build's artifact directory within some bucket.
-func (c *ArtifactsClient) openDir(ctx context.Context, bucket, build string) *directory {
- handle := c.client.Bucket(bucket)
- return &directory{bucket: handle, build: build}
-}
-
-// directory is used to read from a Fuchsia build's Cloud Storage artifact "directory",
-// which is expected to live in a bucket containing the following hierarchy, where
-// "build-N" represents a possible directory:
-//
-// <bucket>/
-// "builds"/
-// <build-1>/
-// <build-2>/
-// ...
-type directory struct {
- bucket *storage.BucketHandle
- build string
-}
-
-// open returns an io.Reader for the given object in this directory.
-func (d *directory) open(ctx context.Context, object string) (io.Reader, error) {
- object = strings.Join([]string{"builds", d.build, object}, "/")
- return d.bucket.Object(object).NewReader(ctx)
-}
-
-// create returns a storage.Writer for the given object in this directory.
-func (d *directory) create(ctx context.Context, object string) *storage.Writer {
- object = strings.Join([]string{"builds", d.build, object}, "/")
- return d.bucket.Object(object).NewWriter(ctx)
-}
-
-// List lists all of the objects in this directory.
-func (d *directory) list(ctx context.Context) ([]string, error) {
- prefix := strings.Join([]string{"builds", d.build}, "/")
- iter := d.bucket.Objects(ctx, &storage.Query{
- Prefix: prefix,
- })
-
- var items []string
- for {
- attrs, err := iter.Next()
- if err == iterator.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- items = append(items, strings.TrimPrefix(attrs.Name, prefix+"/"))
- }
-
- return items, nil
+// GetBuildDir returns the BuildDirectory for the given build. bucket is the GCS bucket.
+// build is the BuildBucket build ID.
+func (c *Client) GetBuildDir(bucket, build string) *BuildDirectory {
+ bkt := c.client.Bucket(bucket)
+ return &BuildDirectory{&Directory{
+ bucket: bkt,
+ root: strings.Join([]string{"builds", build}, "/"),
+ }}
}
diff --git a/artifacts/directory.go b/artifacts/directory.go
new file mode 100644
index 0000000..090d9b7
--- /dev/null
+++ b/artifacts/directory.go
@@ -0,0 +1,88 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package artifacts
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "cloud.google.com/go/storage"
+ "google.golang.org/api/iterator"
+)
+
+// Directory is a handle to a Cloud Storage "directory". It provides a minimal
+// filesystem-like interface for a Cloud Storage object hierarchy where "/" is used as the
+// path separator. Any methods added to this struct are forward to other directory types.
+type Directory struct {
+ bucket *storage.BucketHandle
+ root string
+}
+
+// Object returns a handle to the given object within this directory. path is the path to
+// the object relative to this directory.
+func (d *Directory) Object(path string) *storage.ObjectHandle {
+ path = fmt.Sprintf("%s/%s", d.root, path)
+ return d.bucket.Object(path)
+}
+
+// List lists all of the objects in this directory.
+func (d *Directory) List(ctx context.Context) ([]string, error) {
+ prefix := strings.Join([]string{d.root}, "/")
+ iter := d.bucket.Objects(ctx, &storage.Query{
+ Prefix: prefix,
+ })
+
+ var items []string
+ for {
+ attrs, err := iter.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ items = append(items, strings.TrimPrefix(attrs.Name, prefix+"/"))
+ }
+
+ return items, nil
+}
+
+// CD returns a handle to some child Directory of this Directory. It is up to the caller
+// to ensure that child exists and is actually a directory.
+func (d *Directory) cd(child string) *Directory {
+ return &Directory{
+ bucket: d.bucket,
+ root: fmt.Sprintf("%s/%s", d.root, child),
+ }
+}
+
+// BuildDirectory represents a Fuchsia CI build's artifact directory. Refer to the
+// layout in doc.go for the layout of this directory. When amending the layout, prefer
+// adding methods to create new subdirectories on this type instead of calling Object()
+// with paths containing slashes. This encourages all clients of this package create
+// objects within this BuildDirectory using the same layout.
+type BuildDirectory struct {
+ *Directory
+}
+
+// Test returns the TestDirectory containing artifacts for a particular test. The name is
+// normalized according to normalizePathSegment.
+func (d BuildDirectory) Test(name string) *TestDirectory {
+ subdir := fmt.Sprintf("tests/%s", normalizePathSegment(name))
+ return &TestDirectory{d.Directory.cd(subdir)}
+}
+
+// TestDirectory contains artifacts for a particular test.
+type TestDirectory struct {
+ *Directory
+}
+
+// Env returns a Directory for objects relevant to an execution of this TestDirectory's
+// test in a particular environment. The name is normalized according to
+// normalizePathSegment.
+func (d TestDirectory) Env(name string) *Directory {
+ return d.Directory.cd(normalizePathSegment(name))
+}
diff --git a/artifacts/doc.go b/artifacts/doc.go
new file mode 100644
index 0000000..5071373
--- /dev/null
+++ b/artifacts/doc.go
@@ -0,0 +1,29 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// Package artifacts controls how files generated by Fuchsia's CI tasks are written to and
+// read from Cloud Storage. The directory layout is defined below:
+//
+// gs://BUCKET/
+// builds/
+// BUILD/
+// tests/
+// TEST/
+// ENVIRONMENT/
+//
+// BUCKET
+// The Cloud Storage bucket used to store CI artifacts. Multiple buckets can exist
+// exist using this scheme.
+// BUILD
+// Contains all artifacts for a build. The directory's name is the unique BuildBucket
+// build ID.
+// TEST
+// Contains all artifacts relevant to a particular test. The directory's name is the
+// canonical name of the test - which is usually a test "suite".
+// ENVIRONMENT
+// Contains all test artifacts relevant to a particular test environment. The
+// directory's name is the canonical name of the test environment. Tests will often
+// have multiple environment directories, because they often run in multiple
+// environments per build.
+package artifacts
diff --git a/artifacts/normalize.go b/artifacts/normalize.go
new file mode 100644
index 0000000..5c4d80a
--- /dev/null
+++ b/artifacts/normalize.go
@@ -0,0 +1,31 @@
+// Copyright 2019 The Fuchsia Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+package artifacts
+
+import (
+ "strings"
+)
+
+// Constants used to replace text in GCS object path segments.
+const (
+ // iFwdSlashSub is used in place of inner forward slashes.
+ iFwdSlashSub = "."
+
+ // iSpaceSub is used in place of inner \s characters.
+ iSpaceSub = "_"
+)
+
+// NormalizePathSegment rewrites a string so that it's safe to use as a GCS Object path.
+// This is useful for creating object paths from things like test names, which often
+// correspond to build target names and containing leading and inner forward slashes; Or
+// environment names, which often contain spaces or slashes for separating version
+// numbers.
+func normalizePathSegment(segment string) string {
+ segment = strings.TrimPrefix(segment, "//")
+ segment = strings.TrimSpace(segment)
+ segment = strings.Replace(segment, "/", iFwdSlashSub, -1)
+ segment = strings.Replace(segment, " ", iSpaceSub, -1)
+ return segment
+}
diff --git a/cmd/artifacts/copy.go b/cmd/artifacts/copy.go
index 3dd0b4c..ec7b1ef 100644
--- a/cmd/artifacts/copy.go
+++ b/cmd/artifacts/copy.go
@@ -84,13 +84,15 @@
return cmd.execute(ctx, buildsCli, artifactsCli)
}
-func (cmd *CopyCommand) execute(ctx context.Context, buildsCli buildsClient, artifactsCli artifactsClient) error {
+func (cmd *CopyCommand) execute(ctx context.Context, buildsCli buildsClient, artifactsCli *artifacts.Client) error {
bucket, err := getStorageBucket(ctx, buildsCli, cmd.build)
if err != nil {
return err
}
- input, err := artifactsCli.Open(ctx, bucket, cmd.build, cmd.source)
+ dir := artifactsCli.GetBuildDir(bucket, cmd.build)
+ obj := dir.Object(cmd.source)
+ input, err := obj.NewReader(ctx)
if err != nil {
return err
}
@@ -103,9 +105,3 @@
_, err = io.Copy(output, input)
return err
}
-
-// artifactsClient fetches artifacts produced by Fuchsia builds.
-type artifactsClient interface {
- List(ctx context.Context, bucket, build string) ([]string, error)
- Open(ctx context.Context, bucket, build, path string) (io.Reader, error)
-}
diff --git a/cmd/artifacts/list.go b/cmd/artifacts/list.go
index 314af9a..cc66fe4 100644
--- a/cmd/artifacts/list.go
+++ b/cmd/artifacts/list.go
@@ -66,13 +66,14 @@
return cmd.execute(ctx, buildsCli, artifactsCli)
}
-func (cmd *ListCommand) execute(ctx context.Context, buildsCli buildsClient, artifactsCli artifactsClient) error {
+func (cmd *ListCommand) execute(ctx context.Context, buildsCli buildsClient, artifactsCli *artifacts.Client) error {
bucket, err := getStorageBucket(ctx, buildsCli, cmd.build)
if err != nil {
return err
}
- items, err := artifactsCli.List(ctx, bucket, cmd.build)
+ dir := artifactsCli.GetBuildDir(bucket, cmd.build)
+ items, err := dir.List(ctx)
if err != nil {
return err
}
diff --git a/go.mod b/go.mod
index b4dffba..c3ecbbf 100644
--- a/go.mod
+++ b/go.mod
@@ -5,10 +5,12 @@
github.com/golang/mock v1.2.0
github.com/golang/protobuf v1.2.0
github.com/google/martian v2.1.0+incompatible // indirect
+ github.com/google/pprof v0.0.0-20190309163659-77426154d546 // indirect
github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315
github.com/google/uuid v1.1.0
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e // indirect
+ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 // indirect
github.com/jtolds/gls v4.2.1+incompatible // indirect
github.com/julienschmidt/httprouter v1.2.0 // indirect
github.com/maruel/subcommands v0.0.0-20181220013616-967e945be48b // indirect
@@ -18,6 +20,7 @@
github.com/texttheater/golang-levenshtein v0.0.0-20180516184445-d188e65d659e // indirect
go.chromium.org/luci v0.0.0-20181004001148-1bfb80352368
go.opencensus.io v0.19.0 // indirect
+ golang.org/x/arch v0.0.0-20190226203302-36aee92af9e8 // indirect
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
golang.org/x/net v0.0.0-20181217023233-e147a9138326
golang.org/x/sys v0.0.0-20181218192612-074acd46bca6
diff --git a/go.sum b/go.sum
index c676c8d..1a507c8 100644
--- a/go.sum
+++ b/go.sum
@@ -15,6 +15,8 @@
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20190309163659-77426154d546 h1:r3n/h1Zh7Wpk29Q/b+FdrNjDAmr28WaPcxlI0c4NaeA=
+github.com/google/pprof v0.0.0-20190309163659-77426154d546/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315 h1:WW91Hq2v0qDzoPME+TPD4En72+d2Ue3ZMKPYfwR9yBU=
github.com/google/subcommands v0.0.0-20181012225330-46f0354f6315/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk=
github.com/google/uuid v1.1.0 h1:Jf4mxPC/ziBnoPIdpQdPJ9OeiomAUHLvxmPRSPH9m4s=
@@ -24,6 +26,8 @@
github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/grpc-ecosystem/grpc-gateway v1.6.2/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
github.com/julienschmidt/httprouter v1.2.0 h1:TDTW5Yz1mjftljbcKqRcrYhd4XeOoI98t+9HbQbYf7g=
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
@@ -52,6 +56,8 @@
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
go.opencensus.io v0.19.0 h1:+jrnNy8MR4GZXvwF9PEuSyHxA4NaTf6601oNRwCSXq0=
go.opencensus.io v0.19.0/go.mod h1:AYeH0+ZxYyghG8diqaaIq/9P3VgCCt5GF2ldCY4dkFg=
+golang.org/x/arch v0.0.0-20190226203302-36aee92af9e8 h1:G3kY3WDPiChidkYzLqbniw7jg23paUtzceZorG6YAJw=
+golang.org/x/arch v0.0.0-20190226203302-36aee92af9e8/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -77,6 +83,7 @@
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e h1:XEcLGV2fKy3FrsoJVCkX+lMhqc9Suj7J5L/wldA1wu4=
golang.org/x/tools v0.0.0-20181219222714-6e267b5cc78e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
google.golang.org/api v0.0.0-20181220000619-583d854617af/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0=
