[upload_debug_symbols] Don't enumerate bucket state.
GCS uploads already support no clobber options and will make individual
requests against the bucket to see if a file already exists, there is no
need to map existing files locally.
Change-Id: I175dacf25b7b308caacd69bcd5760a5dfde1962e
diff --git a/cmd/upload_debug_symbols/gcs_bucket.go b/cmd/upload_debug_symbols/gcs_bucket.go
index c2994e2..96b22fe 100644
--- a/cmd/upload_debug_symbols/gcs_bucket.go
+++ b/cmd/upload_debug_symbols/gcs_bucket.go
@@ -13,23 +13,11 @@
"cloud.google.com/go/storage"
"fuchsia.googlesource.com/tools/gcs"
"go.chromium.org/luci/auth"
- "google.golang.org/api/iterator"
)
-// GCSBucket provides access to a cloud storage bucket. GCSBucket only captures the items
-// that were in GCS at the time the GCSBucket was created. It also tracks any items that
-// are uploaded via upload().
+// GCSBucket provides access to a cloud storage bucket.
type GCSBucket struct {
- objects map[string]bool
- bkt *storage.BucketHandle
-}
-
-// previouslyContained returns true iff the given object is known to already exist in this
-// bucket. GCSBucket only captures initial state; The object may already exist in the
-// bucket when this method is invoked if some other process uploaded the object after this
-// GCSBucket was created.
-func (bkt *GCSBucket) previouslyContained(object string) bool {
- return bkt.objects[object]
+ bkt *storage.BucketHandle
}
func (bkt *GCSBucket) upload(ctx context.Context, object string, r io.Reader) error {
@@ -45,35 +33,17 @@
return fmt.Errorf("failed in close: %v", err)
}
}
- bkt.objects[object] = true
return nil
}
-// prepareGCSBucket captures the current state of the GCS bucket with the given name.
-func prepareGCSBucket(ctx context.Context, name string, opts auth.Options) (*GCSBucket, error) {
+// newGCSBucket returns a new GCSBucket object for the given bucket name.
+func newGCSBucket(ctx context.Context, name string, opts auth.Options) (*GCSBucket, error) {
client, err := gcs.NewClient(ctx, opts)
if err != nil {
return nil, fmt.Errorf("failed to create client: %v", err)
}
bkt := client.Bucket(gcsBucket)
- if _, err = bkt.Attrs(ctx); err != nil {
- return nil, fmt.Errorf("failed to fetch bucket attributes: %v", err)
- }
- existingObjects := make(map[string]bool)
- it := bkt.Objects(ctx, nil)
- for {
- objAttrs, err := it.Next()
- if err == iterator.Done {
- break
- }
- if err != nil {
- return nil, err
- }
- existingObjects[objAttrs.Name] = true
- }
-
return &GCSBucket{
- objects: existingObjects,
- bkt: bkt,
+ bkt: bkt,
}, nil
}
diff --git a/cmd/upload_debug_symbols/job.go b/cmd/upload_debug_symbols/job.go
index 03661cd..8f2555d 100644
--- a/cmd/upload_debug_symbols/job.go
+++ b/cmd/upload_debug_symbols/job.go
@@ -7,7 +7,6 @@
import (
"context"
"fmt"
- "log"
"os"
"fuchsia.googlesource.com/tools/elflib"
@@ -33,11 +32,6 @@
return fmt.Errorf("validation failed for %q: %v", j.bfr.Filepath, err)
}
object := j.bfr.BuildID + elflib.DebugFileSuffix
- bucket := j.dstBucket
- if bkt.previouslyContained(object) {
- log.Printf("skipping %q which already exists in %q", object, bucket)
- return nil
- }
filepath := j.bfr.Filepath
reader, err := os.Open(filepath)
if err != nil {
diff --git a/cmd/upload_debug_symbols/main.go b/cmd/upload_debug_symbols/main.go
index b28fe26..7f0629e 100644
--- a/cmd/upload_debug_symbols/main.go
+++ b/cmd/upload_debug_symbols/main.go
@@ -88,7 +88,7 @@
if err != nil {
return fmt.Errorf("failed to fetch GCS bucket information: %v", err)
}
- bkt, err := prepareGCSBucket(ctx, gcsBucket, opts)
+ bkt, err := newGCSBucket(ctx, gcsBucket, opts)
if err != nil {
return err
}
