[upload_debug_symbols] speed up the uploading process
Previouse version of this tool create gcsObjectWriter for every file in
ids.txt, tried to write to it and let the precondition of DoesNotExists
preventing from uploding duplicated files. However, that approach is
time consuming.
In this newer approach, we first get a set of all the objects in the
GCS bucket, then only create objectWriter for and upload the files that
does not show up in the set.
Bug: IN-796
Change-Id: Ib0f093f2637a4bcfb24ca00be8dee0581ba9f77a
diff --git a/cmd/upload_debug_symbols/main.go b/cmd/upload_debug_symbols/main.go
index 74ccfac..2a6c1ef 100644
--- a/cmd/upload_debug_symbols/main.go
+++ b/cmd/upload_debug_symbols/main.go
@@ -20,6 +20,7 @@
"cloud.google.com/go/storage"
"fuchsia.googlesource.com/tools/elflib"
+ "google.golang.org/api/iterator"
)
const usage = `upload_debug_symbols [flags] bucket idsFilePath
@@ -91,12 +92,21 @@
return fmt.Errorf("failed to read %s with elflib: %v", idsFilePath, err)
}
+ objMap, err := client.getObjects(ctx)
+
+ if err != nil {
+ return err
+ }
+
for _, binaryFileRef := range binaries {
- err := client.uploadSingleFile(ctx, binaryFileRef.BuildID+".debug", binaryFileRef.Filepath)
- if err != nil {
- return err
+ fileURL := binaryFileRef.BuildID + ".debug"
+ if _, exist := objMap[fileURL]; !exist {
+ if err := client.uploadSingleFile(ctx, fileURL, binaryFileRef.Filepath); err != nil {
+ return err
+ }
}
}
+
return nil
}
@@ -121,9 +131,28 @@
return nil
}
+// getObjects returns a set of all binaries that currently exist in Cloud Storage.
+// TODO(IN-1050)
+func (client *gcsClient) getObjects(ctx context.Context) (map[string]bool, error) {
+ existingObjects := make(map[string]bool)
+ it := client.bkt.Objects(ctx, nil)
+ for {
+ objAttrs, err := it.Next()
+ if err == iterator.Done {
+ break
+ }
+ if err != nil {
+ return nil, err
+ }
+ existingObjects[objAttrs.Name] = true
+ }
+ return existingObjects, nil
+}
+
// GCSClient provide method to upload single file to gcs
type GCSClient interface {
uploadSingleFile(ctx context.Context, url string, filePath string) error
+ getObjects(ctx context.Context) (map[string]bool, error)
}
// gcsClient is the object that implement GCSClient
diff --git a/cmd/upload_debug_symbols/main_test.go b/cmd/upload_debug_symbols/main_test.go
index 05a17ca..11f6663 100644
--- a/cmd/upload_debug_symbols/main_test.go
+++ b/cmd/upload_debug_symbols/main_test.go
@@ -20,6 +20,10 @@
return nil
}
+func (client *FakeGCSClient) getObjects(ctx context.Context) (map[string]bool, error) {
+ return map[string]bool{"alreadyExistFile.debug": true}, nil
+}
+
func TestRunCommand(t *testing.T) {
tests := []struct {
// The name of this test case.
@@ -34,7 +38,7 @@
output map[string]string
}{
{
- name: "should upload the files in idx.txt",
+ name: "should upload files in idx.txt",
input: "01634b09 /path/to/binaryA.elf\n" +
"02298167 /path/to/binaryB\n" +
"025abbbc /path/to/binaryC.so",
@@ -44,11 +48,23 @@
"025abbbc.debug": "/path/to/binaryC.so",
},
},
- // {
- // name: "should upload nothing if nothing in ids.txt",
- // input: "",
- // output: map[string]string{},
- // },
+ {
+ name: "should not upload files already in cloud",
+ input: "alreadyExistFile /path/to/binaryA.elf\n" +
+ "01634b09 /path/to/binaryA.elf\n" +
+ "02298167 /path/to/binaryB\n" +
+ "025abbbc /path/to/binaryC.so",
+ output: map[string]string{
+ "01634b09.debug": "/path/to/binaryA.elf",
+ "02298167.debug": "/path/to/binaryB",
+ "025abbbc.debug": "/path/to/binaryC.so",
+ },
+ },
+ {
+ name: "should upload nothing if nothing in ids.txt",
+ input: "",
+ output: map[string]string{},
+ },
}
for _, tt := range tests {
