test(httpreplay): fix TestIntegration_RecordAndReplay (#3096)
A bucket this test relied upon no longer exists. Refactored test
to not assume objects exist somewhere else. The test now uploads
and cleans-up gcs objects.
Fixes: #3015
diff --git a/httpreplay/httpreplay_test.go b/httpreplay/httpreplay_test.go
index 199911a..3b3252b 100644
--- a/httpreplay/httpreplay_test.go
+++ b/httpreplay/httpreplay_test.go
@@ -19,11 +19,13 @@
"context"
"encoding/json"
"fmt"
+ "io"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"os"
+ "path/filepath"
"testing"
"time"
@@ -33,6 +35,11 @@
"google.golang.org/api/option"
)
+const (
+ compressedFile = "httpreplay_compressed.txt"
+ uncompressedFile = "httpreplay_uncompressed.txt"
+)
+
func TestIntegration_RecordAndReplay(t *testing.T) {
httpreplay.DebugHeaders()
if testing.Short() {
@@ -45,6 +52,11 @@
t.Skip("Need project ID. See CONTRIBUTING.md for details.")
}
ctx := context.Background()
+ cleanup, err := setup(ctx)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer cleanup()
// Record.
initial := time.Now()
@@ -95,6 +107,49 @@
}
}
+func setup(ctx context.Context) (cleanup func(), err error) {
+ ts := testutil.TokenSource(ctx, storage.ScopeFullControl)
+ client, err := storage.NewClient(ctx, option.WithTokenSource(ts))
+ if err != nil {
+ return nil, err
+ }
+ bucket := testutil.ProjID()
+
+ // upload compressed object
+ f1, err := os.Open(filepath.Join("internal", "testdata", "compressed.txt"))
+ if err != nil {
+ return nil, err
+ }
+ defer f1.Close()
+ w := client.Bucket(bucket).Object(compressedFile).NewWriter(ctx)
+ w.ContentEncoding = "gzip"
+ w.ContentType = "text/plain"
+ if _, err = io.Copy(w, f1); err != nil {
+ return nil, err
+ }
+ if err := w.Close(); err != nil {
+ return nil, err
+ }
+ // upload uncompressed object
+ f2, err := os.Open(filepath.Join("internal", "testdata", "uncompressed.txt"))
+ if err != nil {
+ return nil, err
+ }
+ defer f2.Close()
+ w = client.Bucket(testutil.ProjID()).Object(uncompressedFile).NewWriter(ctx)
+ if _, err = io.Copy(w, f2); err != nil {
+ return nil, err
+ }
+ if err := w.Close(); err != nil {
+ return nil, err
+ }
+ return func() {
+ client.Bucket(bucket).Object(compressedFile).Delete(ctx)
+ client.Bucket(bucket).Object(uncompressedFile).Delete(ctx)
+ client.Close()
+ }, nil
+}
+
// TODO(jba): test errors
func run(t *testing.T, hc *http.Client) (*storage.BucketAttrs, []byte) {
@@ -133,15 +188,6 @@
}
func testReadCRC(t *testing.T, hc *http.Client, mode string) {
- const (
- // This is an uncompressed file.
- // See https://cloud.google.com/storage/docs/public-datasets/landsat
- uncompressedBucket = "gcp-public-data-landsat"
- uncompressedObject = "LC08/PRE/044/034/LC80440342016259LGN00/LC80440342016259LGN00_MTL.txt"
-
- gzippedBucket = "storage-library-test-bucket"
- gzippedObject = "gzipped-text.txt"
- )
ctx := context.Background()
client, err := storage.NewClient(ctx, option.WithHTTPClient(hc))
if err != nil {
@@ -149,8 +195,9 @@
}
defer client.Close()
- uncompressedObj := client.Bucket(uncompressedBucket).Object(uncompressedObject)
- gzippedObj := client.Bucket(gzippedBucket).Object(gzippedObject)
+ bucket := testutil.ProjID()
+ uncompressedObj := client.Bucket(bucket).Object(uncompressedFile)
+ gzippedObj := client.Bucket(bucket).Object(compressedFile)
for _, test := range []struct {
desc string
@@ -167,7 +214,7 @@
offset: 0,
length: -1,
readCompressed: false,
- wantLen: 7903,
+ wantLen: 179,
},
{
desc: "uncompressed, entire file, don't decompress",
@@ -175,15 +222,15 @@
offset: 0,
length: -1,
readCompressed: true,
- wantLen: 7903,
+ wantLen: 179,
},
{
desc: "uncompressed, suffix",
obj: uncompressedObj,
- offset: 3,
+ offset: 9,
length: -1,
readCompressed: false,
- wantLen: 7900,
+ wantLen: 170,
},
{
desc: "uncompressed, prefix",
@@ -204,7 +251,7 @@
offset: 0,
length: -1,
readCompressed: false,
- wantLen: 11,
+ wantLen: 179,
},
{
// When we read a gzipped file uncompressed, it's like reading a regular file:
@@ -214,7 +261,7 @@
offset: 0,
length: -1,
readCompressed: true,
- wantLen: 31,
+ wantLen: 128,
},
{
desc: "compressed, partial, read compressed",
diff --git a/httpreplay/internal/testdata/compressed.txt b/httpreplay/internal/testdata/compressed.txt
new file mode 100644
index 0000000..ebf78d1
--- /dev/null
+++ b/httpreplay/internal/testdata/compressed.txt
Binary files differ
diff --git a/httpreplay/internal/testdata/uncompressed.txt b/httpreplay/internal/testdata/uncompressed.txt
new file mode 100644
index 0000000..b2920fd
--- /dev/null
+++ b/httpreplay/internal/testdata/uncompressed.txt
@@ -0,0 +1,15 @@
+This is a
+
+test file that
+
+includes some
+
+uncompressed text.
+
+It's sister compressed
+
+file is compressed.txt.
+
+This is used to
+
+test httpreplay.
