[amber] retry blob fetches on header timeouts
Observed in the wild header timeouts due to improperly torn down TCP sockets.
We retry up to twice under these conditions before passing the errors up the
stack.
Bug: PKG-691 #comment
Bug: DNO-483 #comment
Change-Id: I7e7e5042012d0be231ac28f4da1d541c239801ce
diff --git a/garnet/go/src/amber/source/source.go b/garnet/go/src/amber/source/source.go
index 40d3be7..e33bacb 100644
--- a/garnet/go/src/amber/source/source.go
+++ b/garnet/go/src/amber/source/source.go
@@ -694,9 +694,24 @@
}
defer dst.Close()
- resp, err := f.requestBlob(blob)
- if err != nil {
- return err
+ var resp *http.Response
+
+ for i := 0; i < 2; i++ {
+ resp, err = f.requestBlob(blob)
+ // If we get an error that is temporary and is a timeout (most typically this
+ // is a header timeout due to a tcp connection that was improperly torn down),
+ // then we'll attempt the make a fresh request.
+ if e, ok := err.(interface {
+ Temporary() bool
+ Timeout() bool
+ }); ok && e.Temporary() && e.Timeout() {
+ f.closeIdleConnections()
+ continue
+ }
+ // Otherwise the error is of a permanent kind, so it's time to bail
+ if err != nil {
+ return err
+ }
}
defer resp.Body.Close()
