storage/invoke.go

// Copyright 2014 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package storage

import (
	"context"
	"errors"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"strings"

	"cloud.google.com/go/internal"
	"cloud.google.com/go/internal/version"
	sinternal "cloud.google.com/go/storage/internal"
	"github.com/google/uuid"
	gax "github.com/googleapis/gax-go/v2"
	"google.golang.org/api/googleapi"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

var defaultRetry *retryConfig = &retryConfig{}
var xGoogDefaultHeader = fmt.Sprintf("gl-go/%s gccl/%s", version.Go(), sinternal.Version)

// run determines whether a retry is necessary based on the config and
// idempotency information. It then calls the function with or without retries
// as appropriate, using the configured settings.
func run(ctx context.Context, call func() error, retry *retryConfig, isIdempotent bool, setHeader func(string, int)) error {
	attempts := 1
	invocationID := uuid.New().String()

	if retry == nil {
		retry = defaultRetry
	}
	if (retry.policy == RetryIdempotent && !isIdempotent) || retry.policy == RetryNever {
		setHeader(invocationID, attempts)
		return call()
	}
	bo := gax.Backoff{}
	if retry.backoff != nil {
		bo.Multiplier = retry.backoff.Multiplier
		bo.Initial = retry.backoff.Initial
		bo.Max = retry.backoff.Max
	}
	var errorFunc func(err error) bool = shouldRetry
	if retry.shouldRetry != nil {
		errorFunc = retry.shouldRetry
	}

	return internal.Retry(ctx, bo, func() (stop bool, err error) {
		setHeader(invocationID, attempts)
		err = call()
		attempts++
		return !errorFunc(err), err
	})
}

func setRetryHeaderHTTP(req interface{ Header() http.Header }) func(string, int) {
	return func(invocationID string, attempts int) {
		if req == nil {
			return
		}
		header := req.Header()
		invocationHeader := fmt.Sprintf("gccl-invocation-id/%v gccl-attempt-count/%v", invocationID, attempts)
		xGoogHeader := strings.Join([]string{invocationHeader, xGoogDefaultHeader}, " ")
		header.Set("x-goog-api-client", xGoogHeader)
	}
}

// TODO: Implement method setting header via context for gRPC
func setRetryHeaderGRPC(_ context.Context) func(string, int) {
	return func(_ string, _ int) {
		return
	}
}

func shouldRetry(err error) bool {
	if err == nil {
		return false
	}
	if errors.Is(err, io.ErrUnexpectedEOF) {
		return true
	}

	switch e := err.(type) {
	case *net.OpError:
		if strings.Contains(e.Error(), "use of closed network connection") {
			// TODO: check against net.ErrClosed (go 1.16+) instead of string
			return true
		}
	case *googleapi.Error:
		// Retry on 408, 429, and 5xx, according to
		// https://cloud.google.com/storage/docs/exponential-backoff.
		return e.Code == 408 || e.Code == 429 || (e.Code >= 500 && e.Code < 600)
	case *url.Error:
		// Retry socket-level errors ECONNREFUSED and ECONNRESET (from syscall).
		// Unfortunately the error type is unexported, so we resort to string
		// matching.
		retriable := []string{"connection refused", "connection reset"}
		for _, s := range retriable {
			if strings.Contains(e.Error(), s) {
				return true
			}
		}
	case interface{ Temporary() bool }:
		if e.Temporary() {
			return true
		}
	}
	// HTTP 429, 502, 503, and 504 all map to gRPC UNAVAILABLE per
	// https://grpc.github.io/grpc/core/md_doc_http-grpc-status-mapping.html.
	//
	// This is only necessary for the experimental gRPC-based media operations.
	if st, ok := status.FromError(err); ok && st.Code() == codes.Unavailable {
		return true
	}
	// Unwrap is only supported in go1.13.x+
	if e, ok := err.(interface{ Unwrap() error }); ok {
		return shouldRetry(e.Unwrap())
	}
	return false
}