vendor/go.chromium.org/luci/common/api/internal/gensupport/retry.go

// Copyright 2021 Google LLC.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gensupport

import (
	"errors"
	"io"
	"net"
	"strings"
	"time"

	"github.com/googleapis/gax-go/v2"
	"google.golang.org/api/googleapi"
)

// Backoff is an interface around gax.Backoff's Pause method, allowing tests to provide their
// own implementation.
type Backoff interface {
	Pause() time.Duration
}

// These are declared as global variables so that tests can overwrite them.
var (
	// Default per-chunk deadline for resumable uploads.
	defaultRetryDeadline = 32 * time.Second
	// Default backoff timer.
	backoff = func() Backoff {
		return &gax.Backoff{Initial: 100 * time.Millisecond}
	}
	// syscallRetryable is a platform-specific hook, specified in retryable_linux.go
	syscallRetryable func(error) bool = func(err error) bool { return false }
)

const (
	// statusTooManyRequests is returned by the storage API if the
	// per-project limits have been temporarily exceeded. The request
	// should be retried.
	// https://cloud.google.com/storage/docs/json_api/v1/status-codes#standardcodes
	statusTooManyRequests = 429

	// statusRequestTimeout is returned by the storage API if the
	// upload connection was broken. The request should be retried.
	statusRequestTimeout = 408
)

// shouldRetry indicates whether an error is retryable for the purposes of this
// package, unless a ShouldRetry func is specified by the RetryConfig instead.
// It follows guidance from
// https://cloud.google.com/storage/docs/exponential-backoff .
func shouldRetry(status int, err error) bool {
	if 500 <= status && status <= 599 {
		return true
	}
	if status == statusTooManyRequests || status == statusRequestTimeout {
		return true
	}
	if err == io.ErrUnexpectedEOF {
		return true
	}
	// Transient network errors should be retried.
	if syscallRetryable(err) {
		return true
	}
	if err, ok := err.(interface{ Temporary() bool }); ok {
		if err.Temporary() {
			return true
		}
	}
	var opErr *net.OpError
	if errors.As(err, &opErr) {
		if strings.Contains(opErr.Error(), "use of closed network connection") {
			// TODO: check against net.ErrClosed (go 1.16+) instead of string
			return true
		}
	}

	// If Go 1.13 error unwrapping is available, use this to examine wrapped
	// errors.
	if err, ok := err.(interface{ Unwrap() error }); ok {
		return shouldRetry(status, err.Unwrap())
	}
	return false
}

// RetryConfig allows configuration of backoff timing and retryable errors.
type RetryConfig struct {
	Backoff     *gax.Backoff
	ShouldRetry func(err error) bool
}

// Get a new backoff object based on the configured values.
func (r *RetryConfig) backoff() Backoff {
	if r == nil || r.Backoff == nil {
		return backoff()
	}
	return &gax.Backoff{
		Initial:    r.Backoff.Initial,
		Max:        r.Backoff.Max,
		Multiplier: r.Backoff.Multiplier,
	}
}

// This is kind of hacky; it is necessary because ShouldRetry expects to
// handle HTTP errors via googleapi.Error, but the error has not yet been
// wrapped with a googleapi.Error at this layer, and the ErrorFunc type
// in the manual layer does not pass in a status explicitly as it does
// here. So, we must wrap error status codes in a googleapi.Error so that
// ShouldRetry can parse this correctly.
func (r *RetryConfig) errorFunc() func(status int, err error) bool {
	if r == nil || r.ShouldRetry == nil {
		return shouldRetry
	}
	return func(status int, err error) bool {
		if status >= 400 {
			return r.ShouldRetry(&googleapi.Error{Code: status})
		}
		return r.ShouldRetry(err)
	}
}