cmd/catapult/upload.go

// Copyright 2018 The Fuchsia Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file

package main

import (
	"context"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/google/subcommands"
	"go.chromium.org/luci/auth"
	"go.chromium.org/luci/auth/client/authcli"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/retry"
	"go.chromium.org/luci/common/retry/transient"
)

// FuchsiaUserAgent is used to set Fuchsia's User-Agent HTTP header for requests
// to the Catapult dashboard. The User-Agent header helps distinguish this
// uploader's entries from other entries in the Catapult logs. If left empty, Go
// inserts its own header "Go-http-client/XX" where XX is the version of the
// agent. The generic header makes our requests indistinguishable from other Go
// HTTP clients' requests.
const FuchsiaUserAgent = "Fuchsia-Uploader/1.0"

// UploadCommand uploads a data file to a URL when executed.
type UploadCommand struct {
	// The timeout for HTTP requests.
	timeout time.Duration

	// The URL to upload data to
	url string

	// LUCI flags used to parse command-line authentication options.
	authFlags authcli.Flags
}

func (*UploadCommand) Name() string {
	return "upload"
}

func (*UploadCommand) Usage() string {
	return "upload [options] json_file"
}

func (*UploadCommand) Synopsis() string {
	return "Uploads a JSON file to a URL"
}

func (cmd *UploadCommand) SetFlags(flags *flag.FlagSet) {
	// TODO(kjharland): For CI builders an --output flag that writes to an
	// output file would be a cleaner solution than writing to stdout and
	// having the caller stream or store the output. Add a flag for this
	// here.
	cmd.authFlags = authcli.Flags{}
	cmd.authFlags.Register(flags, auth.Options{})
	flags.DurationVar(&cmd.timeout, "timeout", 10*time.Second,
		"Request timeout duration string. e.g. 12s or 1m")
	flags.StringVar(&cmd.url, "url", "",
		"(required) The URL to upload to. Must have the format 'scheme://host[:port]'")
}

func (cmd *UploadCommand) Execute(ctx context.Context, f *flag.FlagSet, _ ...any) subcommands.ExitStatus {
	if f.NArg() < 1 {
		fmt.Fprintln(os.Stderr, "missing input file")
		return subcommands.ExitFailure
	}
	if f.NArg() != 1 {
		fmt.Fprintln(os.Stderr, "too many positional arguments")
		return subcommands.ExitFailure
	}
	if len(cmd.url) == 0 {
		fmt.Fprintln(os.Stderr, "url is required")
		return subcommands.ExitFailure
	}

	if cmd.timeout <= 0 {
		fmt.Fprintf(os.Stderr, "timeout must be positive. Got %s\n", cmd.timeout)
		return subcommands.ExitFailure
	}

	opts, err := cmd.authFlags.Options()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		return subcommands.ExitFailure
	}

	inputFile := f.Arg(0)
	if err := uploadData(ctx, cmd.url, inputFile, cmd.timeout, opts); err != nil {
		logging.Errorf(ctx, "%s", err)
		return subcommands.ExitFailure
	}

	return subcommands.ExitSuccess
}

// CreateUploadRequest creates an HTTP request to upload the given data to the
// given catapultURL.
func CreateUploadRequest(catapultURL string, data string) (*http.Request, error) {
	// TODO(kjharland): Use a custom Flag variable to verify the URL instead of
	// a string.
	u, err := url.ParseRequestURI(catapultURL)
	if err != nil {
		return nil, fmt.Errorf("failed to parse url: %w", err)
	}
	if len(u.RawQuery) > 0 || len(u.Fragment) > 0 || (len(u.Path) > 0 && u.Path != "/") {
		return nil, fmt.Errorf(
			"url must have format: scheme://host[:port]. Got %s",
			u.String())
	}

	u.Path = "add_histograms"

	// URL encode the request body. This is weird but the Catapult dashboard
	// will reject the request because it expects a body of the form:
	// `data=url-encoded-histogramset` instead of the more usual JSON body:
	// `{data: ...}`.
	params := url.Values{}
	params.Set("data", data)
	content := params.Encode()

	req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(content))
	if err != nil {
		return nil, err
	}

	req.Header.Set("User-Agent", FuchsiaUserAgent)
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	req.Header.Set("Content-Length", strconv.Itoa(len(content)))

	return req, nil
}

func uploadData(ctx context.Context, catapultURL string, filepath string, timeout time.Duration, authOpts auth.Options) error {
	requestBody, err := os.ReadFile(filepath)
	if err != nil {
		return err
	}

	// Verify that the input data is actual JSON.  Ideally we'd verify the
	// structure of the data also but this isn't always practical. (For
	// example, when parsing a dynamic schema such as Catapult's
	// HistogramSet).
	// TODO(kjharland): use json.Valid after fuchsia's go is updated to v1.10
	var ignored any
	if err := json.Unmarshal(requestBody, &ignored); err != nil {
		return fmt.Errorf("input is not valid JSON: %w", err)
	}

	// The LUCI authenticator used to authenticate the request. Silent login
	// prevents an interactive login session when running from CI bots.
	authenticator := auth.NewAuthenticator(ctx, auth.SilentLogin, authOpts)
	client, err := authenticator.Client()
	if err != nil {
		return err
	}
	client.Timeout = timeout

	retryPolicy := transient.Only(func() retry.Iterator {
		return &retry.ExponentialBackoff{
			Limited: retry.Limited{
				Delay:   5 * time.Second,
				Retries: 3,
			},
			Multiplier: 2,
		}
	})
	return retry.Retry(ctx, retryPolicy, func() error {
		// client.Do mutates the request object so we must create a new request
		// for every attempt.
		req, err := CreateUploadRequest(catapultURL, string(requestBody))
		if err != nil {
			return fmt.Errorf("create request: %w", err)
		}
		response, err := client.Do(req)
		if err != nil {
			return err
		}
		// We don't need to use the body, but we must still read and close it to
		// ensure it's safe to reuse the same client if we retry.
		defer response.Body.Close()
		body, err := io.ReadAll(response.Body)
		if err != nil {
			return err
		}
		if len(body) > 0 {
			logging.Debugf(ctx, "Response from Catapult: %s", body)
		}
		if response.StatusCode < 200 || response.StatusCode >= 300 {
			logging.Warningf(ctx, "Got error from Catapult: %s", response.Status)
			err = errors.New(response.Status)
			if response.StatusCode >= 500 {
				err = transient.Tag.Apply(err)
			}
			return err
		}
		return nil
	}, nil)
}