scripts/fetch-build-artifacts

#!/bin/bash
# Copyright 2018 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# NOTE: This tool must be able to be downloaded and run independently of the
# rest of this repo: the manual release processes that use it do not have a
# fuchsia checkout.

## usage: fetch-build-artifacts \
##            [-r] [-p <parent_dir>] [-o <out_dir>] [-h] \
##            <buildbucket_id>
##
## Downloads artifacts created by the specified build.
##
## <buildbucket_id> is a large number like '8938070794014050064', which you can
## find on the build results page. This tool accepts an optional leading 'b',
## letting you copy-and-paste the ID component from a URL like
## https://ci.chromium.org/p/fuchsia/builders/luci.fuchsia.ci/garnet-arm64-release-qemu_kvm/b8938011100592458048
##
## Prerequisites:
## - Install the 'gsutil' tool:
##   https://cloud.google.com/storage/docs/gsutil_install
## - Run 'gcloud init'. The default project does not matter, but if you need to
##   provide one you can use 'fuchsia-infra'.
##   - If you don't have gcloud installed, you can run 'gsutil config' instead.
## - The user must have permission to access the target artifacts in Google
##   Cloud Storage (GCS).
##
## Optional arguments:
##   -r    Indicates that the build ID points to a release build, whose
##         artifacts are stored in a special GCS bucket.
##   -p    The parent directory of the destination build artifact directory:
##         see -o. Defaults to "${HOME}/.cache/fuchsia/build-artifacts".
##         Mutually exclusive with -o.
##   -o    The directory to write the artifacts to. Defaults to
##         <parent_dir>/<buildbucket_id>. Mutually exclusive with -p.
##   -h    Prints this help message.

set -o errexit
set -o nounset
set -o pipefail

readonly DEFAULT_PARENT_DIR="${HOME}/.cache/fuchsia/build-artifacts"

function usage {
  # Prints lines from this file that begin with ##.
  sed -n -e 's/^## //p' -e 's/^##$//p' < "${BASH_SOURCE[0]}"
}

function main {
  if [[ -z "$(which gsutil)" ]]; then
    echo "ERROR: Please install gsutil and run 'gcloud init'."
    echo "See https://cloud.google.com/storage/docs/gsutil_install"
    return 1
  fi

  local parent_dir=''
  local out_dir=''
  local is_release=0
  while getopts "rp:o:h" arg; do
    case ${arg} in
      r)
        is_release=1
        ;;
      p)
        parent_dir="${OPTARG}"
        ;;
      o)
        out_dir="${OPTARG}"
        ;;
      *)
        usage
        return 1
        ;;
    esac
  done
  shift $((${OPTIND} - 1))

  if [[ "${parent_dir}" ]] && [[ "${out_dir}" ]]; then
    echo "ERROR: Cannot specify both -p and -o."
    usage
    return 1
  fi

  # Expect exactly one build ID.
  # TODO(dbort): We could support multiple build IDs if -o isn't specified.
  if [[ $# -ne 1 ]]; then
    echo "ERROR: Build ID missing."
    usage
    return 1
  fi
  local build_id="$1"

  # Make sure it's a number, but allow a leading 'b' since some LUCI URLs
  # tack it onto the build ID component.
  if [[ ! "${build_id}" =~ ^b?[0-9]+$ ]]; then
    echo "ERROR: Build ID '${build_id}' is not a number."
    usage
    return 1
  fi
  build_id="${build_id#b}"  # Remove leading 'b' if present.

  if [[ -z "${parent_dir}" ]]; then
    parent_dir="${DEFAULT_PARENT_DIR}"
  fi

  if [[ -z "${out_dir}" ]]; then
    out_dir="${parent_dir}/${build_id}"
  fi

  if (( is_release )); then
    local archive_bucket='fuchsia-release-archive'
  else
    local archive_bucket='fuchsia-archive'
  fi

  echo "Fetching build ${build_id} to ${out_dir} ..."
  mkdir -p "${out_dir}"

  local failed=0
  (
    set -x  # Prints the gsutil command
    # -m: Download files in parallel
    # -c: Continue downloading later files even if an earlier one fails
    # -L: Log file to use for resuming; also avoids re-downloading files
    gsutil -m cp \
      -c \
      -L "${out_dir}/.gsutil-cp.log" \
      -r \
      "gs://${archive_bucket}/builds/${build_id}/*" "${out_dir}"
  ) || failed=1

  if (( failed )); then
    if (( ! is_release )); then
      echo "NOTE: If you are trying to download a release build, remember"
      echo "      to specify the '-r' flag."
    fi
    return 1
  fi

  echo "Downloaded artifacts to ${out_dir}"
  echo "DONE"
}

main "$@"