zircon/scripts/download-prebuilt

#!/usr/bin/env bash

# Copyright 2018 The Fuchsia Authors
#
# Use of this source code is governed by a MIT-style
# license that can be found in the LICENSE file or at
# https://opensource.org/licenses/MIT

readonly SCRIPT_DIR="$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"
readonly ZIRCON_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
readonly PREBUILTS_DIR="$(cd "${ZIRCON_ROOT}/prebuilt" && pwd)"
readonly DOWNLOAD_DIR="${PREBUILTS_DIR}/downloads"
readonly ENSURE_FILE="${PREBUILTS_DIR}/zircon.ensure"
readonly VERSIONS_FILE="${PREBUILTS_DIR}/zircon.versions"
readonly URL_PREFIX="https://chrome-infra-packages.appspot.com/dl/fuchsia"

# This script assumes that ENSURE_FILE and VERSIONS_FILE match up.
# `cipd ensure` checks this and `cips ensure-file-resolve` ensures it.
# When a `cipd` binary is available, ENSURE_FILE controls the downloads
# and points `cipd` at VERSIONS_FILE for resolved pinned versions.
# Otherwise, VERSIONS_FILE controls the downloads directly.  In both
# cases we write $DOWNLOAD_DIR/$PACKAGE.stamp files with the versions
# from VERSIONS_FILE on faith that that's what we just unpacked.

set -o pipefail

cipd_ok=true
case "$#:$1" in
0:)
  mode=update
  ;;
1:--verify)
  mode=verify
  ;;
1:--list)
  mode=list
  ;;
1:--resolve)
  mode=resolve
  ;;
1:--no-cipd)
  mode=update
  cipd_ok=false
  ;;
*)
  echo >&2 "Usage: $0 [--verify|--list|--no-cipd]"
  exit 1
  ;;
esac
readonly cipd_ok

case "$(uname)-$(uname -m)" in
Darwin-x86_64)
  PLATFORM=mac-amd64
  ;;
Linux-x86_64)
  PLATFORM=linux-amd64
  ;;
Linux-aarch64)
  PLATFORM=linux-arm64
  ;;
*)
  echo 'Unknown operating system.'
  exit 1
  ;;
esac
readonly PLATFORM

update_stamp() {
  local -r package="$1" version="$2" download_file="$3"
  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
  mkdir -p "$(dirname "$stamp")" && echo "$version" > "$stamp"
}

verify_stamp() {
  local verbose=false
  if [[ "$1" = "--verbose" ]]; then
    verbose=true
    shift
  fi
  local -r package="$1" version="$2" download_file="$3"
  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
  local stamp_version
  if [[ -r "$stamp" ]]; then
    stamp_version="$(< "$stamp")"
  else
    stamp_version="missing"
  fi
  if [[ "$stamp_version" = "$version" ]]; then
    return 0
  fi
  if $verbose; then
    echo "WARNING: unpacked $package $stamp_version != current $version"
  fi
  return 1
}

list_package() {
  local -r package="$1" version="$2" download_file="$3" single_file="$4"
  local -r package_suffix="$5"
  local -r stamp="${DOWNLOAD_DIR}/${download_file%.*}.stamp"
  local stamp_version
  if [[ -r "$stamp" ]]; then
    stamp_version="$(< "$stamp")"
  else
    stamp_version="missing"
  fi
  local installed="${stamp_version:-missing}"
  if [[ "$installed" = "$version" ]]; then
    installed="current"
  fi
  echo "$package$package_suffix" "installed=$installed" "current=$version"
}

verify_file() {
  local -r file="$1"
  local -r sum="$2"
  if [[ "${#sum}" = "40" ]]; then
    shasum --binary --check --status <<EOF
$sum *$file
EOF
  elif [[ "${#sum}" = "44" ]]; then
    digest="$(echo $sum | tr '_-' '/+' | base64 --decode | xxd -p -c 64)"
    shasum -a 256 --binary --check --status <<EOF
${digest:0:64} *$file
EOF
  else
    echo >&2 "$0: unknown digest type for file ${file}: ${sum}"
    return 1
  fi
}

download_package() {
  local -r package="$1" version="$2" download_file="$3" single_file="$4"
  local -r package_suffix="$5"
  local -r url="${URL_PREFIX}/${package}${package_suffix}/+/${version}"
  local -r download_file_with_dir="${DOWNLOAD_DIR}/${download_file}"

  # If the stamp file says it's already in place, do nothing more.
  verify_stamp "$package" "$version" "$download_file" && return

  rm -f -- "$download_file_with_dir"
  echo "Downloading $url"
  curl --progress-bar -continue-at=- --location \
       --create-dirs --output "$download_file_with_dir" "$url" || return

  verify_file "$download_file_with_dir" "$version" || {
    echo >&2 "*** VERIFICATION ERROR ***"
    echo >&2 "*** VERIFICATION ERROR *** $download_file from $url"
    echo >&2 "*** VERIFICATION ERROR *** Not using the file!"
    return 1
  }

  echo "Unpacking $download_file"
  if $single_file; then
    # The archive contains .cipd* metadata files and a single real file
    # whose name is the same as the basename of the package.
    unzip -q -o -d "$DOWNLOAD_DIR" "$download_file_with_dir" \
          "${package##*/}" || return
  else
    local -r dir="${download_file_with_dir%.zip}"
    rm -rf -- "$dir"
    unzip -q -d "$dir" "$download_file_with_dir" || return
  fi

  update_stamp "$package" "$version" "$download_file"
}

for_each_package() {
  local package tag version download_file single_file package_suffix status=0
  local line next=package
  while read line; do
    case "$line" in
    ''|\#*)
      continue
      ;;
    esac
    eval $next=\$line
    case $next in
    package)
      next=tag
      continue
      ;;
    tag)
      next=version
      continue
      ;;
    version)
      next=package
      ;;
    esac
    # Now we've seen all three lines: package, tag, version.
    case "$package" in
    fuchsia/*)
      package="${package#fuchsia/}"
      ;;
    *)
      continue
      ;;
    esac
    case "$package" in
    tools/*/${PLATFORM})
      # These are standalone executables contained in a .zip file.
      package="${package%/*}"
      download_file="${package#tools/}.zip"
      single_file=true
      package_suffix="/${PLATFORM}"
      ;;
    sysroot/*)
      # These are the same for every host platform.
      # The $os-$cpu gets rewritten from CIPD names (amd64) to GN names (x64).
      download_file="${package/%amd64/x64}.zip"
      single_file=false
      package_suffix=''
      ;;
    */${PLATFORM})
      package="${package%/*}"
      # Subdirectories are packed in .zip files.
      if [[ "$package" == third_party/* ]]; then
        download_file="${package#*/}.zip"
      else
        download_file="${package}.zip"
      fi
      single_file=false
      package_suffix="/${PLATFORM}"
      ;;
    firmware/*)
      # These are the same for every host platform.
      download_file="${package}.zip"
      single_file=false
      package_suffix=''
      ;;
    *)
      # Skip packages for other platforms.
      continue
      ;;
    esac
    "$@" "$package" "$version" "$download_file" $single_file "$package_suffix" ||
      status=$?
  done
  return $status
}

find_cipd() {
  # If the Zircon checkout is part of a jiri checkout that includes
  # //buildtools, then find cipd there.  Otherwise, if cipd is in
  # the PATH, take it from there.
  type -p "${ZIRCON_ROOT}/../buildtools/cipd" || type -p cipd
}

run_cipd() {
  local -r CIPD="$1" internal_access="$2" command="$3"
  shift 3
  local -r cipd_args=("$@")

  local -a ensure_files=("${PREBUILTS_DIR}/zircon.ensure")
  if $internal_access; then
    ensure_files+=("${PREBUILTS_DIR}/zircon_internal.ensure")
  fi

  # The $ResolvedVersions file name is taken to be relative to the directory
  # containing the -ensure-file argument.  But since that's - to pipe in the
  # combined file, make sure CIPD runs with its current directory being the
  # one where $VERSIONS_FILE is found.
  (sed '/^\$/!d' "${ensure_files[@]}" && sed '/^\$/d' "${ensure_files[@]}") |
    (cd "$PREBUILTS_DIR" &&
     "$CIPD" "$command" -ensure-file - -log-level warning "${cipd_args[@]}")
  rc=$?
  if [[ $rc -ne 0 ]]; then
    echo >&2 "$0: $CIPD failed.  For direct downloads, remove cipd from PATH."
    return $rc
  fi
}

update_via_cipd() {
  run_cipd "$1" "$2" ensure -root "$DOWNLOAD_DIR"

  # Now update the .stamp files so that --verify will be happy.
  for_each_package update_stamp < "$VERSIONS_FILE"
}

write_sysroot_path() {
  local -r package="$1" version="$2"
  if [[ "$package" = sysroot ]]; then
    echo "SYSROOT_${PLATFORM}_PATH = \$(LKMAKEROOT)/prebuilt/downloads/sysroot"
  fi
}

write_config() {
  local -r internal_access="$1"
  local -r config_gni="${PREBUILTS_DIR}/config.gni"
  rm -f -- "$config_gni"
  echo > "$config_gni" "# Generated by $0.  DO NOT EDIT!
have_firmware = $internal_access"
}

update() {
  local CIPD
  local internal_access=false
  if $cipd_ok && CIPD="$(find_cipd)"; then
    # We have CIPD, so use it.
    if [[ "$("$CIPD" ls fuchsia_internal | tail -1)" != "No matching packages." ]]; then
      internal_access=true
    fi
    update_via_cipd "$CIPD" "$internal_access" || return
  else
    # We don't have CIPD, so don't use it.
    for_each_package download_package < "$VERSIONS_FILE" || return
  fi
  write_config "$internal_access"
}

verify() {
  for_each_package verify_stamp --verbose < "$VERSIONS_FILE"
}

list() {
  for_each_package list_package < "$VERSIONS_FILE"
}

resolve() {
  run_cipd "$(find_cipd)" true ensure-file-resolve
}

$mode