bazel/update_mirror.sh

#!/bin/bash
# Copyright 2020 The gRPC Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Script to upload github archives for bazel dependencies to GCS, creating a reliable mirror link.
# Archives are copied to "grpc-bazel-mirror" GCS bucket (https://console.cloud.google.com/storage/browser/grpc-bazel-mirror?project=grpc-testing)
# and will by downloadable with the https://storage.googleapis.com/grpc-bazel-mirror/ prefix.
#
# This script should be run each time bazel dependencies are updated.

set -e

cd $(dirname $0)/..

# Create a temp directory to hold the versioned tarball,
# and clean it up when the script exits.
tmpdir="$(mktemp -d)"
success=0
failure=0
function cleanup {
  echo "stats: Attempted to upload $((success+failure)) files in total, ${success} succeeded, ${failure} failed."
  rm -rf "$tmpdir"
}
trap cleanup EXIT

function _upload() {
  local uri="$1"
  local dst_path="${uri}"

  if [[ "$dst_path" == https://sourceforge.net/*/download ]]; then
    dst_path="${dst_path%/download}"
  fi

  if gcloud storage objects list --stat --fetch-encrypted-object-hashes "gs://grpc-bazel-mirror/${uri}" > /dev/null; then
    echo "Skipping ${uri}"
    return 0
  fi

  echo "Downloading https://${uri}"
  curl -L --fail --output "${tmpdir}/archive" "https://${uri}"
  if [[ ! -s "${tmpdir}/archive" ]]; then
    echo "Failed to download https://${uri}: zero bytes returned"
    return 0
  fi

  echo "Uploading https://${uri} to https://storage.googleapis.com/grpc-bazel-mirror/${dst_path}"
  gcloud storage cp "${tmpdir}/archive" "gs://grpc-bazel-mirror/${dst_path}"

  rm -rf "${tmpdir}/archive"
}

# Wrapper of _upload() which handles errors and keep track of stats.
function upload() {
  local url="$1"
  local exit_code=0
  _upload "${url}" || exit_code=$?
  if [[ "$exit_code" != 0 ]]; then
    echo "Failed to upload url ${url}"
    failure=$((failure + 1))
  else
    success=$((success + 1))
  fi
  return 0
}

function upload_bzlmod_deps {
  tools/bazel mod show_repo --all_repos --output=streamed_jsonproto > ${tmpdir}/repos.ndjson || true
  python3 bazel/update_mirror_helper.py ${tmpdir}/repos.ndjson ${tmpdir}/repos.txt
  while read -r url; do
      case "$url" in
          *grpc-bazel-mirror*)
            echo "Skipping URL from mirror site: ${url}"
            ;;
          *github.com*)
            echo "Uploading archive from github.com: ${url}"
            upload "${url}"
            ;;
          *)
            echo "Uploading archive from non-github site: ${url}"
            upload "${url}"
            ;;
      esac
  done < "${tmpdir}/repos.txt"
}

# How to check that all mirror URLs work:
# 1. clean $HOME/.cache/bazel
# 2. bazel clean --expunge
# 3. bazel sync (failed downloads will print warnings)

# A specific link can be upload manually by running e.g.
# upload "github.com/google/boringssl/archive/1c2769383f027befac5b75b6cedd25daf3bf4dcf.tar.gz"

# bazel binaries used by the tools/bazel wrapper script
upload github.com/bazelbuild/bazel/releases/download/8.7.0/bazel-8.7.0-linux-arm64
upload github.com/bazelbuild/bazel/releases/download/8.7.0/bazel-8.7.0-linux-x86_64
upload github.com/bazelbuild/bazel/releases/download/8.7.0/bazel-8.7.0-darwin-x86_64
upload github.com/bazelbuild/bazel/releases/download/8.7.0/bazel-8.7.0-windows-x86_64.exe

# Collect the github archives to mirror from grpc_deps.bzl
# TODO(weizheyuan): Considier removing this block completely (since it's used by WORKSPACE only)
# and use bzlmod as the only source of truth.
echo "Updating Bazel dependency mirrors (WORKSPACE)"
grep -o '"https://github.com/[^"]*"' bazel/grpc_deps.bzl | sed 's/^"https:\/\///' | sed 's/"$//' | while read -r line ; do
    echo "Updating mirror for ${line}"
    upload "${line}"
done

echo "Updating Bazel dependency mirrors (Bzlmod)"
upload_bzlmod_deps

# Collect the github archives to mirror from build_autogenerated.yaml
echo "Updating CMake dependency mirrors"
grep -Eo '\s*- https://github.com/.+\.(tar\.gz|zip)' build_autogenerated.yaml | sed 's@^[[:space:]]*- https://@@' | while read -r line ; do
    echo "Updating mirror for ${line}"
    upload "${line}"
done