tools/devshell/contrib/update-rustc-third-party - fuchsia - Git at Google

 #!/bin/bash
 # Copyright 2018 The Fuchsia Authors. All rights reserved.
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.

 #### CATEGORY=Source tree
 ### updates rustc_library and rustc_binary third_party dependencies

 ## usage: fx update-rustc-third-party
 ## Updates third_party/rust_crates based on the contents of
 ## third_party/rust_crates/Cargo.toml
 ##
 ## See https://fuchsia.dev/fuchsia-src/development/languages/rust/third_party.md
 ## for more details.

 set -eo pipefail

 function preserve_files {
   local tmp=$1
   local filename=$2
   # Some crates can vendor dependnecies in them, so limit max depth to avoid
   # picking up files from nested dependencies.
   find . -maxdepth 2 -type f -name "$filename" -print0 | while read -d $'\0' file_to_preserve; do
     local dir=$(dirname "$file_to_preserve")
     local version=$(sed -n "s/^version = \"\(.*\)\"$/\\1/p" "$dir/Cargo.toml" | head -n 1)
     # + in directory names are replaced with -, so they are supported in Bazel
     # labels.
     version="${version/+/-}"
     if [[ $dir != *-"$version" ]]; then
       dir="$dir-$version"
     fi
     local temp_dir="$tmp/$dir"
     mkdir -p "$temp_dir"
     cp "$file_to_preserve" "$temp_dir"
   done
 }

 function latest_version_dir {
   local crate=$1
   local crate_name=$(echo "$crate" | sed -rn "s/^([a-zA-Z][a-zA-Z0-9_\\-]*)-[0-9]+\\.[0-9]+\\.[0-9]+.*$/\\1/p")
   local semver_regex="s/^$crate_name-([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*$)/\\1 \\2 \\3 \\4/p"
   local latest_version=$(echo "$crate" "$crate_name"-* | tr ' ' '\n' | sed -rn "$semver_regex" | sort -rn -k 1,1 -k 2,2 -k 3,3 -k 4,4 | sed -rn "s/^(.+) (.+) (.+) (.*$)/$crate_name-\\1.\\2.\\3\\4/p" | grep -A1 -F "$crate" | tail -n1)
   echo "$latest_version"
 }

 # This function looks for any vendored crates that are missing OWNERS files and
 # attempts to source one from the previous latest version of that crate.
 function propagate_owner_files {
   local owners_tmp=$1
   local vendor_dir=$2
   cd "$vendor_dir"
   for crate in *; do
     # vendor_dir can have generated Bazel build files in it, filter them out.
     # i.e. Only walk into subdirectories, which are vendored third-party crates.
     local path="${vendor_dir}/${crate}"
     if [[ -d "${path}" && ! -f "${path}/OWNERS" ]]; then
       cd "$owners_tmp"
       local latest_version="$(latest_version_dir $crate)"
       if [[ "$latest_version" != "$crate" ]]; then
         cp "$owners_tmp/$latest_version/OWNERS" "$vendor_dir/$crate"
       fi
     fi
   done
 }

 # This function looks for any vendored crates that are missing LICENSE files and
 # attempts to source one from the previous latest version of that crate.
 #
 # NOTE: This function is only intended for LICENSE files created by us, which
 # are always named LICENSE, so it skips copying if one does exist. This is OK
 # because we later rely on check_rust_licenses.py to check other possible
 # license file included by upstream.
 function propagate_license_files {
   local license_tmp=$1
   local vendor_dir=$2
   cd "$vendor_dir"
   for crate in *; do
     if [[ ! -f "$vendor_dir/$crate/LICENSE" ]]; then
       cd "$license_tmp"
       local latest_version="$(latest_version_dir $crate)"
       local src="$license_tmp/$latest_version/LICENSE"
       if [[ "$latest_version" != "$crate" && -f "$src" ]]; then
         cp "$src" "$vendor_dir/$crate"
       fi
     fi
   done
 }

 # Prepares a temporary workspace to run `crates_vendor` to generate BUILD.bazel
 # files for third-party Rust crates.
 function prepare_bazel_workspace {
   local workspace_dir=$1

   ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/BUILD.bazel" "${workspace_dir}/BUILD.bazel"
   ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/MODULE.bazel" "${workspace_dir}/MODULE.bazel"
   ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/crate_annotations.bzl" "${workspace_dir}/crate_annotations.bzl"

   ln -s "${FUCHSIA_DIR}/prebuilt/third_party/bazel/linux-x64/bazel-real" "${workspace_dir}/bazel"
   ln -s "${FUCHSIA_DIR}/prebuilt/third_party/buildifier/linux-x64/buildifier" "${workspace_dir}/buildifier"
   ln -s "${FUCHSIA_DIR}/prebuilt/third_party/rust/linux-x64" "${workspace_dir}/rust_toolchain"
   ln -s "${FUCHSIA_DIR}/third_party/bazel_vendor/rules_rust+" "${workspace_dir}/rules_rust"

   # Relative paths of rust_crates need to be the same to $FUCHSIA_DIR and
   # $workspace_dir, so the generated BUILD.bazel would have correct paths when
   # used in the platform build.
   mkdir "${workspace_dir}/third_party"
   ln -s "${FUCHSIA_DIR}/third_party/rust_crates" "${workspace_dir}/third_party/rust_crates"

   # Download the cargo-bazel from upstream release. This avoids the bootstrap
   # process from crate_universe, which always does a slow full rebuild of
   # cargo-bazel.
   cargo_bazel_version=$(cat "${workspace_dir}/rules_rust/version.bzl" | grep 'VERSION =' | awk -F ' = ' '{print $2}' | tr -d '"')
   case "$(uname -s)" in
   Linux*)
     curl -sSL "https://github.com/bazelbuild/rules_rust/releases/download/${cargo_bazel_version}/cargo-bazel-x86_64-unknown-linux-gnu" -o "${workspace_dir}/cargo-bazel"
     ;;
   Darwin*)
     curl -sSL "https://github.com/bazelbuild/rules_rust/releases/download/${cargo_bazel_version}/cargo-bazel-aarch64-apple-darwin" -o "${workspace_dir}/cargo-bazel"
     ;;
   *)
     fx-error "unrecognized OS"
     exit 1
     ;;
   esac
   chmod +x "${workspace_dir}/cargo-bazel"
 }

 source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/../lib/vars.sh || exit $?
 fx-config-read

 case "$(uname -s)" in
 Linux*) ;;
 Darwin*)
   if ! [[ -x "$(command -v brew)" ]]; then
     fx-error "'brew' binary not found"
     fx-error "A homebrew <https://brew.sh> installation of opensslis required in order to update"
     fx-error "Rust third party crates on the Mac."
     exit 1
   fi

   declare -x LDFLAGS="-L$(brew --prefix)/opt/openssl/lib"
   declare -x CPPFLAGS="-I$(brew --prefix)/opt/openssl/include"
   ;;
 *)
   fx-error "unrecognized OS"
   exit 1
   ;;
 esac

 declare -x PATH=$PREBUILT_CMAKE_DIR/cmake/bin:$PATH
 export RUSTC=$PREBUILT_RUST_DIR/bin/rustc
 export CARGO=$PREBUILT_RUST_DIR/bin/cargo

 GNAW_TARGET="host-tools/gnaw"
 GNAW_BIN="${FUCHSIA_BUILD_DIR}/${GNAW_TARGET}"

 if [[ "$1" == "--no-build" ]]; then
   if [ ! -f "$GNAW_BIN" ]; then
     fx-error "--no-build was specified, but $GNAW_BIN does not exist."
     fx-error "Rerun without --no-build to build cargo-gnaw."
     exit 1
   fi
 else
   fx-run-ninja false ${PREBUILT_NINJA} -C ${FUCHSIA_BUILD_DIR} ${GNAW_TARGET} || (
     fx-error "Failed to build cargo-gnaw."
     fx-error "This can happen after cargo-gnaw exits early."
     fx-error "To retry an old build of cargo-gnaw, specify --no-build."
     exit 1
   )
 fi

 export VENDOR_DIR=$FUCHSIA_DIR/third_party/rust_crates/vendor
 export TMP=$(mktemp -d)
 trap "rm -rf ${TMP}" EXIT

 # Preserve both OWNERS and LICENSE files.
 (
   cd $VENDOR_DIR
   $(preserve_files $TMP 'OWNERS')
 )
 (
   cd $VENDOR_DIR
   $(preserve_files $TMP 'LICENSE')
 )

 bazel_workspace="${TMP}/bazel_workspace"
 mkdir "${bazel_workspace}"
 prepare_bazel_workspace "${bazel_workspace}"
 (
   cd "${bazel_workspace}" &&
     ./bazel run \
       --enable_bzlmod=true \
       --vendor_dir=$FUCHSIA_DIR/third_party/bazel_vendor \
       :crates_vendor
 )

 # Restore files for unchanged directories.
 shopt -s nullglob
 for f in $TMP/*; do
   crate_dir=$(basename "$f")
   # if the exact same version of the crate still exists
   if [ -d "${VENDOR_DIR}/${crate_dir}" ]; then
     # put the OWNERS file back
     cp -r "$f" "${VENDOR_DIR}/"
   fi
 done

 # Copy the OWNERS and LICENSE from the previous latest version for the new
 # latest version for newly-vendored versions
 $(propagate_owner_files $TMP $VENDOR_DIR)
 $(propagate_license_files $TMP $VENDOR_DIR)

 # ensure LICENSE* files exist
 $FUCHSIA_DIR/scripts/rust/check_rust_licenses.py \
   --directory $VENDOR_DIR

 # regenerate BUILD.gn
 (
   cd $FUCHSIA_DIR
   $GNAW_BIN \
     --manifest-path $FUCHSIA_DIR/third_party/rust_crates/Cargo.toml \
     --project-root $FUCHSIA_DIR \
     --cargo $CARGO \
     --output $FUCHSIA_DIR/third_party/rust_crates/BUILD.gn \
     --emit-metadata $FUCHSIA_BUILD_DIR/rustlang/3p-crates-metadata.json \
     --gn-bin $PREBUILT_GN \
     --skip-root
 )

 # Add to gnaw invocation to enable third-party SDK metadata generation:
 # --output-fuchsia-sdk-metadata $FUCHSIA_DIR/third_party/rust_crates/sdk_metas \
	#!/bin/bash
	# Copyright 2018 The Fuchsia Authors. All rights reserved.
	# Use of this source code is governed by a BSD-style license that can be
	# found in the LICENSE file.

	#### CATEGORY=Source tree
	### updates rustc_library and rustc_binary third_party dependencies

	## usage: fx update-rustc-third-party
	## Updates third_party/rust_crates based on the contents of
	## third_party/rust_crates/Cargo.toml
	##
	## See https://fuchsia.dev/fuchsia-src/development/languages/rust/third_party.md
	## for more details.

	set -eo pipefail

	function preserve_files {
	local tmp=$1
	local filename=$2
	# Some crates can vendor dependnecies in them, so limit max depth to avoid
	# picking up files from nested dependencies.
	find . -maxdepth 2 -type f -name "$filename" -print0 \| while read -d $'\0' file_to_preserve; do
	local dir=$(dirname "$file_to_preserve")
	local version=$(sed -n "s/^version = \"\(.*\)\"$/\\1/p" "$dir/Cargo.toml" \| head -n 1)
	# + in directory names are replaced with -, so they are supported in Bazel
	# labels.
	version="${version/+/-}"
	if [[ $dir != *-"$version" ]]; then
	dir="$dir-$version"
	fi
	local temp_dir="$tmp/$dir"
	mkdir -p "$temp_dir"
	cp "$file_to_preserve" "$temp_dir"
	done
	}

	function latest_version_dir {
	local crate=$1
	local crate_name=$(echo "$crate" \| sed -rn "s/^([a-zA-Z][a-zA-Z0-9_\\-])-[0-9]+\\.[0-9]+\\.[0-9]+.$/\\1/p")
	local semver_regex="s/^$crate_name-([0-9]+)\\.([0-9]+)\\.([0-9]+)(.*$)/\\1 \\2 \\3 \\4/p"
	local latest_version=$(echo "$crate" "$crate_name"-* \| tr ' ' '\n' \| sed -rn "$semver_regex" \| sort -rn -k 1,1 -k 2,2 -k 3,3 -k 4,4 \| sed -rn "s/^(.+) (.+) (.+) (.*$)/$crate_name-\\1.\\2.\\3\\4/p" \| grep -A1 -F "$crate" \| tail -n1)
	echo "$latest_version"
	}

	# This function looks for any vendored crates that are missing OWNERS files and
	# attempts to source one from the previous latest version of that crate.
	function propagate_owner_files {
	local owners_tmp=$1
	local vendor_dir=$2
	cd "$vendor_dir"
	for crate in *; do
	# vendor_dir can have generated Bazel build files in it, filter them out.
	# i.e. Only walk into subdirectories, which are vendored third-party crates.
	local path="${vendor_dir}/${crate}"
	if [[ -d "${path}" && ! -f "${path}/OWNERS" ]]; then
	cd "$owners_tmp"
	local latest_version="$(latest_version_dir $crate)"
	if [[ "$latest_version" != "$crate" ]]; then
	cp "$owners_tmp/$latest_version/OWNERS" "$vendor_dir/$crate"
	fi
	fi
	done
	}

	# This function looks for any vendored crates that are missing LICENSE files and
	# attempts to source one from the previous latest version of that crate.
	#
	# NOTE: This function is only intended for LICENSE files created by us, which
	# are always named LICENSE, so it skips copying if one does exist. This is OK
	# because we later rely on check_rust_licenses.py to check other possible
	# license file included by upstream.
	function propagate_license_files {
	local license_tmp=$1
	local vendor_dir=$2
	cd "$vendor_dir"
	for crate in *; do
	if [[ ! -f "$vendor_dir/$crate/LICENSE" ]]; then
	cd "$license_tmp"
	local latest_version="$(latest_version_dir $crate)"
	local src="$license_tmp/$latest_version/LICENSE"
	if [[ "$latest_version" != "$crate" && -f "$src" ]]; then
	cp "$src" "$vendor_dir/$crate"
	fi
	fi
	done
	}

	# Prepares a temporary workspace to run `crates_vendor` to generate BUILD.bazel
	# files for third-party Rust crates.
	function prepare_bazel_workspace {
	local workspace_dir=$1

	ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/BUILD.bazel" "${workspace_dir}/BUILD.bazel"
	ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/MODULE.bazel" "${workspace_dir}/MODULE.bazel"
	ln -s "${FUCHSIA_DIR}/build/bazel/update-rustc-third-party/crate_annotations.bzl" "${workspace_dir}/crate_annotations.bzl"

	ln -s "${FUCHSIA_DIR}/prebuilt/third_party/bazel/linux-x64/bazel-real" "${workspace_dir}/bazel"
	ln -s "${FUCHSIA_DIR}/prebuilt/third_party/buildifier/linux-x64/buildifier" "${workspace_dir}/buildifier"
	ln -s "${FUCHSIA_DIR}/prebuilt/third_party/rust/linux-x64" "${workspace_dir}/rust_toolchain"
	ln -s "${FUCHSIA_DIR}/third_party/bazel_vendor/rules_rust+" "${workspace_dir}/rules_rust"

	# Relative paths of rust_crates need to be the same to $FUCHSIA_DIR and
	# $workspace_dir, so the generated BUILD.bazel would have correct paths when
	# used in the platform build.
	mkdir "${workspace_dir}/third_party"
	ln -s "${FUCHSIA_DIR}/third_party/rust_crates" "${workspace_dir}/third_party/rust_crates"

	# Download the cargo-bazel from upstream release. This avoids the bootstrap
	# process from crate_universe, which always does a slow full rebuild of
	# cargo-bazel.
	cargo_bazel_version=$(cat "${workspace_dir}/rules_rust/version.bzl" \| grep 'VERSION =' \| awk -F ' = ' '{print $2}' \| tr -d '"')
	case "$(uname -s)" in
	Linux*)
	curl -sSL "https://github.com/bazelbuild/rules_rust/releases/download/${cargo_bazel_version}/cargo-bazel-x86_64-unknown-linux-gnu" -o "${workspace_dir}/cargo-bazel"
	;;
	Darwin*)
	curl -sSL "https://github.com/bazelbuild/rules_rust/releases/download/${cargo_bazel_version}/cargo-bazel-aarch64-apple-darwin" -o "${workspace_dir}/cargo-bazel"
	;;
	*)
	fx-error "unrecognized OS"
	exit 1
	;;
	esac
	chmod +x "${workspace_dir}/cargo-bazel"
	}

	source "$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"/../lib/vars.sh \|\| exit $?
	fx-config-read

	case "$(uname -s)" in
	Linux*) ;;
	Darwin*)
	if ! [[ -x "$(command -v brew)" ]]; then
	fx-error "'brew' binary not found"
	fx-error "A homebrew <https://brew.sh> installation of opensslis required in order to update"
	fx-error "Rust third party crates on the Mac."
	exit 1
	fi

	declare -x LDFLAGS="-L$(brew --prefix)/opt/openssl/lib"
	declare -x CPPFLAGS="-I$(brew --prefix)/opt/openssl/include"
	;;
	*)
	fx-error "unrecognized OS"
	exit 1
	;;
	esac

	declare -x PATH=$PREBUILT_CMAKE_DIR/cmake/bin:$PATH
	export RUSTC=$PREBUILT_RUST_DIR/bin/rustc
	export CARGO=$PREBUILT_RUST_DIR/bin/cargo

	GNAW_TARGET="host-tools/gnaw"
	GNAW_BIN="${FUCHSIA_BUILD_DIR}/${GNAW_TARGET}"

	if [[ "$1" == "--no-build" ]]; then
	if [ ! -f "$GNAW_BIN" ]; then
	fx-error "--no-build was specified, but $GNAW_BIN does not exist."
	fx-error "Rerun without --no-build to build cargo-gnaw."
	exit 1
	fi
	else
	fx-run-ninja false ${PREBUILT_NINJA} -C ${FUCHSIA_BUILD_DIR} ${GNAW_TARGET} \|\| (
	fx-error "Failed to build cargo-gnaw."
	fx-error "This can happen after cargo-gnaw exits early."
	fx-error "To retry an old build of cargo-gnaw, specify --no-build."
	exit 1
	)
	fi

	export VENDOR_DIR=$FUCHSIA_DIR/third_party/rust_crates/vendor
	export TMP=$(mktemp -d)
	trap "rm -rf ${TMP}" EXIT

	# Preserve both OWNERS and LICENSE files.
	(
	cd $VENDOR_DIR
	$(preserve_files $TMP 'OWNERS')
	)
	(
	cd $VENDOR_DIR
	$(preserve_files $TMP 'LICENSE')
	)

	bazel_workspace="${TMP}/bazel_workspace"
	mkdir "${bazel_workspace}"
	prepare_bazel_workspace "${bazel_workspace}"
	(
	cd "${bazel_workspace}" &&
	./bazel run \
	--enable_bzlmod=true \
	--vendor_dir=$FUCHSIA_DIR/third_party/bazel_vendor \
	:crates_vendor
	)

	# Restore files for unchanged directories.
	shopt -s nullglob
	for f in $TMP/*; do
	crate_dir=$(basename "$f")
	# if the exact same version of the crate still exists
	if [ -d "${VENDOR_DIR}/${crate_dir}" ]; then
	# put the OWNERS file back
	cp -r "$f" "${VENDOR_DIR}/"
	fi
	done

	# Copy the OWNERS and LICENSE from the previous latest version for the new
	# latest version for newly-vendored versions
	$(propagate_owner_files $TMP $VENDOR_DIR)
	$(propagate_license_files $TMP $VENDOR_DIR)

	# ensure LICENSE* files exist
	$FUCHSIA_DIR/scripts/rust/check_rust_licenses.py \
	--directory $VENDOR_DIR

	# regenerate BUILD.gn
	(
	cd $FUCHSIA_DIR
	$GNAW_BIN \
	--manifest-path $FUCHSIA_DIR/third_party/rust_crates/Cargo.toml \
	--project-root $FUCHSIA_DIR \
	--cargo $CARGO \
	--output $FUCHSIA_DIR/third_party/rust_crates/BUILD.gn \
	--emit-metadata $FUCHSIA_BUILD_DIR/rustlang/3p-crates-metadata.json \
	--gn-bin $PREBUILT_GN \
	--skip-root
	)

	# Add to gnaw invocation to enable third-party SDK metadata generation:
	# --output-fuchsia-sdk-metadata $FUCHSIA_DIR/third_party/rust_crates/sdk_metas \