src/ci/github-actions/ci.yml - third_party/rust - Git at Google

 ######################################################
 #   WARNING! Action needed when changing this file   #
 ######################################################

 # Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
 # CI configuration stored on the repository. To work around that this file is
 # expanded by a tool in the repository, and the expansion is committed as well.
 #
 # After you make any change to the file you'll need to run this command:
 #
 #   ./x.py run src/tools/expand-yaml-anchors
 #
 # ...and commit the file it updated in addition to this one. If you forget this
 # step CI will fail.

 ---
 ###############################
 #   YAML Anchors Definition   #
 ###############################

 # This key contains most of the YAML anchors that will be used later in the
 # document. YAML anchors allows us to greatly reduce duplication inside the CI
 # configuration by reusing parts of the configuration.
 #
 # YAML anchors work by defining an anchor with `&anchor-name` and reusing its
 # content in another place with `*anchor-name`. The special `<<` map key merges
 # the content of the map with the content of the anchor (or list of anchors).
 #
 # The expand-yaml-anchors tool will automatically remove this block from the
 # output YAML file.
 x--expand-yaml-anchors--remove:
   # These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
   # Check out their documentation for more information on why they're needed.

   - &base-outcome-job
     name: bors build finished
     runs-on: ubuntu-latest

   - &base-success-job
     steps:
       - name: mark the job as a success
         run: exit 0
         shell: bash
     <<: *base-outcome-job

   - &base-failure-job
     steps:
       - name: mark the job as a failure
         run: exit 1
         shell: bash
     <<: *base-outcome-job

 ###########################
 #   Builders definition   #
 ###########################

 name: CI
 on:
   push:
     branches:
       - auto
       - try
       - try-perf
       - automation/bors/try
   pull_request:
     branches:
       - "**"

 permissions:
   contents: read
   packages: write

 defaults:
   run:
     # On Linux, macOS, and Windows, use the system-provided bash as the default
     # shell. (This should only make a difference on Windows, where the default
     # shell is PowerShell.)
     shell: bash

 concurrency:
   # For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
   # We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
   # are all triggered on the same branch, but which should be able to run concurrently.
   group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}
   cancel-in-progress: true

 env:
   TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate

 jobs:
   # The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
   # It calculates which jobs should be executed, based on the data of the ${{ github }} context.
   # If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
   calculate_matrix:
     name: Calculate job matrix
     runs-on: ubuntu-latest
     outputs:
       jobs: ${{ steps.jobs.outputs.jobs }}
     steps:
       - name: Checkout the source code
         uses: actions/checkout@v4
       - name: Calculate the CI job matrix
         run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
         id: jobs
   job:
     name: ${{ matrix.name }}
     needs: [ calculate_matrix ]
     runs-on: "${{ matrix.os }}"
     defaults:
       run:
         shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}
     timeout-minutes: 600
     env:
       CI_JOB_NAME: ${{ matrix.image }}
       CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
       # commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
       HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
       DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       SCCACHE_BUCKET: rust-lang-ci-sccache2
       CACHE_DOMAIN: ci-caches.rust-lang.org
     continue-on-error: ${{ matrix.continue_on_error || false }}
     strategy:
       matrix:
         # Check the `calculate_matrix` job to see how is the matrix defined.
         include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
     # GitHub Actions fails the workflow if an empty list of jobs is provided to
     # the workflow, so we need to skip this job if nothing was produced by
     # the Python script.
     #
     # Unfortunately checking whether a list is empty is not possible in a nice
     # way due to GitHub Actions expressions limits.
     # This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
     if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
     steps:
       - if: contains(matrix.os, 'windows')
         uses: msys2/setup-msys2@v2.22.0
         with:
           # i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
           msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}
           # don't try to download updates for already installed packages
           update: false
           # don't try to use the msys that comes built-in to the github runner,
           # so we can control what is installed (i.e. not python)
           release: true
           # Inherit the full path from the Windows environment, with MSYS2's */bin/
           # dirs placed in front. This lets us run Windows-native Python etc.
           path-type: inherit
           install: >
             make
             dos2unix
             diffutils

       - name: disable git crlf conversion
         run: git config --global core.autocrlf false

       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2

       # Rust Log Analyzer can't currently detect the PR number of a GitHub
       # Actions build on its own, so a hint in the log message is needed to
       # point it in the right direction.
       - name: configure the PR in which the error message will be posted
         run: echo "[CI_PR_NUMBER=$num]"
         env:
           num: ${{ github.event.number }}
         if: success() && github.event_name == 'pull_request'

       - name: add extra environment variables
         run: src/ci/scripts/setup-environment.sh
         env:
           # Since it's not possible to merge `${{ matrix.env }}` with the other
           # variables in `job.<name>.env`, the variables defined in the matrix
           # are passed to the `setup-environment.sh` script encoded in JSON,
           # which then uses log commands to actually set them.
           EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

       - name: ensure the channel matches the target branch
         run: src/ci/scripts/verify-channel.sh

       - name: collect CPU statistics
         run: src/ci/scripts/collect-cpu-stats.sh

       - name: show the current environment
         run: src/ci/scripts/dump-environment.sh

       - name: install awscli
         run: src/ci/scripts/install-awscli.sh

       - name: install sccache
         run: src/ci/scripts/install-sccache.sh

       - name: select Xcode
         run: src/ci/scripts/select-xcode.sh

       - name: install clang
         run: src/ci/scripts/install-clang.sh

       - name: install tidy
         run: src/ci/scripts/install-tidy.sh

       - name: install WIX
         run: src/ci/scripts/install-wix.sh

       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh

       - name: checkout submodules
         run: src/ci/scripts/checkout-submodules.sh

       - name: install MSYS2
         run: src/ci/scripts/install-msys2.sh

       - name: install MinGW
         run: src/ci/scripts/install-mingw.sh

       - name: install ninja
         run: src/ci/scripts/install-ninja.sh

       - name: enable ipv6 on Docker
         run: src/ci/scripts/enable-docker-ipv6.sh

       # Disable automatic line ending conversion (again). On Windows, when we're
       # installing dependencies, something switches the git configuration directory or
       # re-enables autocrlf. We've not tracked down the exact cause -- and there may
       # be multiple -- but this should ensure submodules are checked out with the
       # appropriate line endings.
       - name: disable git crlf conversion
         run: src/ci/scripts/disable-git-crlf-conversion.sh

       - name: ensure line endings are correct
         run: src/ci/scripts/verify-line-endings.sh

       - name: ensure backported commits are in upstream branches
         run: src/ci/scripts/verify-backported-commits.sh

       - name: ensure the stable version number is correct
         run: src/ci/scripts/verify-stable-version-number.sh

       - name: run the build
         # Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
         run: src/ci/scripts/run-build-from-ci.sh 2>&1
         env:
           AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

       - name: create github artifacts
         run: src/ci/scripts/create-doc-artifacts.sh

       - name: upload artifacts to github
         uses: actions/upload-artifact@v4
         with:
           # name is set in previous step
           name: ${{ env.DOC_ARTIFACT_NAME }}
           path: obj/artifacts/doc
           if-no-files-found: ignore
           retention-days: 5

       - name: upload artifacts to S3
         run: src/ci/scripts/upload-artifacts.sh
         env:
           AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
         # Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
         # builders *should* have the AWS credentials available. Still, explicitly
         # adding the condition is helpful as this way CI will not silently skip
         # deploying artifacts from a dist builder if the variables are misconfigured,
         # erroring about invalid credentials instead.
         if: success() && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')

   # These jobs don't actually test anything, but they're used to tell bors the
   # build completed, as there is no practical way to detect when a workflow is
   # successful listening to webhooks only.
   try-success:
     needs: [ job ]
     if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-success-job
   try-failure:
     needs: [ job ]
     if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
   auto-success:
     needs: [ job ]
     if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-outcome-job
     steps:
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           fetch-depth: 2
       - name: publish toolstate
         run: src/ci/publish_toolstate.sh
         shell: bash
         env:
           TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

   auto-failure:
     needs: [ job ]
     if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
     <<: *base-failure-job
	######################################################
	# WARNING! Action needed when changing this file #
	######################################################

	# Due to GitHub Actions limitations, we can't use YAML Anchors directly in the
	# CI configuration stored on the repository. To work around that this file is
	# expanded by a tool in the repository, and the expansion is committed as well.
	#
	# After you make any change to the file you'll need to run this command:
	#
	# ./x.py run src/tools/expand-yaml-anchors
	#
	# ...and commit the file it updated in addition to this one. If you forget this
	# step CI will fail.

	---
	###############################
	# YAML Anchors Definition #
	###############################

	# This key contains most of the YAML anchors that will be used later in the
	# document. YAML anchors allows us to greatly reduce duplication inside the CI
	# configuration by reusing parts of the configuration.
	#
	# YAML anchors work by defining an anchor with `&anchor-name` and reusing its
	# content in another place with `*anchor-name`. The special `<<` map key merges
	# the content of the map with the content of the anchor (or list of anchors).
	#
	# The expand-yaml-anchors tool will automatically remove this block from the
	# output YAML file.
	x--expand-yaml-anchors--remove:
	# These snippets are used by the try-success, try-failure, auto-success and auto-failure jobs.
	# Check out their documentation for more information on why they're needed.

	- &base-outcome-job
	name: bors build finished
	runs-on: ubuntu-latest

	- &base-success-job
	steps:
	- name: mark the job as a success
	run: exit 0
	shell: bash
	<<: *base-outcome-job

	- &base-failure-job
	steps:
	- name: mark the job as a failure
	run: exit 1
	shell: bash
	<<: *base-outcome-job

	###########################
	# Builders definition #
	###########################

	name: CI
	on:
	push:
	branches:
	- auto
	- try
	- try-perf
	- automation/bors/try
	pull_request:
	branches:
	- "**"

	permissions:
	contents: read
	packages: write

	defaults:
	run:
	# On Linux, macOS, and Windows, use the system-provided bash as the default
	# shell. (This should only make a difference on Windows, where the default
	# shell is PowerShell.)
	shell: bash

	concurrency:
	# For a given workflow, if we push to the same branch, cancel all previous builds on that branch.
	# We add an exception for try builds (try branch) and unrolled rollup builds (try-perf), which
	# are all triggered on the same branch, but which should be able to run concurrently.
	group: ${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' \|\| github.ref == 'refs/heads/try-perf') && github.sha) \|\| github.ref }}
	cancel-in-progress: true

	env:
	TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate

	jobs:
	# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
	# It calculates which jobs should be executed, based on the data of the ${{ github }} context.
	# If you want to modify CI jobs, take a look at src/ci/github-actions/jobs.yml.
	calculate_matrix:
	name: Calculate job matrix
	runs-on: ubuntu-latest
	outputs:
	jobs: ${{ steps.jobs.outputs.jobs }}
	steps:
	- name: Checkout the source code
	uses: actions/checkout@v4
	- name: Calculate the CI job matrix
	run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
	id: jobs
	job:
	name: ${{ matrix.name }}
	needs: [ calculate_matrix ]
	runs-on: "${{ matrix.os }}"
	defaults:
	run:
	shell: ${{ contains(matrix.os, 'windows') && 'msys2 {0}' \|\| 'bash' }}
	timeout-minutes: 600
	env:
	CI_JOB_NAME: ${{ matrix.image }}
	CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
	# commit of PR sha or commit sha. `GITHUB_SHA` is not accurate for PRs.
	HEAD_SHA: ${{ github.event.pull_request.head.sha \|\| github.sha }}
	DOCKER_TOKEN: ${{ secrets.GITHUB_TOKEN }}
	SCCACHE_BUCKET: rust-lang-ci-sccache2
	CACHE_DOMAIN: ci-caches.rust-lang.org
	continue-on-error: ${{ matrix.continue_on_error \|\| false }}
	strategy:
	matrix:
	# Check the `calculate_matrix` job to see how is the matrix defined.
	include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
	# GitHub Actions fails the workflow if an empty list of jobs is provided to
	# the workflow, so we need to skip this job if nothing was produced by
	# the Python script.
	#
	# Unfortunately checking whether a list is empty is not possible in a nice
	# way due to GitHub Actions expressions limits.
	# This hack is taken from https://github.com/ferrocene/ferrocene/blob/d43edc6b7697cf1719ec1c17c54904ab94825763/.github/workflows/release.yml#L75-L82
	if: fromJSON(needs.calculate_matrix.outputs.jobs)[0] != null
	steps:
	- if: contains(matrix.os, 'windows')
	uses: msys2/setup-msys2@v2.22.0
	with:
	# i686 jobs use mingw32. x86_64 and cross-compile jobs use mingw64.
	msystem: ${{ contains(matrix.name, 'i686') && 'mingw32' \|\| 'mingw64' }}
	# don't try to download updates for already installed packages
	update: false
	# don't try to use the msys that comes built-in to the github runner,
	# so we can control what is installed (i.e. not python)
	release: true
	# Inherit the full path from the Windows environment, with MSYS2's */bin/
	# dirs placed in front. This lets us run Windows-native Python etc.
	path-type: inherit
	install: >
	make
	dos2unix
	diffutils

	- name: disable git crlf conversion
	run: git config --global core.autocrlf false

	- name: checkout the source code
	uses: actions/checkout@v4
	with:
	fetch-depth: 2

	# Rust Log Analyzer can't currently detect the PR number of a GitHub
	# Actions build on its own, so a hint in the log message is needed to
	# point it in the right direction.
	- name: configure the PR in which the error message will be posted
	run: echo "[CI_PR_NUMBER=$num]"
	env:
	num: ${{ github.event.number }}
	if: success() && github.event_name == 'pull_request'

	- name: add extra environment variables
	run: src/ci/scripts/setup-environment.sh
	env:
	# Since it's not possible to merge `${{ matrix.env }}` with the other
	# variables in `job.<name>.env`, the variables defined in the matrix
	# are passed to the `setup-environment.sh` script encoded in JSON,
	# which then uses log commands to actually set them.
	EXTRA_VARIABLES: ${{ toJson(matrix.env) }}

	- name: ensure the channel matches the target branch
	run: src/ci/scripts/verify-channel.sh

	- name: collect CPU statistics
	run: src/ci/scripts/collect-cpu-stats.sh

	- name: show the current environment
	run: src/ci/scripts/dump-environment.sh

	- name: install awscli
	run: src/ci/scripts/install-awscli.sh

	- name: install sccache
	run: src/ci/scripts/install-sccache.sh

	- name: select Xcode
	run: src/ci/scripts/select-xcode.sh

	- name: install clang
	run: src/ci/scripts/install-clang.sh

	- name: install tidy
	run: src/ci/scripts/install-tidy.sh

	- name: install WIX
	run: src/ci/scripts/install-wix.sh

	- name: disable git crlf conversion
	run: src/ci/scripts/disable-git-crlf-conversion.sh

	- name: checkout submodules
	run: src/ci/scripts/checkout-submodules.sh

	- name: install MSYS2
	run: src/ci/scripts/install-msys2.sh

	- name: install MinGW
	run: src/ci/scripts/install-mingw.sh

	- name: install ninja
	run: src/ci/scripts/install-ninja.sh

	- name: enable ipv6 on Docker
	run: src/ci/scripts/enable-docker-ipv6.sh

	# Disable automatic line ending conversion (again). On Windows, when we're
	# installing dependencies, something switches the git configuration directory or
	# re-enables autocrlf. We've not tracked down the exact cause -- and there may
	# be multiple -- but this should ensure submodules are checked out with the
	# appropriate line endings.
	- name: disable git crlf conversion
	run: src/ci/scripts/disable-git-crlf-conversion.sh

	- name: ensure line endings are correct
	run: src/ci/scripts/verify-line-endings.sh

	- name: ensure backported commits are in upstream branches
	run: src/ci/scripts/verify-backported-commits.sh

	- name: ensure the stable version number is correct
	run: src/ci/scripts/verify-stable-version-number.sh

	- name: run the build
	# Redirect stderr to stdout to avoid reordering the two streams in the GHA logs.
	run: src/ci/scripts/run-build-from-ci.sh 2>&1
	env:
	AWS_ACCESS_KEY_ID: ${{ env.CACHES_AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}
	TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

	- name: create github artifacts
	run: src/ci/scripts/create-doc-artifacts.sh

	- name: upload artifacts to github
	uses: actions/upload-artifact@v4
	with:
	# name is set in previous step
	name: ${{ env.DOC_ARTIFACT_NAME }}
	path: obj/artifacts/doc
	if-no-files-found: ignore
	retention-days: 5

	- name: upload artifacts to S3
	run: src/ci/scripts/upload-artifacts.sh
	env:
	AWS_ACCESS_KEY_ID: ${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}
	AWS_SECRET_ACCESS_KEY: ${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}
	# Adding a condition on DEPLOY=1 or DEPLOY_ALT=1 is not needed as all deploy
	# builders should have the AWS credentials available. Still, explicitly
	# adding the condition is helpful as this way CI will not silently skip
	# deploying artifacts from a dist builder if the variables are misconfigured,
	# erroring about invalid credentials instead.
	if: success() && (github.event_name == 'push' \|\| env.DEPLOY == '1' \|\| env.DEPLOY_ALT == '1')

	# These jobs don't actually test anything, but they're used to tell bors the
	# build completed, as there is no practical way to detect when a workflow is
	# successful listening to webhooks only.
	try-success:
	needs: [ job ]
	if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' \|\| github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
	<<: *base-success-job
	try-failure:
	needs: [ job ]
	if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' \|\| github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
	<<: *base-failure-job
	auto-success:
	needs: [ job ]
	if: "success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
	<<: *base-outcome-job
	steps:
	- name: checkout the source code
	uses: actions/checkout@v4
	with:
	fetch-depth: 2
	- name: publish toolstate
	run: src/ci/publish_toolstate.sh
	shell: bash
	env:
	TOOLSTATE_REPO_ACCESS_TOKEN: ${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}

	auto-failure:
	needs: [ job ]
	if: "!success() && github.event_name == 'push' && github.ref == 'refs/heads/auto' && github.repository == 'rust-lang-ci/rust'"
	<<: *base-failure-job