Move CI to new builtin check-toolstate
diff --git a/src/bootstrap/toolstate.rs b/src/bootstrap/toolstate.rs
index bec2853..a6d9ef3 100644
--- a/src/bootstrap/toolstate.rs
+++ b/src/bootstrap/toolstate.rs
@@ -290,6 +290,8 @@
}
}
+ // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date
+ // as well.
git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com");
git_config("user.name", "Rust Toolstate Update");
git_config("credential.helper", "store");
diff --git a/src/ci/azure-pipelines/auto.yml b/src/ci/azure-pipelines/auto.yml
index 70d6bad..5248e41 100644
--- a/src/ci/azure-pipelines/auto.yml
+++ b/src/ci/azure-pipelines/auto.yml
@@ -163,7 +163,7 @@
NO_LLVM_ASSERTIONS: 1
# MSVC tools tests
x86_64-msvc-tools:
- SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py /tmp/toolstate/toolstates.json windows
+ SCRIPT: src/ci/docker/x86_64-gnu-tools/checktools.sh x.py
RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --save-toolstates=/tmp/toolstate/toolstates.json
# 32/64-bit MinGW builds.
diff --git a/src/ci/azure-pipelines/master.yml b/src/ci/azure-pipelines/master.yml
index e2baa92..9c5a15a 100644
--- a/src/ci/azure-pipelines/master.yml
+++ b/src/ci/azure-pipelines/master.yml
@@ -16,10 +16,7 @@
- checkout: self
fetchDepth: 2
-- script: |
- export MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
- . src/ci/docker/x86_64-gnu-tools/repo.sh
- commit_toolstate_change "$MESSAGE_FILE" "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" "$(git log --format=%s -n1 HEAD)" "$MESSAGE_FILE" "$TOOLSTATE_REPO_ACCESS_TOKEN"
+- script: src/ci/publish_toolstate.sh
displayName: Publish toolstate
env:
TOOLSTATE_REPO_ACCESS_TOKEN: $(TOOLSTATE_REPO_ACCESS_TOKEN)
diff --git a/src/ci/docker/x86_64-gnu-tools/Dockerfile b/src/ci/docker/x86_64-gnu-tools/Dockerfile
index 7687a6c..5119340 100644
--- a/src/ci/docker/x86_64-gnu-tools/Dockerfile
+++ b/src/ci/docker/x86_64-gnu-tools/Dockerfile
@@ -17,9 +17,7 @@
COPY scripts/sccache.sh /scripts/
RUN sh /scripts/sccache.sh
-COPY x86_64-gnu-tools/checkregression.py /tmp/
COPY x86_64-gnu-tools/checktools.sh /tmp/
-COPY x86_64-gnu-tools/repo.sh /tmp/
# Run rustbook with `linkcheck` feature enabled
ENV CHECK_LINKS 1
@@ -27,4 +25,4 @@
ENV RUST_CONFIGURE_ARGS \
--build=x86_64-unknown-linux-gnu \
--save-toolstates=/tmp/toolstate/toolstates.json
-ENV SCRIPT /tmp/checktools.sh ../x.py /tmp/toolstate/toolstates.json linux
+ENV SCRIPT /tmp/checktools.sh ../x.py
diff --git a/src/ci/docker/x86_64-gnu-tools/checkregression.py b/src/ci/docker/x86_64-gnu-tools/checkregression.py
deleted file mode 100755
index 4fbb8c4..0000000
--- a/src/ci/docker/x86_64-gnu-tools/checkregression.py
+++ /dev/null
@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-## This script has two purposes: detect any tool that *regressed*, which is used
-## during the week before the beta branches to reject PRs; and detect any tool
-## that *changed* to see if we need to update the toolstate repo.
-
-import sys
-import json
-
-# Regressions for these tools during the beta cutoff week do not cause failure.
-# See `status_check` in `checktools.sh` for tools that have to pass on the
-# beta/stable branches.
-REGRESSION_OK = ["rustc-guide", "miri", "embedded-book"]
-
-if __name__ == '__main__':
- os_name = sys.argv[1]
- toolstate_file = sys.argv[2]
- current_state = sys.argv[3]
- verb = sys.argv[4] # 'regressed' or 'changed'
-
- with open(toolstate_file, 'r') as f:
- toolstate = json.load(f)
- with open(current_state, 'r') as f:
- current = json.load(f)
-
- regressed = False
- for cur in current:
- tool = cur['tool']
- state = cur[os_name]
- new_state = toolstate.get(tool, '')
- if verb == 'regressed':
- updated = new_state < state
- elif verb == 'changed':
- updated = new_state != state
- else:
- print('Unknown verb {}'.format(updated))
- sys.exit(2)
- if updated:
- print(
- 'The state of "{}" has {} from "{}" to "{}"'
- .format(tool, verb, state, new_state)
- )
- if not (verb == 'regressed' and tool in REGRESSION_OK):
- regressed = True
-
- if regressed:
- sys.exit(1)
diff --git a/src/ci/docker/x86_64-gnu-tools/checktools.sh b/src/ci/docker/x86_64-gnu-tools/checktools.sh
index ebb8c0b..e57fe22 100755
--- a/src/ci/docker/x86_64-gnu-tools/checktools.sh
+++ b/src/ci/docker/x86_64-gnu-tools/checktools.sh
@@ -3,18 +3,6 @@
set -eu
X_PY="$1"
-TOOLSTATE_FILE="$(realpath -m $2)"
-OS="$3"
-COMMIT="$(git rev-parse HEAD)"
-CHANGED_FILES="$(git diff --name-status HEAD HEAD^)"
-SIX_WEEK_CYCLE="$(( ($(date +%s) / 86400 - 20) % 42 ))"
-# ^ Number of days after the last promotion of beta.
-# Its value is 41 on the Tuesday where "Promote master to beta (T-2)" happens.
-# The Wednesday after this has value 0.
-# We track this value to prevent regressing tools in the last week of the 6-week cycle.
-
-mkdir -p "$(dirname $TOOLSTATE_FILE)"
-touch "$TOOLSTATE_FILE"
# Try to test all the tools and store the build/test success in the TOOLSTATE_FILE
@@ -34,106 +22,4 @@
set -e
-cat "$TOOLSTATE_FILE"
-echo
-
-# This function checks if a particular tool is *not* in status "test-pass".
-check_tool_failed() {
- grep -vq '"'"$1"'":"test-pass"' "$TOOLSTATE_FILE"
-}
-
-# This function checks that if a tool's submodule changed, the tool's state must improve
-verify_submodule_changed() {
- echo "Verifying status of $1..."
- if echo "$CHANGED_FILES" | grep -q "^M[[:blank:]]$2$"; then
- echo "This PR updated '$2', verifying if status is 'test-pass'..."
- if check_tool_failed "$1"; then
- echo
- echo "⚠️ We detected that this PR updated '$1', but its tests failed."
- echo
- echo "If you do intend to update '$1', please check the error messages above and"
- echo "commit another update."
- echo
- echo "If you do NOT intend to update '$1', please ensure you did not accidentally"
- echo "change the submodule at '$2'. You may ask your reviewer for the"
- echo "proper steps."
- exit 3
- fi
- fi
-}
-
-# deduplicates the submodule check and the assertion that on beta some tools MUST be passing.
-# $1 should be "submodule_changed" to only check tools that got changed by this PR,
-# or "beta_required" to check all tools that have $2 set to "beta".
-check_dispatch() {
- if [ "$1" = submodule_changed ]; then
- # ignore $2 (branch id)
- verify_submodule_changed $3 $4
- elif [ "$2" = beta ]; then
- echo "Requiring test passing for $3..."
- if check_tool_failed "$3"; then
- exit 4
- fi
- fi
-}
-
-# List all tools here.
-# This function gets called with "submodule_changed" for each PR that changed a submodule,
-# and with "beta_required" for each PR that lands on beta/stable.
-# The purpose of this function is to *reject* PRs if a tool is not "test-pass" and
-# (a) the tool's submodule has been updated, or (b) we landed on beta/stable and the
-# tool has to "test-pass" on that branch.
-status_check() {
- check_dispatch $1 beta book src/doc/book
- check_dispatch $1 beta nomicon src/doc/nomicon
- check_dispatch $1 beta reference src/doc/reference
- check_dispatch $1 beta rust-by-example src/doc/rust-by-example
- check_dispatch $1 beta edition-guide src/doc/edition-guide
- check_dispatch $1 beta rls src/tools/rls
- check_dispatch $1 beta rustfmt src/tools/rustfmt
- check_dispatch $1 beta clippy-driver src/tools/clippy
- # These tools are not required on the beta/stable branches, but they *do* cause
- # PRs to fail if a submodule update does not fix them.
- # They will still cause failure during the beta cutoff week, unless `checkregression.py`
- # exempts them from that.
- check_dispatch $1 nightly miri src/tools/miri
- check_dispatch $1 nightly embedded-book src/doc/embedded-book
- check_dispatch $1 nightly rustc-guide src/doc/rustc-guide
-}
-
-# If this PR is intended to update one of these tools, do not let the build pass
-# when they do not test-pass.
-
-status_check "submodule_changed"
-
-CHECK_NOT="$(readlink -f "$(dirname $0)/checkregression.py")"
-# This callback is called by `commit_toolstate_change`, see `repo.sh`.
-change_toolstate() {
- # only update the history
- if python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" changed; then
- echo 'Toolstate is not changed. Not updating.'
- else
- if [ $SIX_WEEK_CYCLE -ge 35 ]; then
- # Reject any regressions during the week before beta cutoff.
- python2.7 "$CHECK_NOT" "$OS" "$TOOLSTATE_FILE" "_data/latest.json" regressed
- fi
- sed -i "1 a\\
-$COMMIT\t$(cat "$TOOLSTATE_FILE")
-" "history/$OS.tsv"
- fi
-}
-
-if [ "$RUST_RELEASE_CHANNEL" = nightly ]; then
- if [ -n "${TOOLSTATE_PUBLISH+is_set}" ]; then
- . "$(dirname $0)/repo.sh"
- MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
- echo "($OS CI update)" > "$MESSAGE_FILE"
- commit_toolstate_change "$MESSAGE_FILE" change_toolstate
- rm -f "$MESSAGE_FILE"
- fi
- exit 0
-fi
-
-# abort compilation if an important tool doesn't build
-# (this code is reachable if not on the nightly channel)
-status_check "beta_required"
+python2.7 "$X_PY" test check-tools
diff --git a/src/ci/docker/x86_64-gnu-tools/repo.sh b/src/ci/docker/x86_64-gnu-tools/repo.sh
deleted file mode 100644
index 82700a0..0000000
--- a/src/ci/docker/x86_64-gnu-tools/repo.sh
+++ /dev/null
@@ -1,68 +0,0 @@
-#!/bin/sh
-
-# This file provides the function `commit_toolstate_change` for pushing a change
-# to the `rust-toolstate` repository.
-#
-# The function relies on a GitHub bot user, which should have a Personal access
-# token defined in the environment variable $TOOLSTATE_REPO_ACCESS_TOKEN. If for
-# some reason you need to change the token, please update the Azure Pipelines
-# variable group.
-#
-# 1. Generate a new Personal access token:
-#
-# * Login to the bot account, and go to Settings -> Developer settings ->
-# Personal access tokens
-# * Click "Generate new token"
-# * Enable the "public_repo" permission, then click "Generate token"
-# * Copy the generated token (should be a 40-digit hexadecimal number).
-# Save it somewhere secure, as the token would be gone once you leave
-# the page.
-#
-# 2. Update the variable group in Azure Pipelines
-#
-# * Ping a member of the infrastructure team to do this.
-#
-# 4. Replace the email address below if the bot account identity is changed
-#
-# * See <https://help.github.com/articles/about-commit-email-addresses/>
-# if a private email by GitHub is wanted.
-
-commit_toolstate_change() {
- OLDFLAGS="$-"
- set -eu
-
- git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com'
- git config --global user.name 'Rust Toolstate Update'
- git config --global credential.helper store
- printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
- > "$HOME/.git-credentials"
- git clone --depth=1 $TOOLSTATE_REPO
-
- cd rust-toolstate
- FAILURE=1
- MESSAGE_FILE="$1"
- shift
- for RETRY_COUNT in 1 2 3 4 5; do
- # Call the callback.
- # - If we are in the `auto` branch (pre-landing), this is called from `checktools.sh` and
- # the callback is `change_toolstate` in that file. The purpose of this is to publish the
- # test results (the new commit-to-toolstate mapping) in the toolstate repo.
- # - If we are in the `master` branch (post-landing), this is called by the CI pipeline
- # and the callback is `src/tools/publish_toolstate.py`. The purpose is to publish
- # the new "current" toolstate in the toolstate repo.
- "$@"
- # `git commit` failing means nothing to commit.
- FAILURE=0
- git commit -a -F "$MESSAGE_FILE" || break
- # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
- git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
- FAILURE=1
- git fetch origin master
- git reset --hard origin/master
- done
- cd ..
-
- set +eu
- set "-$OLDFLAGS"
- return $FAILURE
-}
diff --git a/src/ci/publish_toolstate.sh b/src/ci/publish_toolstate.sh
new file mode 100644
index 0000000..d8ff740
--- /dev/null
+++ b/src/ci/publish_toolstate.sh
@@ -0,0 +1,33 @@
+#!/bin/sh
+
+set -eu
+
+# The following lines are also found in src/bootstrap/toolstate.rs,
+# so if updating here, please also update that file.
+
+export MESSAGE_FILE=$(mktemp -t msg.XXXXXX)
+
+git config --global user.email '7378925+rust-toolstate-update@users.noreply.github.com'
+git config --global user.name 'Rust Toolstate Update'
+git config --global credential.helper store
+printf 'https://%s:x-oauth-basic@github.com\n' "$TOOLSTATE_REPO_ACCESS_TOKEN" \
+ > "$HOME/.git-credentials"
+git clone --depth=1 $TOOLSTATE_REPO
+
+cd rust-toolstate
+FAILURE=1
+for RETRY_COUNT in 1 2 3 4 5; do
+ # The purpose is to publish the new "current" toolstate in the toolstate repo.
+ "$BUILD_SOURCESDIRECTORY/src/tools/publish_toolstate.py" "$(git rev-parse HEAD)" \
+ "$(git log --format=%s -n1 HEAD)" \
+ "$MESSAGE_FILE" \
+ "$TOOLSTATE_REPO_ACCESS_TOKEN"
+ # `git commit` failing means nothing to commit.
+ FAILURE=0
+ git commit -a -F "$MESSAGE_FILE" || break
+ # On failure randomly sleep for 0 to 3 seconds as a crude way to introduce jittering.
+ git push origin master && break || sleep $(LC_ALL=C tr -cd 0-3 < /dev/urandom | head -c 1)
+ FAILURE=1
+ git fetch origin master
+ git reset --hard origin/master
+done
