diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml index b7b5a03bd41f3..8492d4d9a85c0 100644 --- a/.github/workflows/dependencies.yml +++ b/.github/workflows/dependencies.yml @@ -16,84 +16,63 @@ defaults: env: # So cargo doesn't complain about unstable features RUSTC_BOOTSTRAP: 1 - PR_TITLE: Weekly `cargo update` - PR_MESSAGE: | - Automation to keep dependencies in `Cargo.lock` current. - - The following is the output from `cargo update`: - COMMIT_MESSAGE: "cargo update \n\n" jobs: - not-waiting-on-bors: - if: github.repository_owner == 'rust-lang' - name: skip if S-waiting-on-bors - runs-on: ubuntu-latest - steps: - - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - # Fetch state and labels of PR - # Or exit successfully if PR does not exist - JSON=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json labels,state || exit 0) - STATE=$(echo "$JSON" | jq -r '.state') - WAITING_ON_BORS=$(echo "$JSON" | jq '.labels[] | any(.name == "S-waiting-on-bors"; .)') - - # Exit with error if open and S-waiting-on-bors - if [[ "$STATE" == "OPEN" && "$WAITING_ON_BORS" == "true" ]]; then - exit 1 - fi - update: if: github.repository_owner == 'rust-lang' name: update dependencies - needs: not-waiting-on-bors runs-on: ubuntu-latest + outputs: + all_lockfiles: ${{ steps.update_script.outputs.all_lockfiles }} + enqueued_lockfiles: ${{ steps.update_script.outputs.enqueued_lockfiles }} steps: - name: checkout the source code uses: actions/checkout@v4 with: submodules: recursive + - name: install the bootstrap toolchain run: | # Extract the stage0 version - TOOLCHAIN=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0) + toolchain=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0) # Install and set as default - rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN - rustup default $TOOLCHAIN + rustup toolchain install --no-self-update --profile minimal "$toolchain" + rustup default "$toolchain" - - name: cargo update compiler & tools - # Remove first line that always just says "Updating crates.io index" - run: | - echo -e "\ncompiler & tools dependencies:" >> cargo_update.log - cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: cargo update library - run: | - echo -e "\nlibrary dependencies:" >> cargo_update.log - cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: cargo update rustbook + - name: run update script + id: update_script + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | - echo -e "\nrustbook dependencies:" >> cargo_update.log - cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log - - name: upload Cargo.lock artifact for use in PR - uses: actions/upload-artifact@v4 - with: - name: Cargo-lock - path: | - Cargo.lock - library/Cargo.lock - src/tools/rustbook/Cargo.lock - retention-days: 1 - - name: upload cargo-update log artifact for use in PR + # Update all lockfiles + ./src/ci/scripts/update-all-lockfiles.py run-update + + # Make a list of all lockfiles that we manage, as well as a list of + # those that already have PRs in the queue. Save this for use in + # later jobs. + all_lockfiles="$(./src/ci/scripts/update-all-lockfiles.py list-all)" + enqueued_lockfiles="$(./src/ci/scripts/update-all-lockfiles.py list-enqueued)" + echo "all_lockfiles=$all_lockfiles" >> "$GITHUB_OUTPUT" + echo "enqueued_lockfiles=$enqueued_lockfiles" >> "$GITHUB_OUTPUT" + + - name: upload output file uses: actions/upload-artifact@v4 with: - name: cargo-updates - path: cargo_update.log + name: update-output + path: update_output.json retention-days: 1 pr: - if: github.repository_owner == 'rust-lang' - name: amend PR - needs: update + name: create or update PR + needs: [update] + strategy: + fail-fast: false + matrix: + name: ${{ fromJson(needs.update.outputs.all_lockfiles) }} + # Don't run the job for a branch if a merge is already in progress. + if: > + github.repository_owner == 'rust-lang' && + !contains(fromJSON(needs.update.outputs.enqueued_lockfiles), matrix.name) runs-on: ubuntu-latest permissions: contents: write @@ -102,35 +81,34 @@ jobs: - name: checkout the source code uses: actions/checkout@v4 - - name: download Cargo.lock from update job - uses: actions/download-artifact@v4 - with: - name: Cargo-lock - - name: download cargo-update log from update job + - name: download output file from update job uses: actions/download-artifact@v4 with: - name: cargo-updates + name: update-output - name: craft PR body and commit message run: | - echo "${COMMIT_MESSAGE}" > commit.txt - cat cargo_update.log >> commit.txt + # Create `commit.txt` and `pr_body.md` + ./src/ci/scripts/update-all-lockfiles.py prepare-pr-files "${{ matrix.name }}" - echo "${PR_MESSAGE}" > body.md - echo '```txt' >> body.md - cat cargo_update.log >> body.md - echo '```' >> body.md + # Set some environment variables from the JSON file + echo "BRANCH=$(jq -r '.${{ matrix.name }}.branch' update_output.json)" >> "$GITHUB_ENV" + echo "PR_TITLE=$(jq -r '.${{ matrix.name }}.pr_title' update_output.json)" >> "$GITHUB_ENV" - name: commit run: | git config user.name github-actions git config user.email github-actions@github.com - git switch --force-create cargo_update - git add ./Cargo.lock ./library/Cargo.lock ./src/tools/rustbook/Cargo.lock + git switch --force-create "$BRANCH" + + # Update and add only the relevant lockfile + ./src/ci/scripts/update-all-lockfiles.py restore-lockfile "${{ matrix.name }}" + git add "$(jq -r '.${{ matrix.name }}.lockfile_path' update_output.json)" + git commit --no-verify --file=commit.txt - name: push - run: git push --no-verify --force --set-upstream origin cargo_update + run: git push --no-verify --force --set-upstream origin "$BRANCH" - name: edit existing open pull request id: edit @@ -140,16 +118,15 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # Exit with error if PR is closed - STATE=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json state --jq '.state') - if [[ "$STATE" != "OPEN" ]]; then - exit 1 - fi + state=$(gh pr view "$BRANCH" --repo "$GITHUB_REPOSITORY" --json state --jq '.state') + [ "$state" != "OPEN" ] && exit 1 - gh pr edit cargo_update --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY + # Replace the title and body with what we generated + gh pr edit "$BRANCH" --title "$PR_TITLE" --body-file pr_body.md --repo "$GITHUB_REPOSITORY" - name: open new pull request # Only run if there wasn't an existing PR if: steps.edit.outcome != 'success' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: gh pr create --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY + run: gh pr create --title "$PR_TITLE" --body-file pr_body.md --repo "$GITHUB_REPOSITORY" diff --git a/src/ci/scripts/update-all-lockfiles.py b/src/ci/scripts/update-all-lockfiles.py new file mode 100755 index 0000000000000..31534d82f399a --- /dev/null +++ b/src/ci/scripts/update-all-lockfiles.py @@ -0,0 +1,249 @@ +#!/usr/bin/env python3 +""" +Tool for updating `Cargo.lock`s in an automated way. + +This is used by the automated weekly update CI job + + +Update multiple lockfiles in the repository and record the changes to +`update_output.json`. +""" + +import json +import os +import subprocess as sp +import sys +from pathlib import Path +from inspect import cleandoc + +LOCKFILES = { + "root": ".", + "library": "library", + "rustbook": "src/tools/rustbook", + "bootstrap": "src/bootstrap", +} + +UPDATE_JSON_OUTFILE = "update_output.json" +COMMIT_TEXT_OUTFILE = "commit.txt" +PR_BODY_OUTFILE = "pr_body.md" + +USAGE = cleandoc( + f""" + Update all lockfiles in the repository + + Usage: ./src/ci/scripts/update-all-lockfiles.py