From b0cb245a086d6301b1a4a260ab6c8997f9939ab4 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tmgross@umich.edu>
Date: Sat, 28 Sep 2024 11:17:18 -0400
Subject: [PATCH 1/2] Adjust the weekly dependency update job to create
 separate PRs

Previously there was only one main workspace; within the past few
months, it was split into three (root, library, and rustbook). These
workspaces have somewhat different requirements for what can be updated
and what may be problematic, and making one large PR for everything
doesn't suit this well.

Change the jobs to create separate pull requests for each relevant
update. This will also make it easier to create a distinct job for
bootstrap.
---
 .github/workflows/dependencies.yml     | 131 ++++++-------
 src/ci/scripts/update-all-lockfiles.py | 248 +++++++++++++++++++++++++
 2 files changed, 302 insertions(+), 77 deletions(-)
 create mode 100755 src/ci/scripts/update-all-lockfiles.py

diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
index b7b5a03bd41f3..8492d4d9a85c0 100644
--- a/.github/workflows/dependencies.yml
+++ b/.github/workflows/dependencies.yml
@@ -16,84 +16,63 @@ defaults:
 env:
   # So cargo doesn't complain about unstable features
   RUSTC_BOOTSTRAP: 1
-  PR_TITLE: Weekly `cargo update`
-  PR_MESSAGE: |
-    Automation to keep dependencies in `Cargo.lock` current.
-
-    The following is the output from `cargo update`:
-  COMMIT_MESSAGE: "cargo update \n\n"
 
 jobs:
-  not-waiting-on-bors:
-    if: github.repository_owner == 'rust-lang'
-    name: skip if S-waiting-on-bors
-    runs-on: ubuntu-latest
-    steps:
-      - env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          # Fetch state and labels of PR
-          # Or exit successfully if PR does not exist
-          JSON=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json labels,state || exit 0)
-          STATE=$(echo "$JSON" | jq -r '.state')
-          WAITING_ON_BORS=$(echo "$JSON" | jq '.labels[] | any(.name == "S-waiting-on-bors"; .)')
-
-          # Exit with error if open and S-waiting-on-bors
-          if [[ "$STATE" == "OPEN" && "$WAITING_ON_BORS" == "true" ]]; then
-            exit 1
-          fi
-
   update:
     if: github.repository_owner == 'rust-lang'
     name: update dependencies
-    needs: not-waiting-on-bors
     runs-on: ubuntu-latest
+    outputs:
+      all_lockfiles: ${{ steps.update_script.outputs.all_lockfiles }}
+      enqueued_lockfiles: ${{ steps.update_script.outputs.enqueued_lockfiles }}
     steps:
       - name: checkout the source code
         uses: actions/checkout@v4
         with:
           submodules: recursive
+
       - name: install the bootstrap toolchain
         run: |
           # Extract the stage0 version
-          TOOLCHAIN=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0)
+          toolchain=$(awk -F= '{a[$1]=$2} END {print(a["compiler_version"] "-" a["compiler_date"])}' src/stage0)
           # Install and set as default
-          rustup toolchain install --no-self-update --profile minimal $TOOLCHAIN
-          rustup default $TOOLCHAIN
+          rustup toolchain install --no-self-update --profile minimal "$toolchain"
+          rustup default "$toolchain"
 
-      - name: cargo update compiler & tools
-        # Remove first line that always just says "Updating crates.io index"
-        run: |
-          echo -e "\ncompiler & tools dependencies:" >> cargo_update.log
-          cargo update 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update library
-        run: |
-          echo -e "\nlibrary dependencies:" >> cargo_update.log
-          cargo update --manifest-path library/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: cargo update rustbook
+      - name: run update script
+        id: update_script
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          echo -e "\nrustbook dependencies:" >> cargo_update.log
-          cargo update --manifest-path src/tools/rustbook/Cargo.toml 2>&1 | sed '/crates.io index/d' | tee -a cargo_update.log
-      - name: upload Cargo.lock artifact for use in PR
-        uses: actions/upload-artifact@v4
-        with:
-          name: Cargo-lock
-          path: |
-            Cargo.lock
-            library/Cargo.lock
-            src/tools/rustbook/Cargo.lock
-          retention-days: 1
-      - name: upload cargo-update log artifact for use in PR
+          # Update all lockfiles
+          ./src/ci/scripts/update-all-lockfiles.py run-update
+
+          # Make a list of all lockfiles that we manage, as well as a list of
+          # those that already have PRs in the queue. Save this for use in
+          # later jobs.
+          all_lockfiles="$(./src/ci/scripts/update-all-lockfiles.py list-all)"
+          enqueued_lockfiles="$(./src/ci/scripts/update-all-lockfiles.py list-enqueued)"
+          echo "all_lockfiles=$all_lockfiles" >> "$GITHUB_OUTPUT"
+          echo "enqueued_lockfiles=$enqueued_lockfiles" >> "$GITHUB_OUTPUT"
+
+      - name: upload output file
         uses: actions/upload-artifact@v4
         with:
-          name: cargo-updates
-          path: cargo_update.log
+          name: update-output
+          path: update_output.json
           retention-days: 1
 
   pr:
-    if: github.repository_owner == 'rust-lang'
-    name: amend PR
-    needs: update
+    name: create or update PR
+    needs: [update]
+    strategy:
+      fail-fast: false
+      matrix:
+        name: ${{ fromJson(needs.update.outputs.all_lockfiles) }}
+    # Don't run the job for a branch if a merge is already in progress.
+    if: >
+      github.repository_owner == 'rust-lang' && 
+      !contains(fromJSON(needs.update.outputs.enqueued_lockfiles), matrix.name)
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -102,35 +81,34 @@ jobs:
       - name: checkout the source code
         uses: actions/checkout@v4
 
-      - name: download Cargo.lock from update job
-        uses: actions/download-artifact@v4
-        with:
-          name: Cargo-lock
-      - name: download cargo-update log from update job
+      - name: download output file from update job
         uses: actions/download-artifact@v4
         with:
-          name: cargo-updates
+          name: update-output
 
       - name: craft PR body and commit message
         run: |
-          echo "${COMMIT_MESSAGE}" > commit.txt
-          cat cargo_update.log >> commit.txt
+          # Create `commit.txt` and `pr_body.md`
+          ./src/ci/scripts/update-all-lockfiles.py prepare-pr-files "${{ matrix.name }}"
 
-          echo "${PR_MESSAGE}" > body.md
-          echo '```txt' >> body.md
-          cat cargo_update.log >> body.md
-          echo '```' >> body.md
+          # Set some environment variables from the JSON file
+          echo "BRANCH=$(jq -r '.${{ matrix.name }}.branch' update_output.json)" >> "$GITHUB_ENV"
+          echo "PR_TITLE=$(jq -r '.${{ matrix.name }}.pr_title' update_output.json)" >> "$GITHUB_ENV"
 
       - name: commit
         run: |
           git config user.name github-actions
           git config user.email github-actions@github.com
-          git switch --force-create cargo_update
-          git add ./Cargo.lock ./library/Cargo.lock ./src/tools/rustbook/Cargo.lock
+          git switch --force-create "$BRANCH"
+
+          # Update and add only the relevant lockfile
+          ./src/ci/scripts/update-all-lockfiles.py restore-lockfile "${{ matrix.name }}"
+          git add "$(jq -r '.${{ matrix.name }}.lockfile_path' update_output.json)"
+
           git commit --no-verify --file=commit.txt
 
       - name: push
-        run: git push --no-verify --force --set-upstream origin cargo_update
+        run: git push --no-verify --force --set-upstream origin "$BRANCH"
 
       - name: edit existing open pull request
         id: edit
@@ -140,16 +118,15 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           # Exit with error if PR is closed
-          STATE=$(gh pr view cargo_update --repo $GITHUB_REPOSITORY --json state --jq '.state')
-          if [[ "$STATE" != "OPEN" ]]; then
-            exit 1
-          fi
+          state=$(gh pr view "$BRANCH" --repo "$GITHUB_REPOSITORY" --json state --jq '.state')
+          [ "$state" != "OPEN" ] && exit 1
 
-          gh pr edit cargo_update --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY
+          # Replace the title and body with what we generated
+          gh pr edit "$BRANCH" --title "$PR_TITLE" --body-file pr_body.md --repo "$GITHUB_REPOSITORY"
 
       - name: open new pull request
         # Only run if there wasn't an existing PR
         if: steps.edit.outcome != 'success'
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: gh pr create --title "${PR_TITLE}" --body-file body.md --repo $GITHUB_REPOSITORY
+        run: gh pr create --title "$PR_TITLE" --body-file pr_body.md --repo "$GITHUB_REPOSITORY"
diff --git a/src/ci/scripts/update-all-lockfiles.py b/src/ci/scripts/update-all-lockfiles.py
new file mode 100755
index 0000000000000..ffb7149131372
--- /dev/null
+++ b/src/ci/scripts/update-all-lockfiles.py
@@ -0,0 +1,248 @@
+#!/usr/bin/env python3
+"""
+Tool for updating `Cargo.lock`s in an automated way.
+
+This is used by the automated weekly update CI job
+
+
+Update multiple lockfiles in the repository and record the changes to
+`update_output.json`.
+"""
+
+import json
+import os
+import subprocess as sp
+import sys
+from pathlib import Path
+from inspect import cleandoc
+
+LOCKFILES = {
+    "root": ".",
+    "library": "library",
+    "rustbook": "src/tools/rustbook",
+}
+
+UPDATE_JSON_OUTFILE = "update_output.json"
+COMMIT_TEXT_OUTFILE = "commit.txt"
+PR_BODY_OUTFILE = "pr_body.md"
+
+USAGE = cleandoc(
+    f"""
+    Update all lockfiles in the repository
+
+    Usage: ./src/ci/scripts/update-all-lockfiles.py <OPTION>
+
+    Options:
+        run-update                  Update all lockfiles and save output to
+                                    {UPDATE_JSON_OUTFILE}
+        list-all                    Print a JSON array of identifiers for lockfiles
+        list-enqueued               Print a JSON array of lockfile identifiers that
+                                    are in the process of being merged, so do not need
+                                    to be updated
+        prepare-pr-files <NAME>     Prepare {COMMIT_TEXT_OUTFILE} and {PR_BODY_OUTFILE}
+                                    for branch NAME. Needs {UPDATE_JSON_OUTFILE}.
+        restore-lockfile <NAME>     Restore the lockfile for NAME, using the contents
+                                    and path in {UPDATE_JSON_OUTFILE}
+    """
+)
+
+COMMIT_TEMPLATE = cleandoc(
+    """
+    {name}: run `cargo update`
+
+    Dependency count before update: {old_dep_count}
+    Dependency count after update: {new_dep_count}
+
+    Log:
+
+    ```text
+    {log}
+    ```
+    """
+)
+
+PR_BODY_TEMPLATE = cleandoc(
+    """
+    Automation to keep dependencies in `Cargo.lock` current.
+
+    Dependency count before update: {old_dep_count}
+    Dependency count after update: {new_dep_count}
+
+    The following is the output from `cargo update` in `{path}`:
+
+    ```text
+    {log}
+    ```
+    """
+)
+
+
+def tee(args: list[str | Path]) -> str:
+    """Forward stdout and stderr to the user but also return it"""
+    s = ""
+    proc = sp.Popen(args, stdout=sp.PIPE, stderr=sp.STDOUT, text=True)
+
+    if proc.stdout is None:
+        print("no process stdout")
+        sys.exit(1)
+
+    for line in proc.stdout.readlines():
+        sys.stdout.write(line)
+
+        # Remove first line that just says it is updating the index
+        if "crates.io index" not in line:
+            s += line
+
+    return s.rstrip()
+
+
+def branch_name(name: str) -> str:
+    """Get the name of a branch from a lockfile's name"""
+    return f"cargo-update-{name}"
+
+
+def count_lockfile_dependencies(contents: str) -> int:
+    """Given a lockfile's contents, figure out how many dependencies it contains"""
+    # The default Python version in CI is 3.10, which does not have tomllib. Counting
+    # the number of `[[package]]` entries suffices.
+    return contents.count("[[package]]")
+
+
+def run_update():
+    """Update all lockfiles in the repository, saving output to UPDATE_JSON_OUTFILE"""
+    output = {}
+
+    for name, path in LOCKFILES.items():
+        path = Path(path)
+        manifest_path = path / "Cargo.toml"
+        lockfile_path = path / "Cargo.lock"
+
+        old_dep_count = count_lockfile_dependencies(lockfile_path.read_text())
+
+        log = f"{name} dependencies:"
+        print(log)
+
+        log += "\n"
+        log += tee(["cargo", "update", "--manifest-path", manifest_path])
+
+        print()
+
+        new_lockfile = lockfile_path.read_text()
+        new_dep_count = count_lockfile_dependencies(new_lockfile)
+
+        # Schema of each JSON item
+        single_item = {
+            "branch": branch_name(name),
+            "pr_title": f"Weekly {name} `cargo update`",
+            "path": f"{path}",
+            "old_dep_count": old_dep_count,
+            "new_dep_count": new_dep_count,
+            "lockfile_path": f"{lockfile_path}",
+            "lockfile": new_lockfile,
+            "log": log,
+        }
+
+        output[name] = single_item
+
+    with open(UPDATE_JSON_OUTFILE, "w") as f:
+        json.dump(output, f, indent=4)
+
+    print(f"Wrote output to {UPDATE_JSON_OUTFILE}")
+
+
+def list_all():
+    """List all identifiers (keys) for the lockfiles we update, as a JSON array"""
+    keys = [k for k in LOCKFILES.keys()]
+    print(json.dumps(keys, separators=(",", ":")))
+
+
+def list_enqueued():
+    """List relevant branches that are currently being merged by Bors, as a JSON array"""
+    skip = []
+
+    for name in LOCKFILES.keys():
+        branch = branch_name(name)
+
+        try:
+            output = sp.check_output(
+                [
+                    "gh",
+                    "pr",
+                    "view",
+                    branch,
+                    "--repo",
+                    os.getenv("GITHUB_REPOSITORY", "https://github.com/rust-lang/rust"),
+                    "--json",
+                    "labels,state",
+                ]
+            )
+        except sp.CalledProcessError:
+            # Branch doesn't exist or network error
+            continue
+
+        j = json.loads(output)
+        is_open = j["state"] == "OPEN"
+        waiting_on_bors = any(
+            label["name"] == "S-waiting-on-bors" for label in j["labels"]
+        )
+
+        # No need to create a new PR if there is already one in the queue
+        if is_open and waiting_on_bors:
+            skip.append(name)
+
+    print(json.dumps(skip, separators=(",", ":")))
+
+
+def prepare_pr_files(name: str):
+    """Create a commit message and pull request body"""
+    with open(UPDATE_JSON_OUTFILE) as f:
+        j = json.load(f)
+
+    fmt_dict = j[name]
+    fmt_dict["name"] = name
+
+    commit_text = COMMIT_TEMPLATE.format(**fmt_dict)
+    pr_body_text = PR_BODY_TEMPLATE.format(**fmt_dict)
+
+    Path(COMMIT_TEXT_OUTFILE).write_text(commit_text)
+    print(f"Wrote output to {COMMIT_TEXT_OUTFILE}")
+
+    Path(PR_BODY_OUTFILE).write_text(pr_body_text)
+    print(f"Wrote output to {PR_BODY_OUTFILE}")
+
+
+def restore_lockfile(name: str):
+    """Update a lockfile path with contents from UPDATE_JSON_OUTFILE"""
+    with open(UPDATE_JSON_OUTFILE) as f:
+        j = json.load(f)
+
+    path = Path(j[name]["lockfile_path"])
+    contents = j[name]["lockfile"]
+    path.write_text(contents)
+
+    print(f"Updated lockfile at {path}")
+
+
+def exit_printing_usage():
+    print(USAGE)
+    sys.exit(1)
+
+
+def main():
+    match sys.argv[1:]:
+        case ["run-update"]:
+            run_update()
+        case ["list-all"]:
+            list_all()
+        case ["list-enqueued"]:
+            list_enqueued()
+        case ["prepare-pr-files", name]:
+            prepare_pr_files(name)
+        case ["restore-lockfile", name]:
+            restore_lockfile(name)
+        case _:
+            exit_printing_usage()
+
+
+if __name__ == "__main__":
+    main()

From 14de29c54a92348f7bb99f8f49ebf61a6331ef57 Mon Sep 17 00:00:00 2001
From: Trevor Gross <tmgross@umich.edu>
Date: Sat, 28 Sep 2024 16:03:53 -0400
Subject: [PATCH 2/2] Add `src/bootstrap/Cargo.lock` to the weekly update job

---
 src/ci/scripts/update-all-lockfiles.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/ci/scripts/update-all-lockfiles.py b/src/ci/scripts/update-all-lockfiles.py
index ffb7149131372..31534d82f399a 100755
--- a/src/ci/scripts/update-all-lockfiles.py
+++ b/src/ci/scripts/update-all-lockfiles.py
@@ -20,6 +20,7 @@
     "root": ".",
     "library": "library",
     "rustbook": "src/tools/rustbook",
+    "bootstrap": "src/bootstrap",
 }
 
 UPDATE_JSON_OUTFILE = "update_output.json"