diff --git a/.github/workflows/test-cron.yaml b/.github/workflows/test-cron.yaml index 7e42e666e5b..2353a773d7e 100644 --- a/.github/workflows/test-cron.yaml +++ b/.github/workflows/test-cron.yaml @@ -52,7 +52,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: Linux-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: Linux-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -61,20 +61,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'Linux-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'Linux-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - id: get-engine-hash name: Get native engine hash run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" @@ -195,7 +186,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -204,20 +195,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - id: get-engine-hash name: Get native engine hash run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 4277a2f86bf..2a6c1489680 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,7 +55,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: Linux-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: Linux-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -64,20 +64,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'Linux-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'Linux-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - id: get-engine-hash name: Get native engine hash run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" @@ -200,7 +191,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -209,20 +200,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - id: get-engine-hash name: Get native engine hash run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" @@ -415,7 +397,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: macOS10-15-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -424,20 +406,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'macOS10-15-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'macOS10-15-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - env: ARCHFLAGS: -arch x86_64 if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]') @@ -519,7 +492,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: macOS11-ARM64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -528,20 +501,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'macOS11-ARM64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'macOS11-ARM64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - id: get-engine-hash name: Get native engine hash run: 'echo "::set-output name=hash::$(./build-support/bin/rust/print_engine_hash.sh)" @@ -658,7 +622,7 @@ jobs: - name: Cache Rust toolchain uses: actions/cache@v3 with: - key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v1 + key: macOS11-x86_64-rustup-${{ hashFiles('rust-toolchain') }}-v2 path: '~/.rustup/toolchains/1.64.0-* ~/.rustup/update-hashes @@ -667,20 +631,11 @@ jobs: ' - name: Cache Cargo - uses: actions/cache@v3 + uses: benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1 with: - key: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}-${{ hashFiles(''src/rust/engine/Cargo.*'') - }}-v1 - - ' - path: '~/.cargo/registry - - ~/.cargo/git - - ' - restore-keys: 'macOS11-x86_64-cargo-${{ hashFiles(''rust-toolchain'') }}- - - ' + cache-bin: 'false' + shared-key: engine + workspaces: src/rust/engine - env: ARCHFLAGS: -arch x86_64 if: github.event_name == 'push' || !contains(env.COMMIT_MESSAGE, '[ci skip-build-wheels]') diff --git a/build-support/bin/generate_github_workflows.py b/build-support/bin/generate_github_workflows.py index c5efd348154..8b9cd9551e1 100644 --- a/build-support/bin/generate_github_workflows.py +++ b/build-support/bin/generate_github_workflows.py @@ -243,28 +243,6 @@ def install_rustup() -> Step: } -def rust_caches() -> Sequence[Step]: - return [ - { - "name": "Cache Rust toolchain", - "uses": "actions/cache@v3", - "with": { - "path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n", - "key": f"{gha_expr('runner.os')}-rustup-{hashFiles('rust-toolchain')}-v1", - }, - }, - { - "name": "Cache Cargo", - "uses": "actions/cache@v3", - "with": { - "path": "~/.cargo/registry\n~/.cargo/git\n", - "key": f"{gha_expr('runner.os')}-cargo-{hashFiles('rust-toolchain')}-{hashFiles('src/rust/engine/Cargo.*')}-v1\n", - "restore-keys": f"{gha_expr('runner.os')}-cargo-{hashFiles('rust-toolchain')}-\n", - }, - }, - ] - - def install_jdk() -> Step: return { "name": "Install AdoptJDK", @@ -403,16 +381,22 @@ def rust_caches(self) -> Sequence[Step]: "uses": "actions/cache@v3", "with": { "path": f"~/.rustup/toolchains/{rust_channel()}-*\n~/.rustup/update-hashes\n~/.rustup/settings.toml\n", - "key": f"{self.platform_name()}-rustup-{hashFiles('rust-toolchain')}-v1", + "key": f"{self.platform_name()}-rustup-{hashFiles('rust-toolchain')}-v2", }, }, { "name": "Cache Cargo", - "uses": "actions/cache@v3", + "uses": "benjyw/rust-cache@461b9f8eee66b575bce78977bf649b8b7a8d53f1", "with": { - "path": "~/.cargo/registry\n~/.cargo/git\n", - "key": f"{self.platform_name()}-cargo-{hashFiles('rust-toolchain')}-{hashFiles('src/rust/engine/Cargo.*')}-v1\n", - "restore-keys": f"{self.platform_name()}-cargo-{hashFiles('rust-toolchain')}-\n", + # If set, replaces the job id in the cache key, so that the cache is stable across jobs. + # If we don't set this, each job may restore from a previous job's cache entry (via a + # restore key) but will write its own entry, even if there were no rust changes. + # This will cause us to hit the 10GB limit much sooner, and also spend time uploading + # identical cache entries unnecessarily. + "shared-key": "engine", + "workspaces": "src/rust/engine", + # A custom option from our fork of the action. + "cache-bin": "false", }, }, ] @@ -591,6 +575,8 @@ def test_python_linux(shard: str) -> dict[str, Any]: **helper.build_wheels_common, "steps": [ *checkout(containerized=True), + # TODO: Why is this necessary? the hosted container is + # supposed to come with rustup preinstalled. install_rustup(), { "name": "Expose Pythons",