Skip to content

Commit 97ca4d3

Browse files
KobzolKobzol
committed
Use a dynamic environment for PR jobs
1 parent 05f3c0a commit 97ca4d3

File tree

15 files changed

+114
-196
lines changed

15 files changed

+114
-196
lines changed

‎.github/workflows/ci.yml

+8-151
Original file line numberDiff line numberDiff line change
@@ -45,23 +45,19 @@ jobs:
4545
- name: Checkout the source code
4646
uses: actions/checkout@v4
4747
- name: Calculate the CI job matrix
48-
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
48+
env:
49+
GITHUB_CTX: "${{ toJSON(github) }}"
50+
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
4951
id: jobs
50-
pr:
51-
name: "PR - ${{ matrix.name }}"
52+
job:
53+
name: "${{ matrix.name }}"
5254
needs:
5355
- calculate_matrix
5456
env:
55-
PR_CI_JOB: 1
56-
CI_JOB_NAME: "${{ matrix.name }}"
57-
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
58-
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
59-
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
6057
SCCACHE_BUCKET: rust-lang-ci-sccache2
6158
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
6259
CACHE_DOMAIN: ci-caches.rust-lang.org
63-
if: "github.event_name == 'pull_request'"
64-
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
60+
continue-on-error: "${{ matrix.continue_on_error || false }}"
6561
strategy:
6662
matrix:
6763
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
@@ -589,145 +585,6 @@ jobs:
589585
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
590586
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
591587
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
592-
try:
593-
name: "try - ${{ matrix.name }}"
594-
env:
595-
DIST_TRY_BUILD: 1
596-
CI_JOB_NAME: "${{ matrix.name }}"
597-
CARGO_REGISTRIES_CRATES_IO_PROTOCOL: sparse
598-
HEAD_SHA: "${{ github.event.pull_request.head.sha || github.sha }}"
599-
DOCKER_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
600-
SCCACHE_BUCKET: rust-lang-ci-sccache2
601-
DEPLOY_BUCKET: rust-lang-ci2
602-
TOOLSTATE_REPO: "https://github.com/rust-lang-nursery/rust-toolstate"
603-
TOOLSTATE_ISSUES_API_URL: "https://api.github.com/repos/rust-lang/rust/issues"
604-
TOOLSTATE_PUBLISH: 1
605-
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
606-
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
607-
AWS_REGION: us-west-1
608-
CACHE_DOMAIN: ci-caches.rust-lang.org
609-
if: "github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))"
610-
strategy:
611-
matrix:
612-
include:
613-
- name: dist-x86_64-linux
614-
env:
615-
CODEGEN_BACKENDS: "llvm,cranelift"
616-
os: ubuntu-20.04-16core-64gb
617-
defaults:
618-
run:
619-
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"
620-
timeout-minutes: 600
621-
runs-on: "${{ matrix.os }}"
622-
steps:
623-
- if: "contains(matrix.os, 'windows')"
624-
uses: msys2/setup-msys2@v2.22.0
625-
with:
626-
msystem: "${{ contains(matrix.name, 'i686') && 'mingw32' || 'mingw64' }}"
627-
update: false
628-
release: true
629-
path-type: inherit
630-
install: "make dos2unix diffutils\n"
631-
- name: disable git crlf conversion
632-
run: git config --global core.autocrlf false
633-
- name: checkout the source code
634-
uses: actions/checkout@v4
635-
with:
636-
fetch-depth: 2
637-
- name: configure the PR in which the error message will be posted
638-
run: "echo \"[CI_PR_NUMBER=$num]\""
639-
env:
640-
num: "${{ github.event.number }}"
641-
if: "success() && !env.SKIP_JOB && github.event_name == 'pull_request'"
642-
- name: add extra environment variables
643-
run: src/ci/scripts/setup-environment.sh
644-
env:
645-
EXTRA_VARIABLES: "${{ toJson(matrix.env) }}"
646-
if: success() && !env.SKIP_JOB
647-
- name: decide whether to skip this job
648-
run: src/ci/scripts/should-skip-this.sh
649-
if: success() && !env.SKIP_JOB
650-
- name: ensure the channel matches the target branch
651-
run: src/ci/scripts/verify-channel.sh
652-
if: success() && !env.SKIP_JOB
653-
- name: collect CPU statistics
654-
run: src/ci/scripts/collect-cpu-stats.sh
655-
if: success() && !env.SKIP_JOB
656-
- name: show the current environment
657-
run: src/ci/scripts/dump-environment.sh
658-
if: success() && !env.SKIP_JOB
659-
- name: install awscli
660-
run: src/ci/scripts/install-awscli.sh
661-
if: success() && !env.SKIP_JOB
662-
- name: install sccache
663-
run: src/ci/scripts/install-sccache.sh
664-
if: success() && !env.SKIP_JOB
665-
- name: select Xcode
666-
run: src/ci/scripts/select-xcode.sh
667-
if: success() && !env.SKIP_JOB
668-
- name: install clang
669-
run: src/ci/scripts/install-clang.sh
670-
if: success() && !env.SKIP_JOB
671-
- name: install tidy
672-
run: src/ci/scripts/install-tidy.sh
673-
if: success() && !env.SKIP_JOB
674-
- name: install WIX
675-
run: src/ci/scripts/install-wix.sh
676-
if: success() && !env.SKIP_JOB
677-
- name: disable git crlf conversion
678-
run: src/ci/scripts/disable-git-crlf-conversion.sh
679-
if: success() && !env.SKIP_JOB
680-
- name: checkout submodules
681-
run: src/ci/scripts/checkout-submodules.sh
682-
if: success() && !env.SKIP_JOB
683-
- name: install MSYS2
684-
run: src/ci/scripts/install-msys2.sh
685-
if: success() && !env.SKIP_JOB
686-
- name: install MinGW
687-
run: src/ci/scripts/install-mingw.sh
688-
if: success() && !env.SKIP_JOB
689-
- name: install ninja
690-
run: src/ci/scripts/install-ninja.sh
691-
if: success() && !env.SKIP_JOB
692-
- name: enable ipv6 on Docker
693-
run: src/ci/scripts/enable-docker-ipv6.sh
694-
if: success() && !env.SKIP_JOB
695-
- name: disable git crlf conversion
696-
run: src/ci/scripts/disable-git-crlf-conversion.sh
697-
if: success() && !env.SKIP_JOB
698-
- name: ensure line endings are correct
699-
run: src/ci/scripts/verify-line-endings.sh
700-
if: success() && !env.SKIP_JOB
701-
- name: ensure backported commits are in upstream branches
702-
run: src/ci/scripts/verify-backported-commits.sh
703-
if: success() && !env.SKIP_JOB
704-
- name: ensure the stable version number is correct
705-
run: src/ci/scripts/verify-stable-version-number.sh
706-
if: success() && !env.SKIP_JOB
707-
- name: run the build
708-
run: src/ci/scripts/run-build-from-ci.sh 2>&1
709-
env:
710-
AWS_ACCESS_KEY_ID: "${{ env.CACHES_AWS_ACCESS_KEY_ID }}"
711-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.CACHES_AWS_ACCESS_KEY_ID)] }}"
712-
TOOLSTATE_REPO_ACCESS_TOKEN: "${{ secrets.TOOLSTATE_REPO_ACCESS_TOKEN }}"
713-
if: success() && !env.SKIP_JOB
714-
- name: create github artifacts
715-
run: src/ci/scripts/create-doc-artifacts.sh
716-
if: success() && !env.SKIP_JOB
717-
- name: upload artifacts to github
718-
uses: actions/upload-artifact@v4
719-
with:
720-
name: "${{ env.DOC_ARTIFACT_NAME }}"
721-
path: obj/artifacts/doc
722-
if-no-files-found: ignore
723-
retention-days: 5
724-
if: success() && !env.SKIP_JOB
725-
- name: upload artifacts to S3
726-
run: src/ci/scripts/upload-artifacts.sh
727-
env:
728-
AWS_ACCESS_KEY_ID: "${{ env.ARTIFACTS_AWS_ACCESS_KEY_ID }}"
729-
AWS_SECRET_ACCESS_KEY: "${{ secrets[format('AWS_SECRET_ACCESS_KEY_{0}', env.ARTIFACTS_AWS_ACCESS_KEY_ID)] }}"
730-
if: "success() && !env.SKIP_JOB && (github.event_name == 'push' || env.DEPLOY == '1' || env.DEPLOY_ALT == '1')"
731588
master:
732589
name: master
733590
runs-on: ubuntu-latest
@@ -755,7 +612,7 @@ jobs:
755612
if: success() && !env.SKIP_JOB
756613
try-success:
757614
needs:
758-
- try
615+
- job
759616
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
760617
steps:
761618
- name: mark the job as a success
@@ -765,7 +622,7 @@ jobs:
765622
runs-on: ubuntu-latest
766623
try-failure:
767624
needs:
768-
- try
625+
- job
769626
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
770627
steps:
771628
- name: mark the job as a failure

‎library/backtrace

‎library/stdarch

‎src/ci/github-actions/calculate-job-matrix.py

+37-3
Original file line numberDiff line numberDiff line change
@@ -11,15 +11,49 @@
1111
"""
1212

1313
import json
14+
import os
15+
import sys
1416
from pathlib import Path
17+
from typing import List, Dict
1518

1619
import yaml
1720

1821
JOBS_YAML_PATH = Path(__file__).absolute().parent / "jobs.yml"
1922

2023

24+
def name_jobs(jobs: List[Dict], prefix: str) -> List[Dict]:
25+
for job in jobs:
26+
job["name"] = f"{prefix} - {job['image']}"
27+
return jobs
28+
29+
2130
if __name__ == "__main__":
31+
github_ctx = json.loads(os.environ["GITHUB_CTX"])
32+
2233
with open(JOBS_YAML_PATH) as f:
23-
jobs = yaml.safe_load(f)
24-
job_output = jobs["pr"]
25-
print(f"jobs={json.dumps(job_output)}")
34+
data = yaml.safe_load(f)
35+
36+
event_name = github_ctx["event_name"]
37+
ref = github_ctx["ref"]
38+
repository = github_ctx["repository"]
39+
40+
old_bors_try_build = (
41+
ref in ("refs/heads/try", "refs/heads/try-perf") and
42+
repository == "rust-lang-ci/rust"
43+
)
44+
new_bors_try_build = (
45+
ref == "refs/heads/automation/bors/try" and
46+
repository == "rust-lang/rust"
47+
)
48+
try_build = old_bors_try_build or new_bors_try_build
49+
50+
jobs = []
51+
# Pull request CI jobs. Their name is 'PR - <image>'
52+
if event_name == "pull_request":
53+
jobs = name_jobs(data["pr"], "PR")
54+
# Try builds
55+
elif event_name == "push" and try_build:
56+
jobs = name_jobs(data["try"], "try")
57+
58+
print(f"Output:\n{json.dumps(jobs, indent=4)}", file=sys.stderr)
59+
print(f"jobs={json.dumps(jobs)}")

‎src/ci/github-actions/ci.yml

+11-26
Original file line numberDiff line numberDiff line change
@@ -351,17 +351,19 @@ jobs:
351351
- name: Checkout the source code
352352
uses: actions/checkout@v4
353353
- name: Calculate the CI job matrix
354-
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
354+
env:
355+
GITHUB_CTX: ${{ toJSON(github) }}
356+
run: python3 src/ci/github-actions/calculate-job-matrix.py >> $GITHUB_OUTPUT
355357
id: jobs
356-
pr:
358+
job:
357359
<<: *base-ci-job
358-
name: PR - ${{ matrix.name }}
360+
name: ${{ matrix.name }}
359361
needs: [ calculate_matrix ]
360362
env:
361-
<<: [*shared-ci-variables, *public-variables]
362-
PR_CI_JOB: 1
363-
if: github.event_name == 'pull_request'
364-
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
363+
SCCACHE_BUCKET: rust-lang-ci-sccache2
364+
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
365+
CACHE_DOMAIN: ci-caches.rust-lang.org
366+
continue-on-error: ${{ matrix.continue_on_error || false }}
365367
strategy:
366368
matrix:
367369
# Check the `calculate_matrix` job to see how is the matrix defined.
@@ -751,23 +753,6 @@ jobs:
751753
SCRIPT: python x.py dist bootstrap --include-default-paths
752754
<<: *job-windows-8c
753755

754-
try:
755-
<<: *base-ci-job
756-
name: try - ${{ matrix.name }}
757-
env:
758-
DIST_TRY_BUILD: 1
759-
<<: [*shared-ci-variables, *prod-variables]
760-
if: github.event_name == 'push' && (((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust') || ((github.ref == 'refs/heads/automation/bors/try') && github.repository == 'rust-lang/rust'))
761-
strategy:
762-
matrix:
763-
include:
764-
- &dist-x86_64-linux
765-
name: dist-x86_64-linux
766-
env:
767-
CODEGEN_BACKENDS: llvm,cranelift
768-
<<: *job-linux-16c
769-
770-
771756
master:
772757
name: master
773758
runs-on: ubuntu-latest
@@ -791,11 +776,11 @@ jobs:
791776
# build completed, as there is no practical way to detect when a workflow is
792777
# successful listening to webhooks only.
793778
try-success:
794-
needs: [try]
779+
needs: [ job ]
795780
if: "success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
796781
<<: *base-success-job
797782
try-failure:
798-
needs: [try]
783+
needs: [ job ]
799784
if: "!success() && github.event_name == 'push' && (github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.repository == 'rust-lang-ci/rust'"
800785
<<: *base-failure-job
801786
auto-success:

‎src/ci/github-actions/jobs.yml

+47-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
# dynamically in CI from ci.yml.
33
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
44
# modify this file.
5-
shared_defs:
5+
runners:
66
- &base-job
77
env: { }
88

@@ -37,14 +37,56 @@ shared_defs:
3737
- &job-aarch64-linux
3838
os: [ self-hosted, ARM64, linux ]
3939

40+
envs:
41+
- &public-variables
42+
SCCACHE_BUCKET: rust-lang-ci-sccache2
43+
TOOLSTATE_REPO: https://github.com/rust-lang-nursery/rust-toolstate
44+
CACHE_DOMAIN: ci-caches.rust-lang.org
45+
46+
- &prod-variables
47+
<<: *public-variables
48+
DEPLOY_BUCKET: rust-lang-ci2
49+
TOOLSTATE_ISSUES_API_URL: https://api.github.com/repos/rust-lang/rust/issues
50+
TOOLSTATE_PUBLISH: 1
51+
# AWS_SECRET_ACCESS_KEYs are stored in GitHub's secrets storage, named
52+
# AWS_SECRET_ACCESS_KEY_<keyid>. Including the key id in the name allows to
53+
# rotate them in a single branch while keeping the old key in another
54+
# branch, which wouldn't be possible if the key was named with the kind
55+
# (caches, artifacts...).
56+
CACHES_AWS_ACCESS_KEY_ID: AKIA46X5W6CZI5DHEBFL
57+
ARTIFACTS_AWS_ACCESS_KEY_ID: AKIA46X5W6CZN24CBO55
58+
AWS_REGION: us-west-1
59+
60+
- &pr-env
61+
PR_CI_JOB: 1
62+
<<: *public-variables
63+
64+
# Pull request jobs
4065
pr:
41-
- name: mingw-check
66+
- image: mingw-check
67+
env:
68+
<<: *pr-env
4269
<<: *job-linux-4c
43-
- name: mingw-check-tidy
70+
- image: mingw-check-tidy
71+
continue_on_error: true
72+
env:
73+
<<: *pr-env
4474
<<: *job-linux-4c
45-
- name: x86_64-gnu-llvm-17
75+
- image: x86_64-gnu-llvm-17
4676
env:
4777
ENABLE_GCC_CODEGEN: "1"
78+
<<: *pr-env
4879
<<: *job-linux-16c
49-
- name: x86_64-gnu-tools
80+
- image: x86_64-gnu-tools
81+
env:
82+
<<: *pr-env
83+
<<: *job-linux-16c
84+
85+
# Try build jobs
86+
try:
87+
- image: dist-x86_64-linux
88+
env:
89+
DIST_TRY_BUILD: 1
90+
CODEGEN_BACKENDS: llvm,cranelift
91+
<<: *prod-variables
5092
<<: *job-linux-16c

‎src/doc/book

Submodule book updated 28 files

‎src/doc/edition-guide

‎src/doc/embedded-book

‎src/doc/nomicon

0 commit comments

Comments
 (0)