Skip to content

Commit ccfcd95

Browse files
borsbors
committed
Auto merge of rust-lang#123451 - Kobzol:arbitrary-try-build, r=pietroalbini
CI: add a script for dynamically computing CI job matrix It would be great if was easier to run specific CI workflows locally, and also to allow us to spawn a specific CI workflow by bors, to enable running arbitrary try builds. See discussion [here](https://rust-lang.zulipchat.com/#narrow/stream/242791-t-infra/topic/CI.20workflows.20refactoring). This PR is a first step in that direction. - Moves the definition of CI runners and (for now) PR jobs into a separate `jobs.yml` file. - Adds a simple Python script that reads the file, decides which jobs should be active for the current CI workflow, and prints them as JSON to their output. - The PR job then reads this output and generates its job matrix based on it. By moving the job definitions from `ci.yml` into a separate file, we can handle it programmatically, which should make it easier to both do local execution of CI jobs and also to do arbitrary try builds.
2 parents 3493a56 + 72104e2 commit ccfcd95

File tree

5 files changed

+120
-51
lines changed

5 files changed

+120
-51
lines changed

‎.github/workflows/ci.yml

+14-14
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,21 @@ concurrency:
3636
group: "${{ github.workflow }}-${{ ((github.ref == 'refs/heads/try' || github.ref == 'refs/heads/try-perf') && github.sha) || github.ref }}"
3737
cancel-in-progress: true
3838
jobs:
39+
calculate_matrix:
40+
name: Calculate job matrix
41+
runs-on: ubuntu-latest
42+
outputs:
43+
jobs: "${{ steps.jobs.outputs.jobs }}"
44+
steps:
45+
- name: Checkout the source code
46+
uses: actions/checkout@v4
47+
- name: Calculate the CI job matrix
48+
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
49+
id: jobs
3950
pr:
4051
name: "PR - ${{ matrix.name }}"
52+
needs:
53+
- calculate_matrix
4154
env:
4255
PR_CI_JOB: 1
4356
CI_JOB_NAME: "${{ matrix.name }}"
@@ -51,20 +64,7 @@ jobs:
5164
continue-on-error: "${{ matrix.name == 'mingw-check-tidy' }}"
5265
strategy:
5366
matrix:
54-
include:
55-
- name: mingw-check
56-
os: ubuntu-20.04-4core-16gb
57-
env: {}
58-
- name: mingw-check-tidy
59-
os: ubuntu-20.04-4core-16gb
60-
env: {}
61-
- name: x86_64-gnu-llvm-17
62-
env:
63-
ENABLE_GCC_CODEGEN: "1"
64-
os: ubuntu-20.04-16core-64gb
65-
- name: x86_64-gnu-tools
66-
os: ubuntu-20.04-16core-64gb
67-
env: {}
67+
include: "${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}"
6868
defaults:
6969
run:
7070
shell: "${{ contains(matrix.os, 'windows') && 'msys2 {0}' || 'bash' }}"

‎src/ci/github-actions/ci.yml

+15-14
Original file line numberDiff line numberDiff line change
@@ -341,30 +341,31 @@ concurrency:
341341
cancel-in-progress: true
342342

343343
jobs:
344+
# The job matrix for `calculate_matrix` is defined in src/ci/github-actions/jobs.yml.
345+
calculate_matrix:
346+
name: Calculate job matrix
347+
runs-on: ubuntu-latest
348+
outputs:
349+
jobs: ${{ steps.jobs.outputs.jobs }}
350+
steps:
351+
- name: Checkout the source code
352+
uses: actions/checkout@v4
353+
- name: Calculate the CI job matrix
354+
run: python3 src/ci/scripts/calculate-job-matrix.py >> $GITHUB_OUTPUT
355+
id: jobs
344356
pr:
345357
<<: *base-ci-job
346358
name: PR - ${{ matrix.name }}
359+
needs: [ calculate_matrix ]
347360
env:
348361
<<: [*shared-ci-variables, *public-variables]
349362
PR_CI_JOB: 1
350363
if: github.event_name == 'pull_request'
351364
continue-on-error: ${{ matrix.name == 'mingw-check-tidy' }}
352365
strategy:
353366
matrix:
354-
include:
355-
- name: mingw-check
356-
<<: *job-linux-4c
357-
358-
- name: mingw-check-tidy
359-
<<: *job-linux-4c
360-
361-
- name: x86_64-gnu-llvm-17
362-
env:
363-
ENABLE_GCC_CODEGEN: "1"
364-
<<: *job-linux-16c
365-
366-
- name: x86_64-gnu-tools
367-
<<: *job-linux-16c
367+
# Check the `calculate_matrix` job to see how is the matrix defined.
368+
include: ${{ fromJSON(needs.calculate_matrix.outputs.jobs) }}
368369

369370
auto:
370371
<<: *base-ci-job

‎src/ci/github-actions/jobs.yml

+50
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
# This file contains definitions of CI job parameters that are loaded
2+
# dynamically in CI from ci.yml.
3+
# You *do not* need to re-run `src/tools/expand-yaml-anchors` when you
4+
# modify this file.
5+
shared_defs:
6+
- &base-job
7+
env: { }
8+
9+
- &job-linux-4c
10+
os: ubuntu-20.04-4core-16gb
11+
<<: *base-job
12+
13+
- &job-linux-8c
14+
os: ubuntu-20.04-8core-32gb
15+
<<: *base-job
16+
17+
- &job-linux-16c
18+
os: ubuntu-20.04-16core-64gb
19+
<<: *base-job
20+
21+
- &job-macos-xl
22+
os: macos-13 # We use the standard runner for now
23+
<<: *base-job
24+
25+
- &job-macos-m1
26+
os: macos-14
27+
<<: *base-job
28+
29+
- &job-windows-8c
30+
os: windows-2019-8core-32gb
31+
<<: *base-job
32+
33+
- &job-windows-16c
34+
os: windows-2019-16core-64gb
35+
<<: *base-job
36+
37+
- &job-aarch64-linux
38+
os: [ self-hosted, ARM64, linux ]
39+
40+
pr:
41+
- name: mingw-check
42+
<<: *job-linux-4c
43+
- name: mingw-check-tidy
44+
<<: *job-linux-4c
45+
- name: x86_64-gnu-llvm-17
46+
env:
47+
ENABLE_GCC_CODEGEN: "1"
48+
<<: *job-linux-16c
49+
- name: x86_64-gnu-tools
50+
<<: *job-linux-16c

‎src/ci/scripts/calculate-job-matrix.py

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
#!/usr/bin/env python3
2+
3+
"""
4+
This script serves for generating a matrix of jobs that should
5+
be executed on CI.
6+
7+
It reads job definitions from `src/ci/github-actions/jobs.yml`
8+
and filters them based on the event that happened on CI.
9+
10+
Currently, it only supports PR builds.
11+
"""
12+
13+
import json
14+
from pathlib import Path
15+
16+
import yaml
17+
18+
JOBS_YAML_PATH = Path(__file__).absolute().parent.parent / "github-actions" / "jobs.yml"
19+
20+
21+
if __name__ == "__main__":
22+
with open(JOBS_YAML_PATH) as f:
23+
jobs = yaml.safe_load(f)
24+
job_output = jobs["pr"]
25+
print(f"jobs={json.dumps(job_output)}")

‎src/tools/expand-yaml-anchors/src/main.rs

+16-23
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@ use std::error::Error;
22
use std::path::{Path, PathBuf};
33
use yaml_rust::{Yaml, YamlEmitter, YamlLoader};
44

5-
/// List of directories containing files to expand. The first tuple element is the source
6-
/// directory, while the second tuple element is the destination directory.
5+
/// List of files to expand. The first tuple element is the source
6+
/// file, while the second tuple element is the destination file.
77
#[rustfmt::skip]
88
static TO_EXPAND: &[(&str, &str)] = &[
9-
("src/ci/github-actions", ".github/workflows"),
9+
("src/ci/github-actions/ci.yml", ".github/workflows/ci.yml"),
1010
];
1111

1212
/// Name of a special key that will be removed from all the maps in expanded configuration files.
@@ -62,27 +62,20 @@ impl App {
6262
fn run(&self) -> Result<(), Box<dyn Error>> {
6363
for (source, dest) in TO_EXPAND {
6464
let source = self.base.join(source);
65-
let dest = self.base.join(dest);
66-
for entry in std::fs::read_dir(&source)? {
67-
let path = entry?.path();
68-
if !path.is_file() || path.extension().and_then(|e| e.to_str()) != Some("yml") {
69-
continue;
70-
}
71-
72-
let dest_path = dest.join(path.file_name().unwrap());
73-
self.expand(&path, &dest_path).with_context(|| match self.mode {
74-
Mode::Generate => format!(
75-
"failed to expand {} into {}",
76-
self.path(&path),
77-
self.path(&dest_path)
78-
),
79-
Mode::Check => format!(
80-
"{} is not up to date; please run \
65+
let dest_path = self.base.join(dest);
66+
67+
self.expand(&source, &dest_path).with_context(|| match self.mode {
68+
Mode::Generate => format!(
69+
"failed to expand {} into {}",
70+
self.path(&source),
71+
self.path(&dest_path)
72+
),
73+
Mode::Check => format!(
74+
"{} is not up to date; please run \
8175
`x.py run src/tools/expand-yaml-anchors`.",
82-
self.path(&dest_path)
83-
),
84-
})?;
85-
}
76+
self.path(&dest_path)
77+
),
78+
})?;
8679
}
8780
Ok(())
8881
}

0 commit comments

Comments
 (0)