From 83cf44b78d97bb91e14f3c4e5e982990d6954568 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 14:11:17 +1000 Subject: [PATCH 1/9] Add success job with completeness check --- .github/workflows/test-suite.yml | 39 ++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 27c91f22620..f79aa465714 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -400,3 +400,42 @@ jobs: run: rustup override set beta - name: Run make run: make + # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether + # a PR is safe to merge. New jobs should be added here. + test-suite-success: + name: test-suite-success + runs-on: ubuntu-latest + needs: [ + 'target-branch-check', + 'extract-msrv', + 'cargo-fmt', + 'release-tests-ubuntu', + 'release-tests-windows', + 'beacon-chain-tests', + 'op-pool-tests', + 'slasher-tests', + 'debug-tests-ubuntu', + 'state-transition-vectors-ubuntu', + 'ef-tests-ubuntu', + 'dockerfile-ubuntu', + 'eth1-simulator-ubuntu', + 'merge-transition-ubuntu', + 'no-eth1-simulator-ubuntu', + 'syncing-simulator-ubuntu', + 'doppelganger-protection-test', + 'execution-engine-integration-ubuntu', + 'check-benchmarks', + 'clippy', + 'check-msrv', + 'arbitrary-check', + 'cargo-audit', + 'cargo-vendor', + 'cargo-udeps', + 'compile-with-beta-compiler' + ] + steps: + - name: Check completeness + run: | + yq '... comments="" | .jobs | map(. | key)' < .github/workflows/test-suite.yml | grep -v "test-suite-success" | sort > all_jobs.txt + echo "${{ needs }}" | jq -r 'keys | join(",")' | tr "," "\n" | sort > dep_jobs.txt + diff all_jobs.txt dep_jobs.txt From 19221628bef062534475af544cf31b3a17ca7aa8 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 14:13:14 +1000 Subject: [PATCH 2/9] Try quick run --- .github/workflows/test-suite.yml | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index f79aa465714..5da6f5878dd 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -407,31 +407,8 @@ jobs: runs-on: ubuntu-latest needs: [ 'target-branch-check', - 'extract-msrv', 'cargo-fmt', - 'release-tests-ubuntu', - 'release-tests-windows', - 'beacon-chain-tests', - 'op-pool-tests', - 'slasher-tests', - 'debug-tests-ubuntu', - 'state-transition-vectors-ubuntu', - 'ef-tests-ubuntu', - 'dockerfile-ubuntu', - 'eth1-simulator-ubuntu', - 'merge-transition-ubuntu', - 'no-eth1-simulator-ubuntu', - 'syncing-simulator-ubuntu', - 'doppelganger-protection-test', - 'execution-engine-integration-ubuntu', - 'check-benchmarks', 'clippy', - 'check-msrv', - 'arbitrary-check', - 'cargo-audit', - 'cargo-vendor', - 'cargo-udeps', - 'compile-with-beta-compiler' ] steps: - name: Check completeness From b83c5868c603494b46eb89048111093ba99dec56 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 14:40:22 +1000 Subject: [PATCH 3/9] Fixes --- .github/workflows/test-suite.yml | 7 ++----- scripts/ci/check-success-job.sh | 12 ++++++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) create mode 100755 scripts/ci/check-success-job.sh diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 5da6f5878dd..1dd38456e6f 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -408,11 +408,8 @@ jobs: needs: [ 'target-branch-check', 'cargo-fmt', - 'clippy', ] steps: + - uses: actions/checkout@v3 - name: Check completeness - run: | - yq '... comments="" | .jobs | map(. | key)' < .github/workflows/test-suite.yml | grep -v "test-suite-success" | sort > all_jobs.txt - echo "${{ needs }}" | jq -r 'keys | join(",")' | tr "," "\n" | sort > dep_jobs.txt - diff all_jobs.txt dep_jobs.txt + run: ./scripts/ci/check-success-job.sh ./github/workflows/test-suite.yml "${{ needs }}" test-suite-success diff --git a/scripts/ci/check-success-job.sh b/scripts/ci/check-success-job.sh new file mode 100755 index 00000000000..51f992f7a25 --- /dev/null +++ b/scripts/ci/check-success-job.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +set -euf -o pipefail + +YAML=$1 +NEEDS=$2 +SUCCESS_JOB=$3 + +yq '... comments="" | .jobs | map(. | key)' < "$YAML" | grep -v "$SUCCESS_JOB" | sort > all_jobs.txt +echo "$NEEDS" | jq -r 'keys | join(",")' | tr "," "\n" | sort > dep_jobs.txt +diff all_jobs.txt dep_jobs.txt +rm all_jobs.txt dep_jobs.txt From 843206d89eac4a4cf7d481fce062335d1666f88a Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 14:52:05 +1000 Subject: [PATCH 4/9] Expanding `needs` doesn't work, so inspect more --- .github/workflows/test-suite.yml | 4 ++-- scripts/ci/check-success-job.sh | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 1dd38456e6f..d3a11aa1b28 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -411,5 +411,5 @@ jobs: ] steps: - uses: actions/checkout@v3 - - name: Check completeness - run: ./scripts/ci/check-success-job.sh ./github/workflows/test-suite.yml "${{ needs }}" test-suite-success + - name: Check that success job is dependent on all others + run: ./scripts/ci/check-success-job.sh ./github/workflows/test-suite.yml test-suite-success diff --git a/scripts/ci/check-success-job.sh b/scripts/ci/check-success-job.sh index 51f992f7a25..42edd019e27 100755 --- a/scripts/ci/check-success-job.sh +++ b/scripts/ci/check-success-job.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash +# Check that $SUCCESS_JOB depends on all other jobs in the given $YAML set -euf -o pipefail YAML=$1 -NEEDS=$2 -SUCCESS_JOB=$3 +SUCCESS_JOB=$2 -yq '... comments="" | .jobs | map(. | key)' < "$YAML" | grep -v "$SUCCESS_JOB" | sort > all_jobs.txt -echo "$NEEDS" | jq -r 'keys | join(",")' | tr "," "\n" | sort > dep_jobs.txt -diff all_jobs.txt dep_jobs.txt +yq '... comments="" | .jobs | map(. | key)' < "$YAML" | sed "s/^- //"| grep -v "$SUCCESS_JOB" | sort > all_jobs.txt +yq "... comments=\"\" | .jobs.$SUCCESS_JOB.needs[]" < "$YAML" | grep -v "test-suite-success" | sort > dep_jobs.txt +diff all_jobs.txt dep_jobs.txt || echo "COMPLETENESS CHECK FAILED" && exit 1 rm all_jobs.txt dep_jobs.txt From 25476a9ef32673aec03cb2c7ef5aea775f43aaa3 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 14:54:35 +1000 Subject: [PATCH 5/9] Minor cleanups --- .github/workflows/test-suite.yml | 2 +- scripts/ci/check-success-job.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index d3a11aa1b28..13e0abed871 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -412,4 +412,4 @@ jobs: steps: - uses: actions/checkout@v3 - name: Check that success job is dependent on all others - run: ./scripts/ci/check-success-job.sh ./github/workflows/test-suite.yml test-suite-success + run: ./scripts/ci/check-success-job.sh ./.github/workflows/test-suite.yml test-suite-success diff --git a/scripts/ci/check-success-job.sh b/scripts/ci/check-success-job.sh index 42edd019e27..de92f845e28 100755 --- a/scripts/ci/check-success-job.sh +++ b/scripts/ci/check-success-job.sh @@ -6,7 +6,7 @@ set -euf -o pipefail YAML=$1 SUCCESS_JOB=$2 -yq '... comments="" | .jobs | map(. | key)' < "$YAML" | sed "s/^- //"| grep -v "$SUCCESS_JOB" | sort > all_jobs.txt -yq "... comments=\"\" | .jobs.$SUCCESS_JOB.needs[]" < "$YAML" | grep -v "test-suite-success" | sort > dep_jobs.txt +yq '... comments="" | .jobs | map(. | key) | .[]' < "$YAML" | grep -v "$SUCCESS_JOB" | sort > all_jobs.txt +yq "... comments=\"\" | .jobs.$SUCCESS_JOB.needs[]" < "$YAML" | grep -v "$SUCCESS_JOB" | sort > dep_jobs.txt diff all_jobs.txt dep_jobs.txt || echo "COMPLETENESS CHECK FAILED" && exit 1 rm all_jobs.txt dep_jobs.txt From 3afeadf044b0854dfe852abaddf55e988a9aecce Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 15:11:29 +1000 Subject: [PATCH 6/9] Use complete list of deps again, fix script This reverts commit 19221628bef062534475af544cf31b3a17ca7aa8. --- .github/workflows/test-suite.yml | 24 ++++++++++++++++++++++++ scripts/ci/check-success-job.sh | 3 ++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 13e0abed871..6780bf4e027 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -407,7 +407,31 @@ jobs: runs-on: ubuntu-latest needs: [ 'target-branch-check', + 'extract-msrv', 'cargo-fmt', + 'release-tests-ubuntu', + 'release-tests-windows', + 'beacon-chain-tests', + 'op-pool-tests', + 'slasher-tests', + 'debug-tests-ubuntu', + 'state-transition-vectors-ubuntu', + 'ef-tests-ubuntu', + 'dockerfile-ubuntu', + 'eth1-simulator-ubuntu', + 'merge-transition-ubuntu', + 'no-eth1-simulator-ubuntu', + 'syncing-simulator-ubuntu', + 'doppelganger-protection-test', + 'execution-engine-integration-ubuntu', + 'check-benchmarks', + 'clippy', + 'check-msrv', + 'arbitrary-check', + 'cargo-audit', + 'cargo-vendor', + 'cargo-udeps', + 'compile-with-beta-compiler' ] steps: - uses: actions/checkout@v3 diff --git a/scripts/ci/check-success-job.sh b/scripts/ci/check-success-job.sh index de92f845e28..dfa5c03257c 100755 --- a/scripts/ci/check-success-job.sh +++ b/scripts/ci/check-success-job.sh @@ -8,5 +8,6 @@ SUCCESS_JOB=$2 yq '... comments="" | .jobs | map(. | key) | .[]' < "$YAML" | grep -v "$SUCCESS_JOB" | sort > all_jobs.txt yq "... comments=\"\" | .jobs.$SUCCESS_JOB.needs[]" < "$YAML" | grep -v "$SUCCESS_JOB" | sort > dep_jobs.txt -diff all_jobs.txt dep_jobs.txt || echo "COMPLETENESS CHECK FAILED" && exit 1 +diff all_jobs.txt dep_jobs.txt || (echo "COMPLETENESS CHECK FAILED" && exit 1) rm all_jobs.txt dep_jobs.txt +echo "COMPLETENESS CHECK PASSED" From cdadf4dd599ff5bc37c0441d70365615522a3465 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 15:28:54 +1000 Subject: [PATCH 7/9] Completeness check for local-testnet --- .github/workflows/local-testnet.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/local-testnet.yml b/.github/workflows/local-testnet.yml index 1ca1006c1f2..bb180350650 100644 --- a/.github/workflows/local-testnet.yml +++ b/.github/workflows/local-testnet.yml @@ -70,3 +70,11 @@ jobs: - name: Stop local testnet with blinded block production run: ./stop_local_testnet.sh working-directory: scripts/local_testnet + local-testnet-success: + name: test-suite-success + runs-on: ubuntu-latest + needs: ["run-local-testnet"] + steps: + - uses: actions/checkout@v3 + - name: Check that success job is dependent on all others + run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success From 0b1aa021c0d44065c6c21c740744a13c88e88c12 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Tue, 2 May 2023 17:12:58 +1000 Subject: [PATCH 8/9] Update .github/workflows/local-testnet.yml Co-authored-by: Paul Hauner --- .github/workflows/local-testnet.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/local-testnet.yml b/.github/workflows/local-testnet.yml index bb180350650..0f7bc79ebaa 100644 --- a/.github/workflows/local-testnet.yml +++ b/.github/workflows/local-testnet.yml @@ -71,7 +71,7 @@ jobs: run: ./stop_local_testnet.sh working-directory: scripts/local_testnet local-testnet-success: - name: test-suite-success + name: local-testnet-success runs-on: ubuntu-latest needs: ["run-local-testnet"] steps: From 9fec0d1f131275bb0fa55a9e5fb0b453bdf23458 Mon Sep 17 00:00:00 2001 From: Michael Sproul Date: Wed, 3 May 2023 10:24:01 +1000 Subject: [PATCH 9/9] Make target branch check pass on merge group CI --- .github/workflows/test-suite.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test-suite.yml b/.github/workflows/test-suite.yml index 6780bf4e027..152f3515fdd 100644 --- a/.github/workflows/test-suite.yml +++ b/.github/workflows/test-suite.yml @@ -24,7 +24,7 @@ jobs: if: github.event_name == 'pull_request' || github.event_name == 'merge_group' steps: - name: Check that the pull request is not targeting the stable branch - run: test ${{ github.base_ref }} != "stable" + run: test "${{ github.base_ref }}" != "stable" extract-msrv: runs-on: ubuntu-latest steps: