From a6fdd0dfafb5e5d96b2f3e86299fc942ae58812e Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:00:08 +0400 Subject: [PATCH 1/9] paths-filter test --- .github/workflows/tests-linux-stable.yml | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index 8822ba6d250a..b8907b15032a 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -20,10 +20,33 @@ env: FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }} jobs: + + # track changed files + # https://github.com/dorny/paths-filter + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + # Set job outputs to values from filter step + outputs: + backend: ${{ steps.filter.outputs.rust }} + steps: + # For pull requests it's not necessary to checkout the code + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + rust: + - '**' + - '!prdoc/**' + + set-image: # GitHub Actions allows using 'env' in a container context. # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322 # This workaround sets the container image for each job using 'set-image' job output. + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} runs-on: ubuntu-latest outputs: IMAGE: ${{ steps.set_image.outputs.IMAGE }} @@ -32,7 +55,10 @@ jobs: uses: actions/checkout@v4 - id: set_image run: cat .github/env >> $GITHUB_OUTPUT + test-linux-stable-int: + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 needs: [set-image] @@ -50,8 +76,11 @@ jobs: uses: actions/checkout@v4 - name: script run: WASM_BUILD_NO_COLOR=1 time forklift cargo test -p staging-node-cli --release --locked -- --ignored + # https://github.com/paritytech/ci_cd/issues/864 test-linux-stable-runtime-benchmarks: + needs: changes + if: ${{ needs.changes.outputs.rust == 'true' }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 needs: [set-image] From c1f69d7e86bf5c4f0a13dcf2508845fe72c59be8 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:07:44 +0400 Subject: [PATCH 2/9] typo --- .github/workflows/tests-linux-stable.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index b8907b15032a..c1c6f87fa467 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -29,7 +29,7 @@ jobs: pull-requests: read # Set job outputs to values from filter step outputs: - backend: ${{ steps.filter.outputs.rust }} + rust: ${{ steps.filter.outputs.rust }} steps: # For pull requests it's not necessary to checkout the code - uses: dorny/paths-filter@v3 From db25ec5751c131310ece0af5a17cf301f9b5ed70 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:17:59 +0400 Subject: [PATCH 3/9] remove double needs --- .github/workflows/tests-linux-stable.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index c1c6f87fa467..89332e6a14b8 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -57,11 +57,10 @@ jobs: run: cat .github/env >> $GITHUB_OUTPUT test-linux-stable-int: - needs: changes if: ${{ needs.changes.outputs.rust == 'true' }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image] + needs: [set-image, changes] container: image: ${{ needs.set-image.outputs.IMAGE }} env: @@ -79,11 +78,10 @@ jobs: # https://github.com/paritytech/ci_cd/issues/864 test-linux-stable-runtime-benchmarks: - needs: changes if: ${{ needs.changes.outputs.rust == 'true' }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image] + needs: [set-image, changes] container: image: ${{ needs.set-image.outputs.IMAGE }} env: From 298f712c44617712f83fdc87de529da28caf72c2 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:20:05 +0400 Subject: [PATCH 4/9] !.github --- .github/workflows/tests-linux-stable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index 89332e6a14b8..c5edb1e7a3b2 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -39,6 +39,7 @@ jobs: rust: - '**' - '!prdoc/**' + - '!.github/**' set-image: From 46e5964dfa2b332ead40e9be64e4b575c553be91 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:23:47 +0400 Subject: [PATCH 5/9] any --- .github/workflows/tests-linux-stable.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index c5edb1e7a3b2..845be554de2f 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -38,8 +38,8 @@ jobs: filters: | rust: - '**' - - '!prdoc/**' - - '!.github/**' + - '!prdoc/**/*' + - '!.github/**/*' set-image: From 67dfc6538af65e370505351192ccf5b2e86ca76d Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Wed, 22 May 2024 22:26:44 +0400 Subject: [PATCH 6/9] predicate-quantifier --- .github/workflows/tests-linux-stable.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index 845be554de2f..ccbba9c0f3d9 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -35,6 +35,7 @@ jobs: - uses: dorny/paths-filter@v3 id: filter with: + predicate-quantifier: 'every' filters: | rust: - '**' From 78d45900ae5fef931d1bfe31c435620b42cf7d5d Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 28 May 2024 00:26:10 +0400 Subject: [PATCH 7/9] reusable check-changed-files --- .github/workflows/check-changed-files.yml | 35 +++++++++++++++++++++++ .github/workflows/tests-linux-stable.yml | 29 ++++--------------- .github/workflows/tests.yml | 18 ++++++++++-- 3 files changed, 56 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/check-changed-files.yml diff --git a/.github/workflows/check-changed-files.yml b/.github/workflows/check-changed-files.yml new file mode 100644 index 000000000000..a8111c369e01 --- /dev/null +++ b/.github/workflows/check-changed-files.yml @@ -0,0 +1,35 @@ +name: Check changes + +on: + workflow_call: + # Map the workflow outputs to job outputs + outputs: + rust: + value: ${{ jobs.changes.outputs.rust }} + +jobs: + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + # true if current workflow (caller) file is changed + rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }} + steps: + - id: current-file + run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT + - run: echo "${{ steps.current-file.outputs.current-workflow-file }}" + # For pull requests it's not necessary to checkout the code + - id: filter + uses: dorny/paths-filter@v3 + with: + predicate-quantifier: 'every' + # current-workflow - check if the current (caller) workflow file is changed + # rust - check if any Rust (build-related) file is changed + filters: | + current-workflow: + - '${{ steps.current-file.outputs.current-workflow-file }}' + rust: + - '**/*' + - '!.github/**/*' + # \ No newline at end of file diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index ccbba9c0f3d9..5fdfabc437fe 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -21,34 +21,17 @@ env: jobs: - # track changed files - # https://github.com/dorny/paths-filter changes: - runs-on: ubuntu-latest permissions: pull-requests: read - # Set job outputs to values from filter step - outputs: - rust: ${{ steps.filter.outputs.rust }} - steps: - # For pull requests it's not necessary to checkout the code - - uses: dorny/paths-filter@v3 - id: filter - with: - predicate-quantifier: 'every' - filters: | - rust: - - '**' - - '!prdoc/**/*' - - '!.github/**/*' - + uses: ./.github/workflows/check-changed-files.yml set-image: # GitHub Actions allows using 'env' in a container context. # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322 # This workaround sets the container image for each job using 'set-image' job output. needs: changes - if: ${{ needs.changes.outputs.rust == 'true' }} + if: ${{ needs.changes.outputs.rust }} runs-on: ubuntu-latest outputs: IMAGE: ${{ steps.set_image.outputs.IMAGE }} @@ -59,10 +42,10 @@ jobs: run: cat .github/env >> $GITHUB_OUTPUT test-linux-stable-int: - if: ${{ needs.changes.outputs.rust == 'true' }} + needs: [set-image, changes] + if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image, changes] container: image: ${{ needs.set-image.outputs.IMAGE }} env: @@ -80,10 +63,10 @@ jobs: # https://github.com/paritytech/ci_cd/issues/864 test-linux-stable-runtime-benchmarks: - if: ${{ needs.changes.outputs.rust == 'true' }} + needs: [set-image, changes] + if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image, changes] container: image: ${{ needs.set-image.outputs.IMAGE }} env: diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 76bccba86b21..293acadc4e6a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -19,6 +19,12 @@ env: FORKLIFT_metrics_pushEndpoint: ${{ secrets.FORKLIFT_metrics_pushEndpoint }} jobs: + + changes: + permissions: + pull-requests: read + uses: ./.github/workflows/check-changed-files.yml + set-image: # GitHub Actions allows using 'env' in a container context. # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322 @@ -31,10 +37,12 @@ jobs: uses: actions/checkout@v4 - id: set_image run: cat .github/env >> $GITHUB_OUTPUT + quick-benchmarks: + needs: [set-image, changes] + if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image] container: image: ${{ needs.set-image.outputs.IMAGE }} env: @@ -47,11 +55,13 @@ jobs: uses: actions/checkout@v4 - name: script run: time forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet + # cf https://github.com/paritytech/polkadot-sdk/issues/1652 test-syscalls: + needs: [set-image, changes] + if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image] container: image: ${{ needs.set-image.outputs.IMAGE }} continue-on-error: true # this rarely triggers in practice @@ -71,10 +81,12 @@ jobs: # - if [[ "$CI_JOB_STATUS" == "failed" ]]; then # printf "The x86_64 syscalls used by the worker binaries have changed. Please review if this is expected and update polkadot/scripts/list-syscalls/*-worker-syscalls as needed.\n"; # fi + cargo-check-all-benches: + needs: [set-image, changes] + if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 30 - needs: [set-image] container: image: ${{ needs.set-image.outputs.IMAGE }} env: From d48a2097e6d3dbeeb05cc9641633ee15752a3ed4 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 28 May 2024 13:37:24 +0400 Subject: [PATCH 8/9] prdoc --- .github/workflows/check-changed-files.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/check-changed-files.yml b/.github/workflows/check-changed-files.yml index a8111c369e01..ed7487d3f16c 100644 --- a/.github/workflows/check-changed-files.yml +++ b/.github/workflows/check-changed-files.yml @@ -32,4 +32,6 @@ jobs: rust: - '**/*' - '!.github/**/*' + - '!prdoc/**/*' + - '!docs/**/*' # \ No newline at end of file From 4980e528fccbf373ffaffba472f40ec5abd4aed9 Mon Sep 17 00:00:00 2001 From: Evgeny Snitko Date: Tue, 28 May 2024 14:46:45 +0400 Subject: [PATCH 9/9] comments and description --- .github/workflows/check-changed-files.yml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.github/workflows/check-changed-files.yml b/.github/workflows/check-changed-files.yml index ed7487d3f16c..657c05cd047d 100644 --- a/.github/workflows/check-changed-files.yml +++ b/.github/workflows/check-changed-files.yml @@ -1,4 +1,19 @@ -name: Check changes +# Reusable workflow to perform checks and generate conditions for other workflows. +# Currently it checks if any Rust (build-related) file is changed +# and if the current (caller) workflow file is changed. +# Example: +# +# jobs: +# changes: +# permissions: +# pull-requests: read +# uses: ./.github/workflows/check-changed-files.yml +# some-job: +# needs: changes +# if: ${{ needs.changes.outputs.rust }} +# ....... + +name: Check changes files on: workflow_call: @@ -6,6 +21,10 @@ on: outputs: rust: value: ${{ jobs.changes.outputs.rust }} + description: 'true if any of the build-related OR current (caller) workflow files have changed' + current-workflow: + value: ${{ jobs.changes.outputs.current-workflow }} + description: 'true if current (caller) workflow file has changed' jobs: changes: @@ -15,6 +34,7 @@ jobs: outputs: # true if current workflow (caller) file is changed rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }} + current-workflow: ${{ steps.filter.outputs.current-workflow }} steps: - id: current-file run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT