From 8957ce7c14c297b343c0a81aa067fdeda337e558 Mon Sep 17 00:00:00 2001 From: joao-paulo-parity Date: Tue, 15 Nov 2022 11:02:08 -0300 Subject: [PATCH] check all crates individually It's relevant to check workspace crates individually because otherwise their compilation problems due to feature misconfigurations won't be caught, as exemplified by https://github.com/paritytech/substrate/issues/12705 --- .gitlab-ci.yml | 21 +++++++-- Cargo.lock | 2 +- frame/contracts/proc-macro/Cargo.toml | 2 +- scripts/ci/gitlab/check-each-crate.sh | 33 +++++++++++++++ scripts/ci/gitlab/pipeline/build.yml | 3 -- scripts/ci/gitlab/pipeline/test.yml | 61 +++++++++++++++------------ 6 files changed, 86 insertions(+), 36 deletions(-) create mode 100755 scripts/ci/gitlab/check-each-crate.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 75a1d54776eb6..5d94fce852ed2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -136,7 +136,7 @@ default: # exclude cargo-check-benches from such runs .test-refs-check-benches: rules: - - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ + - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ when: never - if: $CI_PIPELINE_SOURCE == "web" - if: $CI_PIPELINE_SOURCE == "schedule" @@ -313,10 +313,25 @@ cancel-pipeline-test-linux-stable-int: needs: - job: test-linux-stable-int -cancel-pipeline-cargo-check-subkey: +cancel-pipeline-cargo-check-each-crate-1: extends: .cancel-pipeline-template needs: - - job: cargo-check-subkey + - job: "cargo-check-each-crate 1/2" + +cancel-pipeline-cargo-check-each-crate-2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate 2/2" + +cancel-pipeline-cargo-check-each-crate-macos-1: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate-macos 1/2" + +cancel-pipeline-cargo-check-each-crate-macos-2: + extends: .cancel-pipeline-template + needs: + - job: "cargo-check-each-crate-macos 2/2" cancel-pipeline-check-tracing: extends: .cancel-pipeline-template diff --git a/Cargo.lock b/Cargo.lock index beb234518041e..239395c95ccab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6329,8 +6329,8 @@ dependencies = [ "frame-support", "frame-system", "pallet-balances", - "pallet-root-testing", "pallet-collective", + "pallet-root-testing", "pallet-timestamp", "parity-scale-codec", "scale-info", diff --git a/frame/contracts/proc-macro/Cargo.toml b/frame/contracts/proc-macro/Cargo.toml index aca1d86ac24a1..08dafcc6a8fca 100644 --- a/frame/contracts/proc-macro/Cargo.toml +++ b/frame/contracts/proc-macro/Cargo.toml @@ -17,7 +17,7 @@ proc-macro = true [dependencies] proc-macro2 = "1" quote = "1" -syn = "1.0.98" +syn = { version = "1.0.98", features = ["full"] } [dev-dependencies] diff --git a/scripts/ci/gitlab/check-each-crate.sh b/scripts/ci/gitlab/check-each-crate.sh new file mode 100755 index 0000000000000..51c4ae7bb931f --- /dev/null +++ b/scripts/ci/gitlab/check-each-crate.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +## A script that checks each workspace crate individually. +## It's relevant to check workspace crates individually because otherwise their compilation problems +## due to feature misconfigurations won't be caught, as exemplified by +## https://github.com/paritytech/substrate/issues/12705 + +set -Eeu -o pipefail +shopt -s inherit_errexit + +set -x + +target_group="$1" +groups_total="$2" + +readarray -t workspace_crates < <(\ + cargo tree --workspace --depth 0 | \ + awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }' +) + +crates_total=${#workspace_crates[*]} +crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) )) + +group=1 +for ((i=0; i < crates_total; i += crates_per_group)); do + if [ $group -eq "$target_group" ]; then + for crate in "${workspace_crates[@]:$i:$crates_per_group}"; do + cargo check --locked --release -p "$crate" + done + break + fi + group=$(( group + 1 )) +done diff --git a/scripts/ci/gitlab/pipeline/build.yml b/scripts/ci/gitlab/pipeline/build.yml index 2e403c10d4520..906c1bcbe242e 100644 --- a/scripts/ci/gitlab/pipeline/build.yml +++ b/scripts/ci/gitlab/pipeline/build.yml @@ -90,9 +90,6 @@ build-linux-substrate: variables: # this variable gets overriden by "rusty-cachier environment inject", use the value as default CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target" - needs: - - job: cargo-check-subkey - artifacts: false before_script: - mkdir -p ./artifacts/subkey - !reference [.rusty-cachier, before_script] diff --git a/scripts/ci/gitlab/pipeline/test.yml b/scripts/ci/gitlab/pipeline/test.yml index 710fbe6227f46..0af79d9658ac9 100644 --- a/scripts/ci/gitlab/pipeline/test.yml +++ b/scripts/ci/gitlab/pipeline/test.yml @@ -133,24 +133,8 @@ node-bench-regression-guard: --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA' after_script: [""] -cargo-check-subkey: - stage: test - extends: - - .docker-env - - .test-refs - - .pipeline-stopper-artifacts - script: - - rusty-cachier snapshot create - - cd ./bin/utils/subkey - - SKIP_WASM_BUILD=1 time cargo check --locked --release - - rusty-cachier cache upload - cargo-check-try-runtime: stage: test - # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs - needs: - - job: cargo-check-subkey - artifacts: false extends: - .docker-env - .test-refs @@ -393,6 +377,10 @@ test-full-crypto-feature: test-wasmer-sandbox: stage: test + # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs + needs: + - job: cargo-check-wasmer-sandbox + artifacts: false extends: - .docker-env - .test-refs-wasmer-sandbox @@ -409,18 +397,6 @@ test-wasmer-sandbox: - time cargo nextest run --locked --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi -cargo-check-macos: - stage: test - extends: .test-refs-no-trigger - before_script: - - !reference [.rust-info-script, script] - variables: - SKIP_WASM_BUILD: 1 - script: - - time cargo check --locked --release - tags: - - osx - check-rustdoc: stage: test variables: @@ -435,3 +411,32 @@ check-rustdoc: - rusty-cachier snapshot create - time cargo +nightly doc --locked --workspace --all-features --verbose --no-deps - rusty-cachier cache upload + +.cargo-check-each-crate_template: &cargo-check-each-crate_template + stage: test + extends: + - .docker-env + - .test-refs + parallel: 2 + +cargo-check-each-crate: + <<: *cargo-check-each-crate_template + variables: + CI_JOB_NAME: cargo-check-each-crate + before_script: + - !reference [.rust-info-script, script] + - !reference [.rusty-cachier, before_script] + script: + - rusty-cachier snapshot create + - ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL" + # need to update cache only from one job + - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi + +cargo-check-each-crate-macos: + <<: *cargo-check-each-crate_template + variables: + CI_JOB_NAME: cargo-check-each-crate-macos + script: + - ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL" + tags: + - osx