Check all crates (#12709)

* check all crates individually

It's relevant to check workspace crates individually because otherwise their compilation problems
due to feature misconfigurations won't be caught, as exemplified by
#12705

* adapt to lack of multiple macos runners

#12709 (comment)

* fix cancel-pipeline-cargo-check-each-crate-macos

* fix cargo-check-each-crate-macos again

* time command execution

* fix YAML anchors

* add explanation for rounding division

* ensure the minimum of one crate per group

* collect artifacts for pipeline stopper

* revert hardcoded crates_per_group

* re-add crates_per_group=1

.gitlab-ci.yml

@@ -136,7 +136,7 @@ default:
 # exclude cargo-check-benches from such runs
 .test-refs-check-benches:
  rules:
- - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/ 
+ - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "parent_pipeline" && $CI_IMAGE =~ /staging$/
  when: never
  - if: $CI_PIPELINE_SOURCE == "web"
  - if: $CI_PIPELINE_SOURCE == "schedule"
@@ -329,10 +329,20 @@ cancel-pipeline-test-linux-stable-int:
  needs:
  - job: test-linux-stable-int
 
-cancel-pipeline-cargo-check-subkey:
+cancel-pipeline-cargo-check-each-crate-1:
  extends: .cancel-pipeline-template
  needs:
- - job: cargo-check-subkey
+ - job: "cargo-check-each-crate 1/2"
+
+cancel-pipeline-cargo-check-each-crate-2:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 2/2"
+
+cancel-pipeline-cargo-check-each-crate-macos:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: cargo-check-each-crate-macos
 
 cancel-pipeline-check-tracing:
  extends: .cancel-pipeline-template

scripts/ci/gitlab/check-each-crate.sh

@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+## A script that checks each workspace crate individually.
+## It's relevant to check workspace crates individually because otherwise their compilation problems
+## due to feature misconfigurations won't be caught, as exemplified by
+## https://github.com/paritytech/substrate/issues/12705
+
+set -Eeu -o pipefail
+shopt -s inherit_errexit
+
+set -x
+
+target_group="$1"
+groups_total="$2"
+
+readarray -t workspace_crates < <(\
+ cargo tree --workspace --depth 0 | \
+ awk '{ if (length($1) == 0 || substr($1, 1, 1) == "[") { skip } else { print $1 } }'
+)
+
+crates_total=${#workspace_crates[*]}
+
+if [ "$crates_total" -lt "$groups_total" ]; then
+ # `crates_total / groups_total` would result in 0, so round it up to 1
+ crates_per_group=1
+else
+ # We add `crates_total % groups_total > 0` (which evaluates to 1 in case there's a remainder for
+ # `crates_total % groups_total`) to round UP `crates_total / groups_total` 's
+ # potentially-fractional result to the nearest integer. Doing that ensures that we'll not miss any
+ # crate in case `crates_total / groups_total` would normally result in a fractional number, since
+ # in those cases Bash would round DOWN the result to the nearest integer. For example, if
+ # `crates_total = 5` and `groups_total = 2`, then `crates_total / groups_total` would round down
+ # to 2; since the loop below would then step by 2, we'd miss the 5th crate.
+ crates_per_group=$(( (crates_total / groups_total) + (crates_total % groups_total > 0) ))
+fi
+
+group=1
+for ((i=0; i < crates_total; i += crates_per_group)); do
+ if [ $group -eq "$target_group" ]; then
+ for crate in "${workspace_crates[@]:$i:$crates_per_group}"; do
+ cargo check --locked --release -p "$crate"
+ done
+ break
+ fi
+ group=$(( group + 1 ))
+done

scripts/ci/gitlab/pipeline/build.yml

@@ -90,9 +90,6 @@ build-linux-substrate:
  variables:
  # this variable gets overriden by "rusty-cachier environment inject", use the value as default
  CARGO_TARGET_DIR: "$CI_PROJECT_DIR/target"
- needs:
- - job: cargo-check-subkey
- artifacts: false
  before_script:
  - mkdir -p ./artifacts/subkey
  - !reference [.rusty-cachier, before_script]

scripts/ci/gitlab/pipeline/test.yml

@@ -133,24 +133,8 @@ node-bench-regression-guard:
  --compare-with artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA'
  after_script: [""]
 
-cargo-check-subkey:
- stage: test
- extends:
- - .docker-env
- - .test-refs
- - .pipeline-stopper-artifacts
- script:
- - rusty-cachier snapshot create
- - cd ./bin/utils/subkey
- - SKIP_WASM_BUILD=1 time cargo check --locked --release
- - rusty-cachier cache upload
-
 cargo-check-try-runtime:
  stage: test
- # this is an artificial job dependency, for pipeline optimization using GitLab's DAGs
- needs:
- - job: cargo-check-subkey
- artifacts: false
  extends:
  - .docker-env
  - .test-refs
@@ -393,6 +377,9 @@ test-full-crypto-feature:
 
 test-wasmer-sandbox:
  stage: test
+ needs:
+ - job: cargo-check-wasmer-sandbox
+ artifacts: false
  extends:
  - .docker-env
  - .test-refs-wasmer-sandbox
@@ -409,18 +396,6 @@ test-wasmer-sandbox:
  - time cargo nextest run --locked --release --features runtime-benchmarks,wasmer-sandbox,disable-ui-tests --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL}
  - if [ ${CI_NODE_INDEX} == 1 ]; then rusty-cachier cache upload; fi
 
-cargo-check-macos:
- stage: test
- extends: .test-refs-no-trigger
- before_script:
- - !reference [.rust-info-script, script]
- variables:
- SKIP_WASM_BUILD: 1
- script:
- - time cargo check --locked --release
- tags:
- - osx
-
 check-rustdoc:
  stage: test
  variables:
@@ -435,3 +410,39 @@ check-rustdoc:
  - rusty-cachier snapshot create
  - time cargo +nightly doc --locked --workspace --all-features --verbose --no-deps
  - rusty-cachier cache upload
+
+cargo-check-each-crate:
+ stage: test
+ extends:
+ - .docker-env
+ - .test-refs
+ - .collect-artifacts
+ - .pipeline-stopper-artifacts
+ variables:
+ # $CI_JOB_NAME is set manually so that rusty-cachier can share the cache for all
+ # "cargo-check-each-crate I/N" jobs
+ CI_JOB_NAME: cargo-check-each-crate
+ script:
+ - rusty-cachier snapshot create
+ - time ./scripts/ci/gitlab/check-each-crate.sh "$CI_NODE_INDEX" "$CI_NODE_TOTAL"
+ # need to update cache only from one job
+ - if [ "$CI_NODE_INDEX" == 1 ]; then rusty-cachier cache upload; fi
+ parallel: 2
+
+cargo-check-each-crate-macos:
+ stage: test
+ extends:
+ - .test-refs
+ - .collect-artifacts
+ - .pipeline-stopper-artifacts
+ before_script:
+ - !reference [.rust-info-script, script]
+ - !reference [.pipeline-stopper-vars, script]
+ variables:
+ SKIP_WASM_BUILD: 1
+ script:
+ # TODO: enable rusty-cachier once it supports Mac
+ # TODO: use parallel jobs, as per cargo-check-each-crate, once more Mac runners are available
+ - time ./scripts/ci/gitlab/check-each-crate.sh 1 1
+ tags:
+ - osx