[ci] Add missing components to CI (#1201)

* [ci] Add missing components to CI * add timestamp * add pipeline-stopper-artifacts to jobs * break clippy * mv definition * fix clippy * comment timestamp, rm verbose * commen cancel * rm --verbose from build jobs * disable rusty-cachier before_script * enable timestamp back * rollback .gitignore * rollback .gitignore * rollback .gitignore
paritytech · Aug 28, 2023 · 6e394f8 · 6e394f8
1 parent c168a77
commit 6e394f8
Show file tree

Hide file tree

Showing 4 changed files with 161 additions and 63 deletions.
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,32 +1,9 @@
-# .gitlab-ci.yml
+# polkadot-sdk | CI definitions (via GitLab CI)
 #
-# substrate
+# FYI: Pipelines can be triggered manually through the web UI (if you have enough permissions)
 #
-# pipelines can be triggered manually in the web
-#
-# Currently the file is divided into subfiles. Each stage has a different file which
-# can be found here: gitlab/pipeline/<stage_name>.yml
-#
-# Instead of YAML anchors "extends" is used.
-# Useful links:
-# https://docs.gitlab.com/ee/ci/yaml/index.html#extends
-# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html#reference-tags
-#
-# SAMPLE JOB TEMPLATE - This is not a complete example but is enough to build a
-# simple CI job. For full documentation, visit https://docs.gitlab.com/ee/ci/yaml/
-#
-# my-example-job:
-# stage: test # One of the stages listed below this job (required)
-# image: paritytech/tools:latest # Any docker image (required)
-# allow_failure: true # Allow the pipeline to continue if this job fails (default: false)
-# needs:
-# - job: test-linux # Any jobs that are required to run before this job (optional)
-# variables:
-# MY_ENVIRONMENT_VARIABLE: "some useful value" # Environment variables passed to the job (optional)
-# script:
-# - echo "List of shell commands to run in your job"
-# - echo "You can also just specify a script here, like so:"
-# - ./gitlab/my_amazing_script.sh
+# Currently, entire CI instructions are split into different subfiles. Each CI stage has a corresponding
+# file which can be found here: .gitlab/pipeline/<stage_name>.yml
 
 stages:
  - check
@@ -84,23 +61,26 @@ default:
 
 .prepare-env:
  before_script:
- # TODO: remove unset invocation when we'll be free from 'ENV RUSTC_WRAPPER=sccache' & sccache
- # itself in all images
- - unset RUSTC_WRAPPER
  # $WASM_BUILD_WORKSPACE_HINT enables wasm-builder to find the Cargo.lock from within generated
  # packages
  - export WASM_BUILD_WORKSPACE_HINT="$PWD"
  # ensure that RUSTFLAGS are set correctly
  - echo $RUSTFLAGS
 
+.common-before-script:
+ before_script:
+ - !reference [.job-switcher, before_script]
+ - !reference [.timestamp, before_script]
+ - !reference [.pipeline-stopper-vars, script]
+
 .job-switcher:
  before_script:
  - if echo "$CI_DISABLED_JOBS" | grep -xF "$CI_JOB_NAME"; then echo "The job has been cancelled in CI settings"; exit 0; fi
 
 .kubernetes-env:
  image: "${CI_IMAGE}"
  before_script:
- # - !reference [.job-switcher, before_script]
+ - !reference [.common-before-script, before_script]
  - !reference [.prepare-env, before_script]
  tags:
  - kubernetes-parity-build
@@ -112,9 +92,10 @@ default:
  - rustup +nightly show
  - cargo +nightly --version
 
+# collecting vars for pipeline stopper
+# they will be used if the job fails
 .pipeline-stopper-vars:
  script:
- - !reference [.job-switcher, before_script]
  - echo "Collecting env variables for the cancel-pipeline job"
  - echo "FAILED_JOB_URL=${CI_JOB_URL}" > pipeline-stopper.env
  - echo "FAILED_JOB_NAME=${CI_JOB_NAME}" >> pipeline-stopper.env
@@ -128,28 +109,26 @@ default:
 .docker-env:
  image: "${CI_IMAGE}"
  before_script:
- - !reference [.job-switcher, before_script]
+ - !reference [.common-before-script, before_script]
  - !reference [.prepare-env, before_script]
  - !reference [.rust-info-script, script]
  - !reference [.rusty-cachier, before_script]
- - !reference [.pipeline-stopper-vars, script]
- after_script:
- - !reference [.rusty-cachier, after_script]
  tags:
  - linux-docker-vm-c2
 
 # rusty-cachier's hidden job. Parts of this job are used to instrument the pipeline's other real jobs with rusty-cachier
-# Description of the commands is available here - https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
+# rusty-cachier's commands are described here: https://gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client#description
 .rusty-cachier:
  before_script:
  # - curl -s https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab.parity.io/parity/infrastructure/ci_cd/rusty-cachier/client/-/raw/release/util/install.sh | bash
- # - rusty-cachier environment check --gracefully
- # - $(rusty-cachier environment inject)
- # - rusty-cachier project mtime
+ # - mkdir -p cargo_home cargo_target_dir
+ # - export CARGO_HOME=$CI_PROJECT_DIR/cargo_home
+ # - export CARGO_TARGET_DIR=$CI_PROJECT_DIR/cargo_target_dir
+ # - find . \( -path ./cargo_target_dir -o -path ./cargo_home \) -prune -o -type f -exec touch -t 202005260100 {} +
+ # - git restore-mtime
+ # - rusty-cachier --version
+ # - rusty-cachier project touch-changed
  - echo tbd
- after_script:
- - echo tbd
- # - env RUSTY_CACHIER_SUPRESS_OUTPUT=true rusty-cachier snapshot destroy
 
 .common-refs:
  rules:
@@ -233,6 +212,125 @@ include:
  # zombienet jobs
  - .gitlab/pipeline/zombienet.yml
  # # timestamp handler
- # - project: parity/infrastructure/ci_cd/shared
- # ref: v0.2
- # file: /common/timestamp.yml
+ - project: parity/infrastructure/ci_cd/shared
+ ref: v0.2
+ file: /common/timestamp.yml
+
+# This job cancels the whole pipeline if any of provided jobs fail.
+# In a DAG, every jobs chain is executed independently of others. The `fail_fast` principle suggests
+# to fail the pipeline as soon as possible to shorten the feedback loop.
+.cancel-pipeline-template:
+ stage: .post
+ rules:
+ - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+ when: on_failure
+ variables:
+ PROJECT_ID: "${CI_PROJECT_ID}"
+ PROJECT_NAME: "${CI_PROJECT_NAME}"
+ PIPELINE_ID: "${CI_PIPELINE_ID}"
+ FAILED_JOB_URL: "${FAILED_JOB_URL}"
+ FAILED_JOB_NAME: "${FAILED_JOB_NAME}"
+ PR_NUM: "${PR_NUM}"
+ trigger:
+ project: "parity/infrastructure/ci_cd/pipeline-stopper"
+ branch: "as-improve"
+
+remove-cancel-pipeline-message:
+ stage: .post
+ rules:
+ - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs
+ variables:
+ PROJECT_ID: "${CI_PROJECT_ID}"
+ PROJECT_NAME: "${CI_PROJECT_NAME}"
+ PIPELINE_ID: "${CI_PIPELINE_ID}"
+ FAILED_JOB_URL: "https://gitlab.com"
+ FAILED_JOB_NAME: "nope"
+ PR_NUM: "${CI_COMMIT_REF_NAME}"
+ trigger:
+ project: "parity/infrastructure/ci_cd/pipeline-stopper"
+# need to copy jobs this way because otherwise gitlab will wait
+# for all 3 jobs to finish instead of cancelling if one fails
+cancel-pipeline-test-linux-stable1:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "test-linux-stable 1/3"
+
+cancel-pipeline-test-linux-stable2:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "test-linux-stable 2/3"
+
+cancel-pipeline-test-linux-stable3:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "test-linux-stable 3/3"
+
+cancel-pipeline-test-linux-stable-additional-tests:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "test-linux-stable-additional-tests"
+
+cancel-pipeline-test-linux-stable-slow:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "test-linux-stable-slow"
+
+cancel-pipeline-cargo-check-benches1:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-benches 1/2"
+
+cancel-pipeline-cargo-check-benches2:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-benches 2/2"
+
+cancel-pipeline-test-linux-stable-int:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: test-linux-stable-int
+
+cancel-pipeline-cargo-check-each-crate-1:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 1/6"
+
+cancel-pipeline-cargo-check-each-crate-2:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 2/6"
+
+cancel-pipeline-cargo-check-each-crate-3:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 3/6"
+
+cancel-pipeline-cargo-check-each-crate-4:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 4/6"
+
+cancel-pipeline-cargo-check-each-crate-5:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 5/6"
+
+cancel-pipeline-cargo-check-each-crate-6:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: "cargo-check-each-crate 6/6"
+
+cancel-pipeline-cargo-check-each-crate-macos:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: cargo-check-each-crate-macos
+
+cancel-pipeline-check-tracing:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: check-tracing
+
+cancel-pipeline-cargo-clippy:
+ extends: .cancel-pipeline-template
+ needs:
+ - job: cargo-clippy
diff --git a/.gitlab/pipeline/build.yml b/.gitlab/pipeline/build.yml
@@ -46,8 +46,8 @@ build-test-collators:
  - .run-immediately
  - .collect-artifacts
  script:
- - time cargo build --locked --profile testnet --verbose -p test-parachain-adder-collator
- - time cargo build --locked --profile testnet --verbose -p test-parachain-undying-collator
+ - time cargo build --locked --profile testnet -p test-parachain-adder-collator
+ - time cargo build --locked --profile testnet -p test-parachain-undying-collator
  # pack artifacts
  - mkdir -p ./artifacts
  - mv ./target/testnet/adder-collator ./artifacts/.
@@ -66,7 +66,7 @@ build-malus:
  - .run-immediately
  - .collect-artifacts
  script:
- - time cargo build --locked --profile testnet --verbose -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
+ - time cargo build --locked --profile testnet -p polkadot-test-malus --bin malus --bin polkadot-prepare-worker --bin polkadot-execute-worker
  # pack artifacts
  - mkdir -p ./artifacts
  - mv ./target/testnet/malus ./artifacts/.
@@ -111,7 +111,7 @@ build-rustdoc:
  script:
  # FIXME: it fails with `RUSTDOCFLAGS="-Dwarnings"` and `--all-features`
  # FIXME: return to stable when https://github.com/rust-lang/rust/issues/96937 gets into stable
- - time cargo doc --workspace --verbose --no-deps
+ - time cargo doc --workspace --no-deps
  - rm -f ./target/doc/.lock
  - mv ./target/doc ./crate-docs
  # FIXME: remove me after CI image gets nonroot
@@ -280,7 +280,7 @@ build-linux-substrate:
  # see https://github.com/paritytech/ci_cd/issues/682#issuecomment-1340953589
  - git checkout -B "$CI_COMMIT_REF_NAME" "$CI_COMMIT_SHA"
  script:
- - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release --verbose
+ - WASM_BUILD_NO_COLOR=1 time cargo build --locked --release
  - mv $CARGO_TARGET_DIR/release/substrate-node ./artifacts/substrate/substrate
  - echo -n "Substrate version = "
  - if [ "${CI_COMMIT_TAG}" ]; then
@@ -308,7 +308,7 @@ build-linux-substrate:
  - mkdir -p ./artifacts/subkey
  script:
  - cd ./substrate/bin/utils/subkey
- - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
+ - SKIP_WASM_BUILD=1 time cargo build --locked --release
  # - cd -
  # - mv $CARGO_TARGET_DIR/release/subkey ./artifacts/subkey/.
  # - echo -n "Subkey version = "
@@ -331,7 +331,7 @@ build-subkey-linux:
 # - mkdir -p ./artifacts/subkey
 # script:
 # - cd ./bin/utils/subkey
-# - SKIP_WASM_BUILD=1 time cargo build --locked --release --verbose
+# - SKIP_WASM_BUILD=1 time cargo build --locked --release
 # - cd -
 # - mv ./target/release/subkey ./artifacts/subkey/.
 # - echo -n "Subkey version = "

diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml
@@ -3,10 +3,9 @@ cargo-clippy:
  extends:
  - .docker-env
  - .common-refs
+ - .pipeline-stopper-artifacts
  script:
  - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo clippy --all-targets --locked --workspace
- # fixme!
- allow_failure: true
 
 check-try-runtime:
  stage: check
@@ -43,7 +42,6 @@ cargo-deny-licenses:
  script:
  - $CARGO_DENY_CMD --hide-inclusion-graph
  after_script:
- # - !reference [.rusty-cachier, after_script]
  - echo "___The complete log is in the artifacts___"
  - $CARGO_DENY_CMD 2> deny.log
  - if [ $CI_JOB_STATUS != 'success' ]; then