Feature/update to polkadot 0.8.30 substrate

.github/workflows/build-and-push-docker-image.yml

@@ -36,9 +36,9 @@ jobs:
       - name: Copy wasm artifact from the image
         run: |
           container_id=$(docker create ${{ env.IMAGE_ID }})
-          artifact_name=node_runtime.compact.${GITHUB_SHA:0:7}.wasm
+          artifact_name=node_runtime.compact.compressed.${GITHUB_SHA:0:7}.wasm
           echo ARTIFACT_NAME=$artifact_name >> $GITHUB_ENV
-          docker cp $container_id:/home/cere/node-runtime-artifacts/node_runtime.compact.wasm ./$artifact_name
+          docker cp $container_id:/home/cere/node-runtime-artifacts/node_runtime.compact.compressed.wasm ./$artifact_name
       - name: Upload wasm artifact
         uses: actions/upload-artifact@v3
         with:

.github/workflows/check-labels.yml

@@ -9,8 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
         run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}

.gitignore

@@ -24,3 +24,4 @@ rls*.log
 .cargo/
 .cargo-remote.toml
 *.bin
+*.iml

.gitlab-ci.yml

@@ -119,6 +119,11 @@ default:
     - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
+.nightly-pipeline:                 &nightly-pipeline
+  rules:
+    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
+    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+
 #### stage:                       .pre
 
 skip-if-draft:
@@ -199,17 +204,7 @@ test-prometheus-alerting-rules:
 cargo-deny:
   stage:                           test
   <<:                              *docker-env
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-      when: never
-    - changes:
-      - "Cargo.lock"
-      - "**/Cargo.toml"
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+  <<:                              *nightly-pipeline
   script:
     - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
   after_script:
@@ -277,6 +272,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -468,6 +464,30 @@ build-macos-subkey:
   tags:
     - osx
 
+build-rust-doc:
+  stage:                           build
+  <<:                              *docker-env
+  <<:                              *test-refs
+  needs:
+    - job:                         test-linux-stable
+      artifacts:                   false
+  variables:
+    <<:                            *default-vars
+    SKIP_WASM_BUILD:               1
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}-doc"
+    when:                          on_success
+    expire_in:                     7 days
+    paths:
+    - ./crate-docs/
+  script:
+    - RUSTDOCFLAGS="--html-in-header $(pwd)/.maintain/rustdoc-header.html"
+        time cargo +nightly doc --no-deps --workspace --all-features --verbose
+    - mv ./target/doc ./crate-docs
+    - echo "<meta http-equiv=refresh content=0;url=sc_service/index.html>" > ./crate-docs/index.html
+    - sccache -s
+  allow_failure:                   true
+
 #### stage:                        publish
 
 .build-push-docker-image:          &build-push-docker-image
@@ -499,6 +519,10 @@ build-macos-subkey:
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
+    # pass artifacts to the trigget-simnet job
+    - echo "VERSION=${VERSION}" > build.env
+    - echo "TRIGGERER=${CI_PROJECT_NAME}" >> build.env
+  after_script:
     - buildah logout "$IMAGE_NAME"
 
 publish-docker-substrate:
@@ -510,8 +534,6 @@ publish-docker-substrate:
   variables:
     <<:                            *docker-build-vars
     PRODUCT:                       substrate
-  after_script:
-    - echo "VERSION=${VERSION}" >> build.env
   artifacts:
     reports:
       # this artifact is used in trigger-simnet job
@@ -550,6 +572,32 @@ publish-s3-release:
     - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/
         --recursive --human-readable --summarize
 
+publish-s3-doc:
+  stage:                           publish
+  image:                           paritytech/awscli:latest
+  allow_failure:                   true
+  needs:
+    - job:                         build-rust-doc
+      artifacts:                   true
+    - job:                         build-linux-substrate
+      artifacts:                   false
+  <<:                              *publish-refs
+  <<:                              *kubernetes-build
+  variables:
+    GIT_STRATEGY:                  none
+    BUCKET:                        "releases.parity.io"
+    PREFIX:                        "substrate-rustdoc"
+  script:
+    - test -r ./crate-docs/index.html || (
+        echo "./crate-docs/index.html not present, build:rust:doc:release job not complete";
+        exit 1
+      )
+    - aws s3 sync --delete --size-only --only-show-errors
+        ./crate-docs/ s3://${BUCKET}/${PREFIX}/
+  after_script:
+    - aws s3 ls s3://${BUCKET}/${PREFIX}/
+        --human-readable --summarize
+
 publish-draft-release:
   stage:                           publish
   image:                           paritytech/tools:latest
@@ -603,9 +651,7 @@ deploy-prometheus-alerting-rules:
 
 trigger-simnet:
   stage:                           deploy
-  rules:
-    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+  <<:                              *nightly-pipeline
   needs:
     - job:                         publish-docker-substrate
   trigger:

.maintain/common/lib.sh

@@ -82,7 +82,7 @@ has_label(){
 
 # Formats a message into a JSON string for posting to Matrix
 # message: 'any plaintext message'
-# formatted_message: '<strong>optional message formatted in <em>html</em></strong>' 
+# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
 # Usage: structure_message $content $formatted_content (optional)
 structure_message() {
   if [ -z "$2" ]; then
@@ -101,3 +101,17 @@ structure_message() {
 send_message() {
 curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
 }
+
+# Check for runtime changes between two commits. This is defined as any changes
+# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
+has_runtime_changes() {
+  from=$1
+  to=$2
+  if git diff --name-only "${from}...${to}" \
+    | grep -q -e '^frame/' -e '^primitives/'
+  then
+    return 0
+  else
+    return 1
+  fi
+}

.maintain/frame-weight-template.hbs

@@ -18,7 +18,7 @@
 //! Autogenerated weights for {{pallet}}
 //!
 //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION {{version}}
-//! DATE: {{date}}, STEPS: {{cmd.steps}}, REPEAT: {{cmd.repeat}}, LOW RANGE: {{cmd.lowest_range_values}}, HIGH RANGE: {{cmd.highest_range_values}}
+//! DATE: {{date}}, STEPS: `{{cmd.steps}}`, REPEAT: {{cmd.repeat}}, LOW RANGE: `{{cmd.lowest_range_values}}`, HIGH RANGE: `{{cmd.highest_range_values}}`
 //! EXECUTION: {{cmd.execution}}, WASM-EXECUTION: {{cmd.wasm_execution}}, CHAIN: {{cmd.chain}}, DB CACHE: {{cmd.db_cache}}
 
 // Executed Command:

.maintain/github/check_labels.sh

@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -e
 
 #shellcheck source=../common/lib.sh
 source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
@@ -24,10 +25,18 @@ releasenotes_labels=(
 )
 
 criticality_labels=(
-  'C1-low'
-  'C3-medium'
-  'C7-high'
-  'C9-critical'
+  'C1-low 📌'
+  'C3-medium 📣'
+  'C7-high ❗️'
+  'C9-critical ‼️'
+)
+
+audit_labels=(
+  'D1-audited 👍'
+  'D2-notlive 💤'
+  'D3-trivial 🧸'
+  'D5-nicetohaveaudit ⚠️'
+  'D9-needsaudit 👮'
 )
 
 echo "[+] Checking release notes (B) labels"
@@ -46,4 +55,14 @@ else
   exit 1
 fi
 
+if has_runtime_changes origin/master "${HEAD_SHA}"; then
+  echo "[+] Runtime changes detected. Checking audit (D) labels"
+  if ensure_labels "${audit_labels[@]}";  then
+    echo "[+] Release audit label detected. All is well."
+  else
+    echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
+    exit 1
+  fi
+fi
+
 exit 0

.maintain/gitlab/check_polkadot_companion_build.sh

@@ -86,10 +86,9 @@ else
 fi
 
 # Patch all Substrate crates in Polkadot
-diener patch --crates-to-patch ../ --substrate
+diener patch --crates-to-patch ../ --substrate --path Cargo.toml
 
 # Test Polkadot pr or master branch with this Substrate commit.
-cargo update -p sp-io
 time cargo test --all --release --verbose --features=real-overseer
 
 cd parachain/test-parachains/adder/collator/

.maintain/gitlab/check_runtime.sh

@@ -8,12 +8,13 @@
 
 set -e # fail on any error
 
-
+#shellcheck source=../common/lib.sh
+. "$(dirname "${0}")/../common/lib.sh"
 
 VERSIONS_FILE="bin/node/runtime/src/lib.rs"
 
-boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
-boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
+boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
+boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
 
 github_label () {
 	echo
@@ -23,24 +24,22 @@ github_label () {
 		-F "ref=master" \
 		-F "variables[LABEL]=${1}" \
 		-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
-		${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline
+		"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
 }
 
 
 boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
 git log --graph --oneline --decorate=short -n 10
 
 boldprint "make sure the master branch and release tag are available in shallow clones"
-git fetch --depth=${GIT_DEPTH:-100} origin master
-git fetch --depth=${GIT_DEPTH:-100} origin release
+git fetch --depth="${GIT_DEPTH:-100}" origin master
+git fetch --depth="${GIT_DEPTH:-100}" origin release
 git tag -f release FETCH_HEAD
 git log -n1 release
 
 
 boldprint "check if the wasm sources changed"
-if ! git diff --name-only origin/master...${CI_COMMIT_SHA} \
-	| grep -v -e '^primitives/sr-arithmetic/fuzzer' \
-	| grep -q -e '^bin/node/src/runtime' -e '^frame/' -e '^primitives/sr-'
+if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
 then
 	boldcat <<-EOT
 
@@ -57,18 +56,16 @@ fi
 # consensus-critical logic that has changed. the runtime wasm blobs must be
 # rebuilt.
 
-add_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+add_spec_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 	| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-sub_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+sub_spec_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 	| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
 
 
 
 if [ "${add_spec_version}" != "${sub_spec_version}" ]
 then
 
-	github_label "D2-breaksapi"
-
 	boldcat <<-EOT
 
 		changes to the runtime sources and changes in the spec version.
@@ -82,9 +79,9 @@ else
 	# check for impl_version updates: if only the impl versions changed, we assume
 	# there is no consensus-critical logic that has changed.
 
-	add_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	add_impl_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 		| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
-	sub_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	sub_impl_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 		| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
 
 

.maintain/monitoring/alerting-rules/alerting-rules.yaml

@@ -135,7 +135,7 @@ groups:
 
   - alert: ContinuousTaskEnded
     expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer"} == 1)
-        - on(instance, task_name) (polkadot_tasks_ended_total == 1)'
+        - on(instance, task_name) group_left() (polkadot_tasks_ended_total == 1)'
     for: 5m
     labels:
       severity: warning