Merge branch 'master' into ao-update-wasmtime

* master: (84 commits) Duplicate logging to stdout (#8495) Fix sync restart (#8497) client: fix justifications migration (#8489) helper macro to create storage types on the fly (#8456) Make `BlockImport` and `Verifier` async (#8472) Get rid of `test-helpers` feature in sc-consensus-babe (#8486) Enhancement on Substrate Node Template (#8473) Add Social Network (#8065) Prepare UI tests for Rust 1.51 & new CI image (#8474) Benchmarking pallet-example (#8301) Use pathbuf for remote externalities (#8480) Bring back the on_finalize weight of staking. (#8463) Implement `fungible::*` for Balances (#8454) make types within `generate_solution_type` macro explicit (#8447) [pallet-staking] Refund unused weight for `payout_stakers` (#8458) Use `async_trait` in sc-consensus-slots (#8461) Repot frame_support::traits; introduce some new currency stuff (#8435) Fix &mut self -> &self in add_known_address (#8468) Add NetworkService::add_known_address (#8467) Fix companion check (#8464) ...
paritytech · Mar 31, 2021 · ded49c0 · ded49c0
2 parents d0676ed + 70f1b61
commit ded49c0
Show file tree

Hide file tree

Showing 461 changed files with 18,361 additions and 15,572 deletions.
diff --git a/.github/workflows/check-labels.yml b/.github/workflows/check-labels.yml
@@ -9,8 +9,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v2
+        with:
+          fetch-depth: 0
+          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
       - name: Check labels
         run: bash ${{ github.workspace }}/.maintain/github/check_labels.sh
         env:
           GITHUB_PR: ${{ github.event.pull_request.number }}
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,4 @@ rls*.log
 .cargo/
 .cargo-remote.toml
 *.bin
+*.iml
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -119,6 +119,11 @@ default:
     - if: $CI_COMMIT_REF_NAME == "tags"
     - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
 
+.nightly-pipeline:                 &nightly-pipeline
+  rules:
+    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
+    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+
 #### stage:                       .pre
 
 skip-if-draft:
@@ -199,17 +204,7 @@ test-prometheus-alerting-rules:
 cargo-deny:
   stage:                           test
   <<:                              *docker-env
-  rules:
-    - if: $CI_PIPELINE_SOURCE == "pipeline"
-      when: never
-    - changes:
-      - "Cargo.lock"
-      - "**/Cargo.toml"
-    - if: $CI_PIPELINE_SOURCE == "web"
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-    - if: $CI_COMMIT_REF_NAME == "master"
-    - if: $CI_COMMIT_REF_NAME == "tags"
-    - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/              # i.e. v1.0, v2.1rc1
+  <<:                              *nightly-pipeline
   script:
     - cargo deny check --hide-inclusion-graph -c .maintain/deny.toml
   after_script:
@@ -277,6 +272,7 @@ test-linux-stable:                 &test-linux
   script:
     # this job runs all tests in former runtime-benchmarks, frame-staking and wasmtime tests
     - time cargo test --workspace --locked --release --verbose --features runtime-benchmarks --manifest-path bin/node/cli/Cargo.toml
+    - time cargo test -p frame-support-test --features=conditional-storage --manifest-path frame/support/test/Cargo.toml
     - SUBSTRATE_TEST_TIMEOUT=1 time cargo test -p substrate-test-utils --release --verbose --locked -- --ignored timeout
     - sccache -s
 
@@ -340,7 +336,7 @@ check-web-wasm:
     # Note: we don't need to test crates imported in `bin/node/cli`
     - time cargo build --manifest-path=client/consensus/aura/Cargo.toml --target=wasm32-unknown-unknown --features getrandom
     # Note: the command below is a bit weird because several Cargo issues prevent us from compiling the node in a more straight-forward way.
-    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown -Z features=itarget
+    - time cargo +nightly build --manifest-path=bin/node/cli/Cargo.toml --no-default-features --features browser --target=wasm32-unknown-unknown
     # with-tracing must be explicitly activated, we run a test to ensure this works as expected in both cases
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features
     - time cargo +nightly test --manifest-path primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
@@ -411,7 +407,7 @@ test-browser-node:
     CARGO_TARGET_WASM32_UNKNOWN_UNKNOWN_RUNNER: "wasm-bindgen-test-runner"
     WASM_BINDGEN_TEST_TIMEOUT:                  120
   script:
-    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing -Z features=itarget
+    - cargo +nightly test --target wasm32-unknown-unknown -p node-browser-testing
 
 build-linux-substrate:             &build-binary
   stage:                           build
@@ -523,6 +519,10 @@ build-rust-doc:
     - buildah info
     - buildah push --format=v2s2 "$IMAGE_NAME:$VERSION"
     - buildah push --format=v2s2 "$IMAGE_NAME:latest"
+    # pass artifacts to the trigget-simnet job
+    - echo "VERSION=${VERSION}" > build.env
+    - echo "TRIGGERER=${CI_PROJECT_NAME}" >> build.env
+  after_script:
     - buildah logout "$IMAGE_NAME"
 
 publish-docker-substrate:
@@ -534,8 +534,6 @@ publish-docker-substrate:
   variables:
     <<:                            *docker-build-vars
     PRODUCT:                       substrate
-  after_script:
-    - echo "VERSION=${VERSION}" >> build.env
   artifacts:
     reports:
       # this artifact is used in trigger-simnet job
@@ -653,9 +651,7 @@ deploy-prometheus-alerting-rules:
 
 trigger-simnet:
   stage:                           deploy
-  rules:
-    # this job runs only on nightly pipeline with the mentioned variable, against `master` branch
-    - if: $CI_COMMIT_REF_NAME == "master" && $CI_PIPELINE_SOURCE == "schedule" && $PIPELINE == "nightly"
+  <<:                              *nightly-pipeline
   needs:
     - job:                         publish-docker-substrate
   trigger:

diff --git a/.maintain/common/lib.sh b/.maintain/common/lib.sh
@@ -82,7 +82,7 @@ has_label(){
 
 # Formats a message into a JSON string for posting to Matrix
 # message: 'any plaintext message'
-# formatted_message: '<strong>optional message formatted in <em>html</em></strong>' 
+# formatted_message: '<strong>optional message formatted in <em>html</em></strong>'
 # Usage: structure_message $content $formatted_content (optional)
 structure_message() {
   if [ -z "$2" ]; then
@@ -101,3 +101,17 @@ structure_message() {
 send_message() {
 curl -XPOST -d "$1" "https://matrix.parity.io/_matrix/client/r0/rooms/$2/send/m.room.message?access_token=$3"
 }
+
+# Check for runtime changes between two commits. This is defined as any changes
+# to bin/node/src/runtime, frame/ and primitives/sr_* trees.
+has_runtime_changes() {
+  from=$1
+  to=$2
+  if git diff --name-only "${from}...${to}" \
+    | grep -q -e '^frame/' -e '^primitives/'
+  then
+    return 0
+  else
+    return 1
+  fi
+}
diff --git a/.maintain/github/check_labels.sh b/.maintain/github/check_labels.sh
@@ -1,4 +1,5 @@
 #!/usr/bin/env bash
+set -e
 
 #shellcheck source=../common/lib.sh
 source "$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )/../common/lib.sh"
@@ -24,10 +25,18 @@ releasenotes_labels=(
 )
 
 criticality_labels=(
-  'C1-low'
-  'C3-medium'
-  'C7-high'
-  'C9-critical'
+  'C1-low 📌'
+  'C3-medium 📣'
+  'C7-high ❗️'
+  'C9-critical ‼️'
+)
+
+audit_labels=(
+  'D1-audited 👍'
+  'D2-notlive 💤'
+  'D3-trivial 🧸'
+  'D5-nicetohaveaudit ⚠️'
+  'D9-needsaudit 👮'
 )
 
 echo "[+] Checking release notes (B) labels"
@@ -46,4 +55,14 @@ else
   exit 1
 fi
 
+if has_runtime_changes origin/master "${HEAD_SHA}"; then
+  echo "[+] Runtime changes detected. Checking audit (D) labels"
+  if ensure_labels "${audit_labels[@]}";  then
+    echo "[+] Release audit label detected. All is well."
+  else
+    echo "[!] Release audit label not detected. Please add one of: ${audit_labels[*]}"
+    exit 1
+  fi
+fi
+
 exit 0
diff --git a/.maintain/gitlab/check_polkadot_companion_build.sh b/.maintain/gitlab/check_polkadot_companion_build.sh
@@ -86,10 +86,9 @@ else
 fi
 
 # Patch all Substrate crates in Polkadot
-diener patch --crates-to-patch ../ --substrate
+diener patch --crates-to-patch ../ --substrate --path Cargo.toml
 
 # Test Polkadot pr or master branch with this Substrate commit.
-cargo update -p sp-io
 time cargo test --all --release --verbose --features=real-overseer
 
 cd parachain/test-parachains/adder/collator/

diff --git a/.maintain/gitlab/check_polkadot_companion_status.sh b/.maintain/gitlab/check_polkadot_companion_status.sh
@@ -56,28 +56,14 @@ fi
 boldprint "companion pr: #${pr_companion}"
 
 # check the status of that pull request - needs to be
-# mergable and approved
+# approved and mergable
 
 curl -H "${github_header}" -sS -o companion_pr.json \
   ${github_api_polkadot_pull_url}/${pr_companion}
 
 pr_head_sha=$(jq -r -e '.head.sha' < companion_pr.json)
 boldprint "Polkadot PR's HEAD SHA: $pr_head_sha"
 
-if jq -e .merged < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} already merged"
-  exit 0
-fi
-
-if jq -e '.mergeable' < companion_pr.json >/dev/null
-then
-  boldprint "polkadot pr #${pr_companion} mergeable"
-else
-  boldprint "polkadot pr #${pr_companion} not mergeable"
-  exit 1
-fi
-
 curl -H "${github_header}" -sS -o companion_pr_reviews.json \
   ${github_api_polkadot_pull_url}/${pr_companion}/reviews
 
@@ -98,6 +84,19 @@ if [ -z "$(jq -r -e '.[].state | select(. == "APPROVED")' < companion_pr_reviews
 fi
 
 boldprint "polkadot pr #${pr_companion} state APPROVED"
-exit 0
 
+if jq -e .merged < companion_pr.json >/dev/null
+then
+  boldprint "polkadot pr #${pr_companion} already merged"
+  exit 0
+fi
+
+if jq -e '.mergeable' < companion_pr.json >/dev/null
+then
+  boldprint "polkadot pr #${pr_companion} mergeable"
+else
+  boldprint "polkadot pr #${pr_companion} not mergeable"
+  exit 1
+fi
 
+exit 0
diff --git a/.maintain/gitlab/check_runtime.sh b/.maintain/gitlab/check_runtime.sh
@@ -8,12 +8,13 @@
 
 set -e # fail on any error
 
-
+#shellcheck source=../common/lib.sh
+. "$(dirname "${0}")/../common/lib.sh"
 
 VERSIONS_FILE="bin/node/runtime/src/lib.rs"
 
-boldprint () { printf "|\n| \033[1m${@}\033[0m\n|\n" ; }
-boldcat () { printf "|\n"; while read l; do printf "| \033[1m${l}\033[0m\n"; done; printf "|\n" ; }
+boldprint () { printf "|\n| \033[1m%s\033[0m\n|\n" "${@}"; }
+boldcat () { printf "|\n"; while read -r l; do printf "| \033[1m%s\033[0m\n" "${l}"; done; printf "|\n" ; }
 
 github_label () {
 	echo
@@ -23,24 +24,22 @@ github_label () {
 		-F "ref=master" \
 		-F "variables[LABEL]=${1}" \
 		-F "variables[PRNO]=${CI_COMMIT_REF_NAME}" \
-		${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline
+		"${GITLAB_API}/projects/${GITHUB_API_PROJECT}/trigger/pipeline"
 }
 
 
 boldprint "latest 10 commits of ${CI_COMMIT_REF_NAME}"
 git log --graph --oneline --decorate=short -n 10
 
 boldprint "make sure the master branch and release tag are available in shallow clones"
-git fetch --depth=${GIT_DEPTH:-100} origin master
-git fetch --depth=${GIT_DEPTH:-100} origin release
+git fetch --depth="${GIT_DEPTH:-100}" origin master
+git fetch --depth="${GIT_DEPTH:-100}" origin release
 git tag -f release FETCH_HEAD
 git log -n1 release
 
 
 boldprint "check if the wasm sources changed"
-if ! git diff --name-only origin/master...${CI_COMMIT_SHA} \
-	| grep -v -e '^primitives/sr-arithmetic/fuzzer' \
-	| grep -q -e '^bin/node/src/runtime' -e '^frame/' -e '^primitives/sr-'
+if ! has_runtime_changes origin/master "${CI_COMMIT_SHA}"
 then
 	boldcat <<-EOT
 
@@ -57,18 +56,16 @@ fi
 # consensus-critical logic that has changed. the runtime wasm blobs must be
 # rebuilt.
 
-add_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+add_spec_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 	| sed -n -r "s/^\+[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
-sub_spec_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+sub_spec_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 	| sed -n -r "s/^\-[[:space:]]+spec_version: +([0-9]+),$/\1/p")"
 
 
 
 if [ "${add_spec_version}" != "${sub_spec_version}" ]
 then
 
-	github_label "D2-breaksapi"
-
 	boldcat <<-EOT
 
 		changes to the runtime sources and changes in the spec version.
@@ -82,9 +79,9 @@ else
 	# check for impl_version updates: if only the impl versions changed, we assume
 	# there is no consensus-critical logic that has changed.
 
-	add_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	add_impl_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 		| sed -n -r 's/^\+[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
-	sub_impl_version="$(git diff tags/release...${CI_COMMIT_SHA} ${VERSIONS_FILE} \
+	sub_impl_version="$(git diff "tags/release...${CI_COMMIT_SHA}" "${VERSIONS_FILE}" \
 		| sed -n -r 's/^\-[[:space:]]+impl_version: +([0-9]+),$/\1/p')"
 
 

diff --git a/.maintain/monitoring/alerting-rules/alerting-rules.yaml b/.maintain/monitoring/alerting-rules/alerting-rules.yaml
@@ -135,7 +135,7 @@ groups:
 
   - alert: ContinuousTaskEnded
     expr: '(polkadot_tasks_spawned_total{task_name != "basic-authorship-proposer"} == 1)
-        - on(instance, task_name) (polkadot_tasks_ended_total == 1)'
+        - on(instance, task_name) group_left() (polkadot_tasks_ended_total == 1)'
     for: 5m
     labels:
       severity: warning

diff --git a/.maintain/node-template-release.sh b/.maintain/node-template-release.sh
@@ -10,7 +10,7 @@ if [ "$#" -ne 1 ]; then
   exit 1
 fi
 
-PATH_TO_ARCHIVE=$(pwd)/$1
+PATH_TO_ARCHIVE=$1
 cd $PROJECT_ROOT/.maintain/node-template-release
 
 cargo run $PROJECT_ROOT/bin/node-template $PATH_TO_ARCHIVE
diff --git a/.maintain/node-template-release/Cargo.toml b/.maintain/node-template-release/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "node-template-release"
-version = "2.0.0"
+version = "3.0.0"
 authors = ["Parity Technologies <admin@parity.io>"]
 edition = "2018"
 license = "GPL-3.0"