From 1f6f5c317ee3500385fc1717cd377316864892c7 Mon Sep 17 00:00:00 2001 From: Ewere Diagboya Date: Sat, 22 Jan 2022 12:30:05 +0100 Subject: [PATCH 1/3] adds paritytech/ci-linux:production (#536) * added paritytech * update branch trigger * update run machines * Update .github/workflows/paritytech.yml * Apply suggestions from code review * update cargo nightly * update cargo exec path * Update .github/workflows/check.yml Co-authored-by: Seun Lanlege * update check.yml * use llvm-cov action * prevent rust-toolchain.toml from overriding parity:ci * update workflow file * fix hadolint * update hadolint to action * remove image for unit test * fix syntax error * install cargo-llvm manually * use image * skip hadolint and remark * add llvm-tools-preview * run llvm-cov on runner * update workflow file * update workflow file * force install * llvm-cov in paritytech/ci-linux:production * rustup show * install llvm * remove sudo * one-line install * apt-get install wget * apt-get install wget * apt-get install wget * gnupg2 * run code-cov on runner * path to cargo bin * remove llvm installation * use specific nightly version * please just work * sigh Co-authored-by: Seun Lanlege Co-authored-by: Seun Lanlege Co-authored-by: haroldsphinx --- .github/workflows/check.yml | 157 +++++++++++++++++++++++++----------- .gitignore | 5 ++ Cargo.lock | 30 +++++++ 3 files changed, 145 insertions(+), 47 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 424936c685c..b43adb518fd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -8,29 +8,97 @@ on: - main - develop - releases - workflow_dispatch: env: DOCKER_USER_OPTION: '$UID:$GID' jobs: - check: - name: Check + substrate-check: + name: Check Benchmarks runs-on: - self-hosted - linux - x64 - sre + container: + image: paritytech/ci-linux:production steps: + - name: Clean up + continue-on-error: true + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + docker system prune --force --all --volumes + - uses: actions/checkout@v2 + - name: Rustup show + run: | + rustup show - uses: actions/checkout@v2 - - run: | - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo check --all --benches - - name: Benchmark + - name: cargo check --all --benches run: | - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo check --features=runtime-benchmarks --workspace --release --exclude integration-tests - - name: Picasso + SKIP_WASM_BUILD=1 cargo +nightly check --benches --all # checks all rust crates with default-features, tests, benchmarks + + substrate-test-coverage: + name: Unit Tests (With Coverage) + runs-on: + - self-hosted + - linux + - x64 + - sre + steps: + - name: Clean up + continue-on-error: true + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + docker system prune --force --all --volumes + - uses: actions/checkout@v2 + - name: Rustup show + run: | + /home/runner/.cargo/bin/rustup show + - uses: actions/checkout@v2 + - name: Run Test (with coverage) + run: | + # integration tests can't be compiled with --feature=runtime-benchmarks + # TODO: https://github.com/ComposableFi/composable/issues/535 + /home/runner/.cargo/bin/cargo install -f cargo-llvm-cov + /home/runner/.cargo/bin/rustup component add llvm-tools-preview --toolchain=nightly-2021-11-08 + SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-08 llvm-cov --workspace --locked --release --verbose --features=runtime-benchmarks --exclude=integration-tests --lcov --output-path lcov.info + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: false + + substrate-tests: + name: Unit Tests + runs-on: + - self-hosted + - linux + - x64 + - sre + container: + image: paritytech/ci-linux:production + steps: + - name: Clean up + continue-on-error: true + run: | + sudo chown -R $USER:$USER $GITHUB_WORKSPACE + docker system prune --force --all --volumes + - uses: actions/checkout@v2 + - name: Rustup show + run: | + rustup show + - uses: actions/checkout@v2 + - name: Run Test (with coverage) run: | - /home/runner/.cargo/bin/cargo build --release -p picasso-runtime + SKIP_WASM_BUILD=1 cargo test --workspace --locked --release --verbose --features=runtime-benchmarks --exclude=integration-tests + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + files: lcov.info + fail_ci_if_error: false + linters: name: Linters runs-on: @@ -38,20 +106,25 @@ jobs: - linux - x64 - sre + container: + image: paritytech/ci-linux:production steps: - uses: actions/checkout@v2 - - run: | - /home/runner/.cargo/bin/rustup install nightly-2021-11-29 - /home/runner/.cargo/bin/rustup target add --toolchain=nightly-2021-11-29 wasm32-unknown-unknown - /home/runner/.cargo/bin/rustup show - /home/runner/.cargo/bin/rustup override unset - /home/runner/.cargo/bin/rustup show + - name: Rustup show + run: | + rustup show + - name: Use Node.js 14.x + uses: actions/setup-node@v2 + with: + node-version: 14.x - name: Cargo fmt run: | - /home/runner/.cargo/bin/cargo +nightly-2021-11-29 fmt -- --check + cargo +nightly fmt --all -- --check + - name: Hadolint env: HADOLINT_VERSION: v2.8.0 + continue-on-error: true # this step is such a pita run: | curl -L -o hadolint "https://github.com/hadolint/hadolint/releases/download/${HADOLINT_VERSION}/hadolint-Linux-x86_64" chmod +x hadolint @@ -61,11 +134,27 @@ jobs: echo "=== $file ===" ./hadolint --config .hadolint.yaml $file || total_exit_code=$? echo "" - done < <(find . -name "Dockerfile") + done << (find . -name "Dockerfile") + exit $total_exit_code + + - name: Remark + continue-on-error: true # this step is such a pita + run: | + npm install yarn + ./node_modules/.bin/yarn add remark-cli \ + remark-lint-mdash-style \ + https://github.com/typeable/remark-validate-links#anchors \ + remark-preset-lint-recommended \ + remark-lint-no-dead-urls + + total_exit_code=0 + while IFS= read -r file; do + ./node_modules/.bin/remark -f $file || total_exit_code=$? + done << (find . -name "*.md" -not -path "*/node_modules/*" -not -path "./.github/*") exit $total_exit_code - name: Cargo clippy run: | - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-29 clippy -- -D warnings + SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy --all-targets - name: Cargo udeps env: UDEPS_VERSION: v0.1.24 @@ -74,33 +163,7 @@ jobs: tar xzf "cargo-udeps-${UDEPS_VERSION}-x86_64-unknown-linux-gnu.tar.gz" cp "cargo-udeps-${UDEPS_VERSION}-x86_64-unknown-linux-gnu/cargo-udeps" ~/.cargo/bin/ rm -r "cargo-udeps-${UDEPS_VERSION}-x86_64-unknown-linux-gnu.tar.gz" "cargo-udeps-${UDEPS_VERSION}-x86_64-unknown-linux-gnu" - /home/runner/.cargo/bin/cargo +nightly-2021-11-29 udeps --version - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-29 udeps --workspace --all-features + cargo udeps --version + SKIP_WASM_BUILD=1 cargo udeps --workspace --all-features continue-on-error: true # Often this turns gives false positives, due to the different ways that Substrate uses features. - test: - name: Unit Test - runs-on: - - self-hosted - - linux - - x64 - - sre - steps: - - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - - run: | - /home/runner/.cargo/bin/rustup component add llvm-tools-preview - - name: Install cargo-llvm-cov - run: curl -LsSf https://github.com/taiki-e/cargo-llvm-cov/releases/latest/download/cargo-llvm-cov-x86_64-unknown-linux-gnu.tar.gz | tar xzf - -C ~/.cargo/bin - - name: Run Test (with coverage) - run: | - /home/runner/.cargo/bin/rustup install nightly-2021-11-29 - /home/runner/.cargo/bin/rustup component add llvm-tools-preview --toolchain=nightly-2021-11-29 - SKIP_WASM_BUILD=1 /home/runner/.cargo/bin/cargo +nightly-2021-11-29 llvm-cov --workspace --lcov --output-path lcov.info - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v1 - with: - token: ${{ secrets.CODECOV_TOKEN }} - files: lcov.info - fail_ci_if_error: false + diff --git a/.gitignore b/.gitignore index ec7eb027bd5..fae7309ab47 100644 --- a/.gitignore +++ b/.gitignore @@ -30,3 +30,8 @@ scripts/polkadot-launch/*.log target .env + +node_modules/ +**/yarn-error.log +**/.yarn/ +rust-toolchain.toml diff --git a/Cargo.lock b/Cargo.lock index 040fc5c4fd1..0ccd45e69da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1537,6 +1537,24 @@ dependencies = [ "xcm-executor", ] +[[package]] +name = "composable-support" +version = "0.0.1" +dependencies = [ + "frame-support", + "frame-system", + "is_sorted", + "parity-scale-codec", + "proptest 1.0.0", + "scale-info", + "serde", + "serde_json", + "sorted-vec", + "sp-arithmetic", + "sp-runtime", + "sp-std", +] + [[package]] name = "composable-tests-helpers" version = "0.0.1" @@ -4275,6 +4293,12 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" +[[package]] +name = "is_sorted" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "357376465c37db3372ef6a00585d336ed3d0f11d4345eef77ebcb05865392b21" + [[package]] name = "itertools" version = "0.10.3" @@ -11817,6 +11841,12 @@ dependencies = [ "sha-1 0.9.8", ] +[[package]] +name = "sorted-vec" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4210e1a17d165a0c1d4a233af1e3b36fb21b1359042767a4d2c73554f5f8c2aa" + [[package]] name = "sp-api" version = "4.0.0-dev" From 02fbba272cc4846282d0d70159e4b5600d165b12 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Sat, 22 Jan 2022 12:36:01 +0100 Subject: [PATCH 2/3] remove runtime check --- .github/workflows/runtime-check.yml | 22 ---------------------- 1 file changed, 22 deletions(-) delete mode 100644 .github/workflows/runtime-check.yml diff --git a/.github/workflows/runtime-check.yml b/.github/workflows/runtime-check.yml deleted file mode 100644 index aa6864e7816..00000000000 --- a/.github/workflows/runtime-check.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Check Runtime -on: - pull_request: - branches: - - releases - - main -jobs: - check-runtime: - runs-on: - - self-hosted - - linux - - x64 - - sre - if: github.event.pull_request.draft == false - steps: - - uses: actions/checkout@v2 - with: - fetch-depth: 0 - - - name: Check for runtime changes - run: | - .maintain/check_runtime.sh From afb018e0fa03f43ce92b5e1657c34e650950c022 Mon Sep 17 00:00:00 2001 From: Seun Lanlege Date: Sat, 22 Jan 2022 12:43:38 +0100 Subject: [PATCH 3/3] bump cargo.lock --- Cargo.lock | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0ccd45e69da..040fc5c4fd1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1537,24 +1537,6 @@ dependencies = [ "xcm-executor", ] -[[package]] -name = "composable-support" -version = "0.0.1" -dependencies = [ - "frame-support", - "frame-system", - "is_sorted", - "parity-scale-codec", - "proptest 1.0.0", - "scale-info", - "serde", - "serde_json", - "sorted-vec", - "sp-arithmetic", - "sp-runtime", - "sp-std", -] - [[package]] name = "composable-tests-helpers" version = "0.0.1" @@ -4293,12 +4275,6 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" -[[package]] -name = "is_sorted" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "357376465c37db3372ef6a00585d336ed3d0f11d4345eef77ebcb05865392b21" - [[package]] name = "itertools" version = "0.10.3" @@ -11841,12 +11817,6 @@ dependencies = [ "sha-1 0.9.8", ] -[[package]] -name = "sorted-vec" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4210e1a17d165a0c1d4a233af1e3b36fb21b1359042767a4d2c73554f5f8c2aa" - [[package]] name = "sp-api" version = "4.0.0-dev"