diff --git a/.github/workflows/joystream-apps-docker.yml b/.github/workflows/joystream-apps-docker.yml index d4886d1a19..de706b772d 100644 --- a/.github/workflows/joystream-apps-docker.yml +++ b/.github/workflows/joystream-apps-docker.yml @@ -87,7 +87,7 @@ jobs: - name: Test with latest images if: github.event_name == 'pull_request' run: | - yarn build:packages + yarn build cp docker-compose-no-bind-volumes.yml docker-compose.yml ./tests/network-tests/run-tests.sh content-directory diff --git a/.github/workflows/lint-typescript.yml b/.github/workflows/lint-typescript.yml index ef00cf96bf..98a741080e 100644 --- a/.github/workflows/lint-typescript.yml +++ b/.github/workflows/lint-typescript.yml @@ -16,7 +16,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: lint run: | - yarn build:packages + yarn build yarn lint lint_osx: @@ -33,5 +33,5 @@ jobs: node-version: ${{ matrix.node-version }} - name: lint run: | - yarn build:packages + yarn build yarn lint diff --git a/.github/workflows/run-network-tests.yml b/.github/workflows/run-network-tests.yml index 99ea16c2bd..806e64d06a 100644 --- a/.github/workflows/run-network-tests.yml +++ b/.github/workflows/run-network-tests.yml @@ -21,7 +21,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: checks run: | - yarn build:packages + yarn build tests_build_osx: name: MacOS Checks @@ -37,7 +37,7 @@ jobs: node-version: ${{ matrix.node-version }} - name: checks run: | - yarn build:packages + yarn build build_images: name: Build joystream/node @@ -166,7 +166,7 @@ jobs: docker images - name: Install packages and dependencies run: | - yarn build:packages + yarn build - name: Ensure tests are runnable run: yarn workspace network-tests build - name: Execute network tests @@ -231,7 +231,7 @@ jobs: - name: Install packages and dependencies if: env.GIT_DIFF run: | - yarn build:packages + yarn build - name: Ensure tests are runnable if: env.GIT_DIFF run: yarn workspace network-tests build diff --git a/RUNTIME-CONTRIBUTING.md b/RUNTIME-CONTRIBUTING.md index 7f9844c03d..3dbea11909 100644 --- a/RUNTIME-CONTRIBUTING.md +++ b/RUNTIME-CONTRIBUTING.md @@ -225,7 +225,7 @@ be followed below: 1. Re run code formatting with `cargo fmt --all` 1. Do another test build with modified benchmarks `yarn cargo-checks && yarn cargo-build` 1. Extract the new runtime metadata `yarn update-chain-metadata` - 1. Build all npm packages: `yarn build:packages` + 1. Build all npm packages: `yarn build` 1. Add any new integration tests, query-node mappings that cover the changes implemented. 1. Lint typescript `yarn lint` 1. Build the testing runtime joystream/node docker image and run the full integration test suite (see Integration tests section below) diff --git a/devops/ansible/roles/common/tasks/build-apps.yml b/devops/ansible/roles/common/tasks/build-apps.yml index ee1ce649da..9777543156 100644 --- a/devops/ansible/roles/common/tasks/build-apps.yml +++ b/devops/ansible/roles/common/tasks/build-apps.yml @@ -4,7 +4,7 @@ - name: Build joystream apps shell: cmd: | - bash -ic "yarn build:packages" + bash -ic "yarn build" args: chdir: '{{ remote_code_path }}' # Always try to rebuild incase code has been updated for existing repo diff --git a/devops/extrinsic-ordering/filter.sh b/devops/extrinsic-ordering/filter.sh new file mode 100755 index 0000000000..4fd3337f64 --- /dev/null +++ b/devops/extrinsic-ordering/filter.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# This script is used in a Github Workflow. It helps filtering out what is interesting +# when comparing metadata and spot what would require a tx version bump. + +# shellcheck disable=SC2002,SC2086 + +FILE=$1 + +# Higlight indexes that were deleted +function find_deletions() { + echo "\n## Deletions\n" + RES=$(cat "$FILE" | grep -n '\[\-\]' | tr -s " ") + if [ "$RES" ]; then + echo "$RES" | awk '{ printf "%s\\n", $0 }' + else + echo "n/a" + fi +} + +# Highlight indexes that have been deleted +function find_index_changes() { + echo "\n## Index changes\n" + RES=$(cat "$FILE" | grep -E -n -i 'idx:\s*([0-9]+)\s*(->)\s*([0-9]+)' | tr -s " ") + if [ "$RES" ]; then + echo "$RES" | awk '{ printf "%s\\n", $0 }' + else + echo "n/a" + fi +} + +# Highlight values that decreased +function find_decreases() { + echo "\n## Decreases\n" + OUT=$(cat "$FILE" | grep -E -i -o '([0-9]+)\s*(->)\s*([0-9]+)' | awk '$1 > $3 { printf "%s;", $0 }') + IFS=$';' LIST=("$OUT") + unset RES + for line in "${LIST[@]}"; do + RES="$RES\n$(cat "$FILE" | grep -E -i -n \"$line\" | tr -s " ")" + done + + if [ "$RES" ]; then + echo "$RES" | awk '{ printf "%s\\n", $0 }' | sort -u -g | uniq + else + echo "n/a" + fi +} + +echo "\n------------------------------ SUMMARY -------------------------------" +echo "\n⚠️ This filter is here to help spotting changes that should be reviewed carefully." +echo "\n⚠️ It catches only index changes, deletions and value decreases". + +find_deletions "$FILE" +find_index_changes "$FILE" +find_decreases "$FILE" +echo "\n----------------------------------------------------------------------\n" diff --git a/devops/extrinsic-ordering/tx-ordering.yml b/devops/extrinsic-ordering/tx-ordering.yml new file mode 100644 index 0000000000..0613ed04d3 --- /dev/null +++ b/devops/extrinsic-ordering/tx-ordering.yml @@ -0,0 +1,81 @@ +# This workflow performs the Extrinsic Ordering Check on demand using a binary + +name: Release - Extrinsic Ordering Check +on: + workflow_dispatch: + inputs: + reference_url: + description: The WebSocket url of the reference node + default: wss://kusama-rpc.polkadot.io + required: true + binary_url: + description: A url to a Linux binary for the node containing the runtime to test + default: https://releases.parity.io/polkadot/x86_64-debian:stretch/v0.9.10/polkadot + required: true + chain: + description: The name of the chain under test. Usually, you would pass a local chain + default: kusama-local + required: true + +jobs: + check: + name: Run check + runs-on: ubuntu-latest + env: + CHAIN: ${{github.event.inputs.chain}} + BIN_URL: ${{github.event.inputs.binary_url}} + REF_URL: ${{github.event.inputs.reference_url}} + + steps: + - name: Checkout sources + uses: actions/checkout@v3 + + - name: Fetch binary + run: | + echo Fetching $BIN_URL + wget $BIN_URL + chmod a+x polkadot + ./polkadot --version + + - name: Start local node + run: | + echo Running on $CHAIN + ./polkadot --chain=$CHAIN & + + - name: Prepare output + run: | + VERSION=$(./polkadot --version) + echo "Metadata comparison:" >> output.txt + echo "Date: $(date)" >> output.txt + echo "Reference: $REF_URL" >> output.txt + echo "Target version: $VERSION" >> output.txt + echo "Chain: $CHAIN" >> output.txt + echo "----------------------------------------------------------------------" >> output.txt + + - name: Pull polkadot-js-tools image + run: docker pull jacogr/polkadot-js-tools + + - name: Compare the metadata + run: | + CMD="docker run --pull always --network host jacogr/polkadot-js-tools metadata $REF_URL ws://localhost:9944" + echo -e "Running:\n$CMD" + $CMD >> output.txt + sed -z -i 's/\n\n/\n/g' output.txt + cat output.txt | egrep -n -i '' + SUMMARY=$(./scripts/ci/github/extrinsic-ordering-filter.sh output.txt) + echo -e $SUMMARY + echo -e $SUMMARY >> output.txt + + - name: Show result + run: | + cat output.txt + + - name: Stop our local node + run: pkill polkadot + + - name: Save output as artifact + uses: actions/upload-artifact@v3 + with: + name: ${{ env.CHAIN }} + path: | + output.txt diff --git a/package.json b/package.json index ee24769c78..7b843081c3 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "scripts": { "build:node:docker": "./build-node-docker.sh", "build:packages": "./build-packages.sh", - "build": "yarn build:packages", + "build": "./build-packages.sh", "setup": "./setup.sh", "start": "./start.sh", "test": "./tests/network-tests/run-tests.sh", diff --git a/query-node/README.md b/query-node/README.md index ae9e673918..85d6b4291d 100644 --- a/query-node/README.md +++ b/query-node/README.md @@ -10,7 +10,7 @@ Run ```bash # Make sure these steps are executed first: -# yarn build:packages; # OR: +# yarn build; # OR: # yarn && yarn workspace @joystream/types build && yarn workspace @joystream/metadata-protobuf build yarn workspace query-node-root build ```