diff --git a/.github/workflows/action-validator_darwin_arm64 b/.github/workflows/action-validator_darwin_arm64 new file mode 100755 index 0000000000..9fb38860af Binary files /dev/null and b/.github/workflows/action-validator_darwin_arm64 differ diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml new file mode 100644 index 0000000000..131769c136 --- /dev/null +++ b/.github/workflows/build.yaml @@ -0,0 +1,22 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +name: Manual Workflow Dispatch + +on: + workflow_dispatch: + +jobs: + example-job: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Print template message + run: echo "This is a template for testing workflows in branches." + \ No newline at end of file diff --git a/.github/workflows/kagome-builder-deb.yaml b/.github/workflows/kagome-builder-deb.yaml new file mode 100644 index 0000000000..83344e6ea6 --- /dev/null +++ b/.github/workflows/kagome-builder-deb.yaml @@ -0,0 +1,102 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +name: Kagome Builder | Debian (Bookworm) + +on: + # schedule: + # - cron: '0 2 * * 0' # Every Sunday at 2:00 AM + workflow_dispatch: + inputs: + kagome_image_rebuild: + description: 'Rebuild Kagome Builder Minideb image' + required: false + default: 'false' + type: 'boolean' + runtime_cache_rebuild: + description: 'Rebuild Kagome Runtime Cache' + required: false + default: 'false' + type: 'boolean' + +env: + DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ + +jobs: + build_polkadot_builder: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name : "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Check if image tag exists" + id: check-tag + working-directory: ./housekeeping/docker/kagome-dev + run: | + IMAGE_TAG=$(grep '^MINIDEB_IMAGE' Makefile | awk -F '[@:]' '{print $3}' | cut -c 1-7)_rust-$(grep '^RUST_VERSION' Makefile | sed 's/.*?= //') + KAGOME_BUILDER_DEB_IMAGE=${DOCKER_REGISTRY_PATH}kagome_builder_deb:${IMAGE_TAG} + TAG_EXISTS=$(docker manifest inspect ${KAGOME_BUILDER_DEB_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") + echo "exists=$TAG_EXISTS" >> $GITHUB_ENV + echo "kagome_builder_deb=KAGOME_BUILDER_DEB_IMAGE" >> GITHUB_OUTPUT + + - name: "Build Polkadot Builder" + working-directory: ./housekeeping/docker/kagome-dev + if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true' + run: make kagome_builder_deb \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + + - name: "Push Polkadot Builder" + working-directory: ./housekeeping/docker/kagome-dev + if: env.exists == 'false' || github.event.inputs.kagome_image_rebuild == 'true' + run: make kagome_builder_deb_push \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + + build_kagome_runtime: + runs-on: [ actions-runner-controller ] + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name : "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Build Runtime Cache" + working-directory: ./housekeeping/docker/kagome-dev + if: github.event.inputs.runtime_cache_rebuild == 'true' + run: make runtime_cache \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + + - name: "Push Runtime Cache" + working-directory: ./housekeeping/docker/kagome-dev + if: github.event.inputs.runtime_cache_rebuild == 'true' + run: make upload_apt_package_runtime \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + \ No newline at end of file diff --git a/.github/workflows/telegram_notify.yml b/.github/workflows/telegram_notify.yml index c97836d843..0693b6fb94 100644 --- a/.github/workflows/telegram_notify.yml +++ b/.github/workflows/telegram_notify.yml @@ -1,3 +1,9 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + name: Telegram Notification on: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index c7c67a6786..586753eb51 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ # SPDX-License-Identifier: Apache-2.0 # -name: Build +name: Main Build Pipeline on: push: @@ -12,6 +12,15 @@ on: tags: [ '*' ] pull_request: workflow_dispatch: + inputs: + run_all_steps: + description: 'Run all steps' + required: true + default: 'false' + type: 'choice' + options: + - 'true' + - 'false' env: BUILD_DIR: build @@ -21,13 +30,18 @@ env: GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }} # Change all container: values to this variable once this feature is available # https://github.community/t/how-to-set-and-access-a-workflow-variable/17335/6 - CACHE_VERSION: v07 + CACHE_VERSION: v01 CACHE_PATHS: | ~/Library/Caches/pip ~/.cargo ~/.ccache ~/.hunter ~/.rustup + DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ + CACHE_PATH: ./build/cache + DEFAULT_BUILD_TYPE: Release + CI: true + jobs: MacOS: @@ -51,7 +65,10 @@ jobs: Linux: - if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} + if: false # Need to fix + # ${{ + # !(github.ref == 'refs/heads/master' || + # startsWith( github.ref, 'refs/tags/' )) }} strategy: fail-fast: false matrix: @@ -93,8 +110,9 @@ jobs: run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM - name: "Self-hosted: Linux: clang-16 UBSAN" run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON - - name: "Self-hosted: Linux: clang-16 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake + # Need to fix + # - name: "Self-hosted: Linux: clang-16 External Project" + # run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake name: "${{ matrix.options.name }}" runs-on: [ actions-runner-controller ] @@ -152,12 +170,16 @@ jobs: env: BUILD_FINAL_TARGET: ctest_coverage run: ./housekeeping/make_build.sh -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-12_cxx20.cmake -DCOVERAGE=ON -DCLEAR_OBJS=ON - - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + - if: ${{ + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository }} name: Submit Coverage env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} run: if [ "CODECOV_TOKEN" != "null" ]; then ./housekeeping/codecov.sh; else echo "Some secret undefined. Step passed..."; fi - - if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }} + - if: ${{ + github.event_name != 'pull_request' || + github.event.pull_request.head.repo.full_name == github.repository }} name: Sonar env: SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }} @@ -167,7 +189,9 @@ jobs: run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi Minideb: - if: ${{ !(github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' )) }} + if: ${{ + !(github.ref == 'refs/heads/master' || + startsWith( github.ref, 'refs/tags/' )) }} strategy: fail-fast: false matrix: @@ -191,161 +215,198 @@ jobs: BUILD_TYPE: "${{ matrix.options.build-type }}" run: ./housekeeping/docker/kagome-dev/make.sh - Minideb-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} + kagome_dev_docker_build: + runs-on: [ actions-runner-controller ] + timeout-minutes: 180 + if: ${{ + github.ref == 'refs/heads/master' || + startsWith( github.ref, 'refs/tags/') || + contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} || + github.event.inputs.run_all_steps == 'true' }} strategy: fail-fast: false matrix: options: - name: "Self-hosted: Minideb: Build Debug" - build-type: "Debug" + build_type: "Debug" - name: "Self-hosted: Minideb: Build Release" - build-type: "Release" + build_type: "Release" name: "${{ matrix.options.name }}" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: "${{ matrix.options.name }}" - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - push-self-hosted: - if: ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/' ) }} - needs: [clang-tidy, MacOS, Minideb-self-hosted] - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Push Debug Docker image" - build-type: "Debug" - - name: "Self-hosted: Push Release Docker image" - build-type: "Release" - - name: "Self-hosted: Push RelWithDebInfo Docker image" - build-type: "RelWithDebInfo" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ env.CACHE_VERSION }} - - run: git config --global --add safe.directory /__w/kagome/kagome - - run: git fetch --prune --unshallow - - name: build - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - run: ./housekeeping/docker/kagome-dev/make.sh - - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: docker pack and push - id: docker-pack-and-push - env: - VERSION: ${{ github.ref }} - BUILD_TYPE: "${{ matrix.options.build-type }}" + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" run: | - ./housekeeping/docker/kagome/build_and_push.sh - - name: Save TAG - id: save-tag - if: matrix.options.build-type == 'Release' - run: TAG=$(cat tag_output.txt) && echo "tag=$TAG" >> $GITHUB_OUTPUT - outputs: - release-tag: ${{ steps.save-tag.outputs.tag }} - - Prepare-Zombie-Tests-Docker: - runs-on: [ actions-runner-controller ] - timeout-minutes: 60 - needs: [push-self-hosted] - steps: - - uses: azure/docker-login@83efeb77770c98b620c73055fbb59b2847e17dc0 + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Check package version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') + REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ matrix.options.build_type }}" + echo "PACKAGE_VERSION=${PACKAGE_VERSION}" + echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV + echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV + gcloud config set artifacts/repository $ARTIFACTS_REPO + gcloud config set artifacts/location $REGION + PACKAGE_EXIST=$(gcloud artifacts versions list --package=kagome-dev --format=json | jq -e ".[] | select(.name | endswith(\"${PACKAGE_VERSION}\"))" > /dev/null && echo "True" || echo "False") + echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV + + - name: "Cache dependencies" + if: env.package_exist == 'False' + id: cache-restore + uses: actions/cache/restore@v4 with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Pull Kagome Docker Image - run: docker pull ${{ needs.push-self-hosted.outputs.release-tag }} - - name: Load image - run: docker image ls -a - - name: Set owner of working dir recursively - run: sudo chown -R $(whoami) . - - uses: actions/checkout@v4 - - name: Extract branch name - shell: bash + path: ${{ env.CACHE_PATH }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}-${{ env.short_commit_hash }} + restore-keys: | + ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ matrix.options.build_type }}- + + - name: "Build target" + if: env.package_exist == 'False' + working-directory: ./housekeeping/docker/kagome-dev + run: make kagome_dev_docker_build \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ + GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ + BUILD_TYPE=${{ matrix.options.build_type }} + + - name: "Push Kagome APT Package" + if: env.package_exist == 'False' + working-directory: ./housekeeping/docker/kagome-dev run: | - branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} - echo "branch=${branch}" >> $GITHUB_OUTPUT - echo "Branch name: ${branch}" - id: extract_branch - - name: Build zombie-tester - working-directory: ./zombienet/docker - run: make tester BRANCH_NAME=${{ steps.extract_branch.outputs.branch }} KAGOME_IMAGE=${{ needs.push-self-hosted.outputs.release-tag }} - - name: Archive zombie-tester - shell: bash - run: docker save -o /tmp/zombie-tester.tar zombie-tester:latest - - name: Upload artifact - uses: actions/upload-artifact@v4 + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + make upload_apt_package \ + BUILD_TYPE=${{ matrix.options.build_type }} + + - name: "Cleaning cache" + if: env.package_exist == 'False' + run: | + find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; + + - name: "Always Save Cache" + id: cache-save + if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') + uses: actions/cache/save@v4 with: - name: zombie-tester - path: /tmp/zombie-tester.tar + path: ${{ env.CACHE_PATH }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + Zombie-Tests: runs-on: [ actions-runner-controller ] timeout-minutes: 60 - needs: [Prepare-Zombie-Tests-Docker] + needs: [kagome_dev_docker_build] strategy: fail-fast: false matrix: options: - name: "PVF preparation & execution time" test: "test-polkadot-functional-0001-parachains-pvf" - - name: "Disputes initiation, conclusion and lag" - test: "test-polkadot-functional-0002-parachains-disputes" + # - name: "Disputes initiation, conclusion and lag" + # test: "test-polkadot-functional-0002-parachains-disputes" - name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute" test: "test-polkadot-functional-0003-beefy-and-mmr" - - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" - test: "test-polkadot-functional-0004-parachains-garbage-candidate" - - name: "Past-session dispute slashing" - test: "test-polkadot-functional-0005-parachains-disputes-past-session" - - name: "Test if parachains make progress with most of approvals being tranch0" - test: "test-polkadot-functional-0006-parachains-max-tranche0" + # - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" + # test: "test-polkadot-functional-0004-parachains-garbage-candidate" + # - name: "Past-session dispute slashing" + # test: "test-polkadot-functional-0005-parachains-disputes-past-session" + # - name: "Test if parachains make progress with most of approvals being tranch0" + # test: "test-polkadot-functional-0006-parachains-max-tranche0" - name: "Test if disputes triggered on finalized blocks within scope always end as valid" test: "test-polkadot-functional-0007-dispute-freshly-finalized" - name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded" test: "test-polkadot-functional-0008-dispute-old-finalized" - name: "Approval voting coalescing does not lag finality" test: "test-polkadot-functional-0009-approval-voting-coalescing" - - name: "Test validator disabling effects" - test: "test-polkadot-functional-0010-validator-disabling" + # - name: "Test validator disabling effects" + # test: "test-polkadot-functional-0010-validator-disabling" - name: "Test we are producing blocks at 6 seconds clip" test: "test-polkadot-functional-0011-async-backing-6-seconds-rate" steps: - - name: Download artifact - uses: actions/download-artifact@v4 + - name: "Checkout repository" + uses: actions/checkout@v4 + +# - name: Create and enable swap +# run: | +# free -h +# df -h +# sudo fallocate -l 4G /swapfile +# sudo chmod 600 /swapfile +# sudo mkswap /swapfile +# sudo swapon /swapfile +# sudo swapon --show +# free -h +# df -h + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' with: - name: zombie-tester - path: /tmp - - name: Load image + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" run: | - docker load --input /tmp/zombie-tester.tar - docker image ls -a - - uses: actions/checkout@v4 - - name: Run test + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Get Package Version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') + REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ env.DEFAULT_BUILD_TYPE }}" + echo "PACKAGE_VERSION=${PACKAGE_VERSION}" + echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV + + - name: "Get Zombie Tester Version" + run: | + ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:latest + echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" + echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV + + - name: "Run test" working-directory: ./zombienet/docker - run: make ${{ matrix.options.test }} ZOMBIE_TESTER_IMAGE=zombie-tester:latest - - name: Upload test logs as artifact + run: | + make ${{ matrix.options.test }} \ + ZOMBIE_TESTER_IMAGE=${{ env.zombie_tester_image }} \ + KAGOME_PACKAGE_VERSION=${{ env.package_version }} + free -h + df -h + + - name: "Upload test logs as artifact" if: always() uses: actions/upload-artifact@v4 with: name: logs-${{ matrix.options.test }} path: /tmp/test_logs + + - name: "Show system info" + if: always() + run: | + free -h + df -h diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml index 8290eab64f..df44f49701 100644 --- a/.github/workflows/zombie-builder.yml +++ b/.github/workflows/zombie-builder.yml @@ -4,28 +4,218 @@ # SPDX-License-Identifier: Apache-2.0 # -name : Zombie Builder +name: Zombie Builder on: - schedule: - - cron: '0 2 * * 0' # Update zombie builder image each Sunday night +# schedule: +# - cron: '0 2 * * 0' # Update zombie builder image each Sunday night workflow_dispatch: + inputs: + polkadot_image_rebuild: + description: 'Rebuild the polkadot builder image' + required: false + default: 'false' + type: 'boolean' + polkadot_binaries_rebuild: + description: 'Rebuild the polkadot binaries' + required: false + default: 'true' + type: 'boolean' + zombie_tester_image_rebuild: + description: 'Rebuild the zombie tester image' + required: false + default: 'false' + type: 'boolean' + +env: + DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ + GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }} + GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }} + CACHE_VERSION: v001 + CACHE_PATHS: ./zombienet/docker/cargo jobs: - build_and_push_zombie_builder: + build_polkadot_builder: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name : "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./zombienet/docker + run: make get_versions + + - name: "Check if image tag exists" + id: check-tag + working-directory: ./zombienet/docker + run: | + POLKADOT_VERSION=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) + POLKADOT_BUILDER_IMAGE=${DOCKER_REGISTRY_PATH}polkadot_builder:${POLKADOT_VERSION} + TAG_EXISTS=$(docker manifest inspect ${POLKADOT_BUILDER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") + echo "exists=$TAG_EXISTS" >> $GITHUB_ENV + echo "polkadot_builder_image=$POLKADOT_BUILDER_IMAGE" >> $GITHUB_OUTPUT + + - name: "Build Polkadot Builder" + working-directory: ./zombienet/docker + if: env.exists == 'false' || github.event.inputs.polkadot_image_rebuild == 'true' + run: make polkadot_builder \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + + - name: "Push Polkadot Builder" + working-directory: ./zombienet/docker + if: env.exists == 'false' || github.event.inputs.polkadot_image_rebuild == 'true' + run: make polkadot_builder_push \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} + + + building_binaries: runs-on: [ actions-runner-controller ] + needs: [build_polkadot_builder] timeout-minutes: 120 - container: qdrvm/kagome-dev@sha256:2d70246c32418a3dd45c246d3f5c2dd99bdafde145b903271849affe476c4cfc + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/cache@v4 + with: + path: ${{ env.CACHE_PATHS }} + key: ${{ runner.os }}-rust-${{ env.CACHE_VERSION }} + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./zombienet/docker + run: make get_versions + + - name: "Setup package version if not rebuilding" + if: github.event.inputs.polkadot_binaries_rebuild == 'false' + working-directory: ./zombienet/docker + run: | + ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') + REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') + gcloud config set artifacts/repository $ARTIFACTS_REPO + gcloud config set artifacts/location $REGION + POLKADOT_BINARY_PACKAGE_VERSION=$(gcloud artifacts versions list --package=polkadot-binary --format=json | \ + jq -r 'sort_by(.createTime) | .[-1] | {version: (.name | split("/") | last), createTime: .createTime} | .version') + echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" | tee -a variables.env + + - name: "Build target" + if: github.event.inputs.polkadot_binaries_rebuild == 'true' + working-directory: ./zombienet/docker + run: make polkadot_binary \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + SCCACHE_GCS_BUCKET=${{ secrets.SCCACHE_GCS_BUCKET }} \ + + - name: "Push Polkadot APT Package" + if: github.event.inputs.polkadot_binaries_rebuild == 'true' + working-directory: ./zombienet/docker + id: push_apt_package + run: | + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + POLKADOT_RELEASE_GLOBAL_NUMERIC=$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) + POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_RELEASE_GLOBAL_NUMERIC}-${SHORT_COMMIT_HASH}" + echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" | tee -a variables.env + make upload_apt_package + + - name: "Upload Variables Artifact" + uses: actions/upload-artifact@v4 + with: + name: variables-artifact + path: ./zombienet/docker/variables.env + +# - name: Archive build outputs +# uses: actions/upload-artifact@v4 +# with: +# name: build-output +# path: ./zombienet/docker/polkadot_binary/ + + + build_zombie_tester: + runs-on: ubuntu-latest + needs: [building_binaries] + timeout-minutes: 60 steps: - - uses: actions/checkout@v4 - - name: build zombie-builder - working-directory: ./zombienet/docker - run: make builder_last - - uses: azure/docker-login@v2 - with: - login-server: https://index.docker.io/v1/ - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_TOKEN }} - - name: Push zombie-builder docker image - working-directory: ./zombienet/docker - run: make builder_push + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name : "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' + with: + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./zombienet/docker + run: make get_versions + + - name: "Download Variables Artifact" + uses: actions/download-artifact@v4 + with: + name: variables-artifact + path: . + + - name: "Load Variables" + id: load-variables + run: | + source variables.env + echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" >> $GITHUB_ENV + + - name: "Check if image tag exists" + id: check-tag + working-directory: ./zombienet/docker + run: | + ZOMBIENET_RELEASE=$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2) + POLKADOT_SDK_RELEASE=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2) + ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${POLKADOT_SDK_RELEASE}_${ZOMBIENET_RELEASE} + TAG_EXISTS=$(docker manifest inspect ${ZOMBIE_TESTER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false") + echo "exists=$TAG_EXISTS" >> $GITHUB_ENV + echo "zombie_tester_image=$ZOMBIE_TESTER_IMAGE" >> $GITHUB_OUTPUT + + - name: "Build Polkadot Builder" + working-directory: ./zombienet/docker + if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' + run: make zombie_tester \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} \ + PROJECT_ID=${{ secrets.GCP_PROJECT }} \ + POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_BINARY_PACKAGE_VERSION}" + + - name: "Push Polkadot Builder" + working-directory: ./zombienet/docker + if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true' + run: make zombie_tester_push \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + \ No newline at end of file diff --git a/.github/workflows/zombie-tests.yaml b/.github/workflows/zombie-tests.yaml index 06f136572a..d9ae6ade3d 100644 --- a/.github/workflows/zombie-tests.yaml +++ b/.github/workflows/zombie-tests.yaml @@ -8,146 +8,209 @@ name: Zombie-Tests on: workflow_dispatch: - + inputs: + build_type: + description: 'Build type' + required: true + default: 'Release' + type: 'choice' + options: + - Release + - Debug + - RelWithDebInfo + werror: + description: 'Enable -Werror' + required: true + default: 'OFF' + type: 'choice' + options: + - 'ON' + - 'OFF' + env: - CACHE_VERSION: v07 - CACHE_PATHS: | - ~/Library/Caches/pip - ~/.cargo - ~/.ccache - ~/.hunter - ~/.rustup + DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/ + BUILD_DIR: build + CTEST_OUTPUT_ON_FAILURE: 1 + CI: true + + CACHE_VERSION: v01 + CACHE_PATH: ./build/cache jobs: - Prepare-Kagome-Docker: - strategy: - fail-fast: false - matrix: - options: -# - name: "Self-hosted: Debug Docker image" -# build-type: "Debug" - - name: "Self-hosted: Release Docker image" - build-type: "Release" -# - name: "Self-hosted: RelWithDebInfo Docker image" -# build-type: "RelWithDebInfo" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - run: git config --global --add safe.directory /__w/kagome/kagome - - run: git fetch --prune --unshallow - - name: Build Kagome - env: - BUILD_TYPE: "${{ matrix.options.build-type }}" - KAGOME_ROOT: /__w/kagome/kagome - run: bash -c "cd $KAGOME_ROOT && source /venv/bin/activate && ./housekeeping/docker/kagome-dev/make.sh" - - name: Copy Kagome - env: - KAGOME_BUILD_DIR: /__w/kagome/kagome/build - run: bash -c "mkdir -p ${KAGOME_BUILD_DIR}/docker_context && cp -a ${KAGOME_BUILD_DIR}/node/kagome ${KAGOME_BUILD_DIR}/docker_context" - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3 - - name: Build Docker Image - uses: docker/build-push-action@v5 - with: - tags: kagome-dev:${{ github.run_number }} - file: ./housekeeping/docker/kagome/minideb-release.Dockerfile - context: /__w/kagome/kagome/build/docker_context - outputs: type=docker,dest=/tmp/kagome_dev.tar - - name: upload artifact - uses: actions/upload-artifact@v4 - with: - name: kagome-dev - path: /tmp/kagome_dev.tar - - Prepare-Zombie-Tests-Docker: + kagome_dev_docker_build: runs-on: [ actions-runner-controller ] - timeout-minutes: 60 - needs: [Prepare-Kagome-Docker] + timeout-minutes: 180 + steps: - - name: Download artifact - uses: actions/download-artifact@v4 + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' with: - name: kagome-dev - path: /tmp - - name: Load image + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" run: | - docker load --input /tmp/kagome_dev.tar - docker image ls -a - - name: Set owner of working dir recursively - run: sudo chown -R $(whoami) . - - uses: actions/checkout@v4 - - name: Extract branch name - shell: bash + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Check package version" + working-directory: ./housekeeping/docker/kagome-dev run: | - branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}} - echo "branch=${branch}" >> $GITHUB_OUTPUT - echo "Branch name: ${branch}" - id: extract_branch - - name: Build zombie-tester - working-directory: ./zombienet/docker - run: make tester BRANCH_NAME=${{ steps.extract_branch.outputs.branch }} KAGOME_IMAGE=kagome-dev:${{ github.run_number }} - - name: Archive zombie-tester - shell: bash - run: docker save -o /tmp/zombie-tester.tar zombie-tester:latest - - name: Upload artifact - uses: actions/upload-artifact@v4 + ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') + REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" + echo "PACKAGE_VERSION=${PACKAGE_VERSION}" + echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV + echo "short_commit_hash=${SHORT_COMMIT_HASH}" >> $GITHUB_ENV + gcloud config set artifacts/repository $ARTIFACTS_REPO + gcloud config set artifacts/location $REGION + PACKAGE_EXIST=$(gcloud artifacts versions list --package=kagome-dev --format=json | jq -e ".[] | select(.name | endswith(\"${PACKAGE_VERSION}\"))" > /dev/null && echo "True" || echo "False") + echo "package_exist=${PACKAGE_EXIST}" >> $GITHUB_ENV + + - name: "Cache dependencies" + if: env.package_exist == 'False' + id: cache-restore + uses: actions/cache/restore@v4 with: - name: zombie-tester - path: /tmp/zombie-tester.tar - + path: ${{ env.CACHE_PATH }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ github.event.inputs.build_type }}-${{ env.short_commit_hash }} + restore-keys: | + ${{ github.job }}-${{ env.CACHE_VERSION }}-${{ github.event.inputs.build_type }}- + + - name: "Build target" + if: env.package_exist == 'False' + working-directory: ./housekeeping/docker/kagome-dev + run: make kagome_dev_docker_build \ + DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \ + GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ + GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ + BUILD_TYPE=${{ github.event.inputs.build_type }} \ + WERROR=${{ github.event.inputs.werror }} + + - name: "Push Kagome APT Package" + if: env.package_exist == 'False' + working-directory: ./housekeeping/docker/kagome-dev + run: | + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + make upload_apt_package \ + BUILD_TYPE=${{ github.event.inputs.build_type }} + + - name: "Always Save Cache" + id: cache-save + if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') + uses: actions/cache/save@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + + Zombie-Tests: - runs-on: [ actions-runner-controller ] + runs-on: ubuntu-latest timeout-minutes: 60 - needs: [Prepare-Zombie-Tests-Docker] + needs: [kagome_dev_docker_build] strategy: fail-fast: false matrix: options: - name: "PVF preparation & execution time" test: "test-polkadot-functional-0001-parachains-pvf" - - name: "Disputes initiation, conclusion and lag" - test: "test-polkadot-functional-0002-parachains-disputes" +# - name: "Disputes initiation, conclusion and lag" +# test: "test-polkadot-functional-0002-parachains-disputes" - name: "BEEFY voting and finality, test MMR proofs. Assumes Rococo sessions of 1 minute" test: "test-polkadot-functional-0003-beefy-and-mmr" - - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" - test: "test-polkadot-functional-0004-parachains-garbage-candidate" - - name: "Past-session dispute slashing" - test: "test-polkadot-functional-0005-parachains-disputes-past-session" - - name: "Test if parachains make progress with most of approvals being tranch0" - test: "test-polkadot-functional-0006-parachains-max-tranche0" +# - name: "Dispute finality lag when 1/3 of parachain validators always attempt to include an invalid block" +# test: "test-polkadot-functional-0004-parachains-garbage-candidate" +# - name: "Past-session dispute slashing" +# test: "test-polkadot-functional-0005-parachains-disputes-past-session" +# - name: "Test if parachains make progress with most of approvals being tranch0" +# test: "test-polkadot-functional-0006-parachains-max-tranche0" - name: "Test if disputes triggered on finalized blocks within scope always end as valid" test: "test-polkadot-functional-0007-dispute-freshly-finalized" - name: "Test if disputes triggered on finalized blocks out of scope never get to be confirmed and concluded" test: "test-polkadot-functional-0008-dispute-old-finalized" - name: "Approval voting coalescing does not lag finality" test: "test-polkadot-functional-0009-approval-voting-coalescing" - - name: "Test validator disabling effects" - test: "test-polkadot-functional-0010-validator-disabling" +# - name: "Test validator disabling effects" +# test: "test-polkadot-functional-0010-validator-disabling" - name: "Test we are producing blocks at 6 seconds clip" test: "test-polkadot-functional-0011-async-backing-6-seconds-rate" steps: - - name: Download artifact - uses: actions/download-artifact@v4 + - name: "Checkout repository" + uses: actions/checkout@v4 + + - name: Create and enable swap + run: | + free -h + df -h + sudo fallocate -l 4G /swapfile + sudo chmod 600 /swapfile + sudo mkswap /swapfile + sudo swapon /swapfile + sudo swapon --show + free -h + df -h + + - name: "Authenticate with Google Cloud" + uses: 'google-github-actions/auth@v2' with: - name: zombie-tester - path: /tmp - - name: Load image + credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }} + + - name: "Set up Cloud SDK" + uses: 'google-github-actions/setup-gcloud@v2' + + - name: "Configure Docker for GCR" + run: | + gcloud auth configure-docker --quiet + gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet + + - name: "Get Polkadot and Zombienet versions" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Get Package Version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + ARTIFACTS_REPO=$(grep '^ARTIFACTS_REPO ?=' Makefile | sed 's/ARTIFACTS_REPO ?= //') + REGION=$(grep '^REGION ?=' Makefile | sed 's/REGION ?= //') + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + PACKAGE_VERSION="$(date +'%y.%m.%d')-${SHORT_COMMIT_HASH}-${{ github.event.inputs.build_type }}" + echo "PACKAGE_VERSION=${PACKAGE_VERSION}" + echo "package_version=${PACKAGE_VERSION}" >> $GITHUB_ENV + + - name: "Get Zombie Tester Version" run: | - docker load --input /tmp/zombie-tester.tar - docker image ls -a - - uses: actions/checkout@v4 - - name: Run test + ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:latest + echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" + echo "zombie_tester_image=${ZOMBIE_TESTER_IMAGE}" >> $GITHUB_ENV + + - name: "Run test" working-directory: ./zombienet/docker - run: make ${{ matrix.options.test }} ZOMBIE_TESTER_IMAGE=zombie-tester:latest - - name: Upload test logs as artifact + run: | + make ${{ matrix.options.test }} \ + ZOMBIE_TESTER_IMAGE=${{ env.zombie_tester_image }} \ + KAGOME_PACKAGE_VERSION=${{ env.package_version }} + free -h + df -h + + - name: "Upload test logs as artifact" if: always() uses: actions/upload-artifact@v4 with: name: logs-${{ matrix.options.test }} path: /tmp/test_logs + + - name: "Show system info" + if: always() + run: | + free -h + df -h diff --git a/CMakeLists.txt b/CMakeLists.txt index 1805bfdf37..633785b192 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ option(EMBEDDINGS "Embed developers assets" ON) option(PROFILING "Enable internal profiling instruments" OFF) option(BACKWARD "Enable stacktrace logging instruments" ON) option(CLEAR_OBJS "Clear object files" OFF) +option(WERROR "Turn all warnings into errors" OFF) if (NOT ($ENV{CI}) OR NOT ($ENV{GITHUB_ACTIONS})) set(_EXTERNAL_PROJECT_DEFAULT ON) @@ -125,6 +126,10 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang|GNU)$") add_flag(-Wno-format-nonliteral) # prints way too many warnings from spdlog add_flag(-Wno-gnu-zero-variadic-macro-arguments) # https://stackoverflow.com/questions/21266380/is-the-gnu-zero-variadic-macro-arguments-safe-to-ignore + if(WERROR) + add_flag(-Werror) # turn all warnings into errors + endif() + # cmake-format: on if ((("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang)$") AND (${CMAKE_CXX_COMPILER_VERSION} VERSION_GREATER_EQUAL "12.0")) @@ -148,7 +153,7 @@ if ("${CMAKE_CXX_COMPILER_ID}" MATCHES "^(AppleClang|Clang|GNU)$") add_flag(-Werror-non-virtual-dtor) # warn the user if a class with virtual functions has a non-virtual destructor. This helps catch hard to track down memory errors add_flag(-Werror-sign-compare) # warn the user if they compare a signed and unsigned numbers add_flag(-Werror-reorder) # field '$1' will be initialized after field '$2' - add_flag(-Werror-switch) # unhandled values in a switch statement + add_flag(-Werror-switch) # unhandled values in a switch statement # cmake-format: on endif () elseif ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC") diff --git a/cmake/functions.cmake b/cmake/functions.cmake index b031fec310..de822b4930 100644 --- a/cmake/functions.cmake +++ b/cmake/functions.cmake @@ -36,8 +36,9 @@ endfunction() # conditionally applies flag. If flag is supported by current compiler, it will be added to compile options. function(add_flag flag) - check_cxx_compiler_flag(${flag} FLAG_${flag}) - if(FLAG_${flag} EQUAL 1) + string(REPLACE "-" "_" flag_var ${flag}) + check_cxx_compiler_flag(${flag} FLAG${flag_var}) + if(FLAG${flag_var} EQUAL 1) add_compile_options(${flag}) endif() endfunction() diff --git a/housekeeping/docker/kagome-dev/.gitignore b/housekeeping/docker/kagome-dev/.gitignore new file mode 100644 index 0000000000..a27562ecf5 --- /dev/null +++ b/housekeeping/docker/kagome-dev/.gitignore @@ -0,0 +1,10 @@ +*-versions.txt +commit_hash.txt + +cumulus +polkadot + +cargo/ +pkg/ +polkadot_binary/ +kagome_runtime/ diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile new file mode 100644 index 0000000000..04396177c0 --- /dev/null +++ b/housekeeping/docker/kagome-dev/Makefile @@ -0,0 +1,173 @@ +# Shared Environment Variables +# bookworm (Debian 12) +MINIDEB_IMAGE ?= bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16 +## DOCKER_REGISTRY_PATH empty for local usage +DOCKER_REGISTRY_PATH ?= +GOOGLE_APPLICATION_CREDENTIALS ?= +PLATFORM ?= linux/amd64 +ARCHITECTURE ?= x86_64 +PACKAGE_ARCHITECTURE ?= amd64 + +# Debug, Release, RelWithDebInfo +BUILD_TYPE ?= Release + +# Generated versions +MINIDEB_HASH := $(subst bitnami/minideb@sha256:,,$(MINIDEB_IMAGE)) +MINIDEB_SHORT_HASH := $(shell echo $(MINIDEB_HASH) | cut -c1-7) +MINIDEB_TAG = $(MINIDEB_SHORT_HASH)_rust-$(RUST_VERSION) + +# kagome_builder_deb Variables +RUST_VERSION ?= 1.79.0 + +# kagome_dev_docker_build Variables +BUILD_DIR ?= build +CACHE_DIR := $(shell pwd)/../../../../kagome/$(BUILD_DIR)/cache +DEPENDENCIES ?= libstdc++6, zlib1g, libgcc-s1, libc6, libtinfo6, libseccomp2, libatomic1 +GITHUB_HUNTER_USERNAME ?= +GITHUB_HUNTER_TOKEN ?= +CTEST_OUTPUT_ON_FAILURE ?= 1 +WERROR ?= OFF + +# kagome_runtime_cache Variables +KAGOME_PACKAGE_VERSION ?= + +# upload_apt_package Variables +ARTIFACTS_REPO ?= kagome-apt +REGION ?= europe-north1 + + +export DOCKER_BUILDKIT=1 +# BUILDKIT_PROGRESS - auto, plain, tty, rawjson +export BUILDKIT_PROGRESS=auto + + +get_versions: + echo "full_commit_hash: `git rev-parse HEAD`" > commit_hash.txt + echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" >> commit_hash.txt + +kagome_builder_deb: + docker build --platform $(PLATFORM) \ + -t $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(MINIDEB_TAG) \ + -t $(DOCKER_REGISTRY_PATH)kagome_builder_deb:latest \ + -f kagome_builder_deb.Dockerfile \ + --build-arg RUST_VERSION=$(RUST_VERSION) \ + --build-arg BASE_IMAGE=$(MINIDEB_IMAGE) \ + --build-arg ARCHITECTURE=$(ARCHITECTURE) . + +kagome_builder_deb_push: + docker push $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(MINIDEB_TAG) ; \ + docker push $(DOCKER_REGISTRY_PATH)kagome_builder_deb:latest + +kagome_dev_docker_build: + $(MAKE) get_versions + mkdir -p \ + $(CACHE_DIR)/.cargo/git \ + $(CACHE_DIR)/.cargo/registry \ + $(CACHE_DIR)/.hunter \ + $(CACHE_DIR)/.cache/ccache ; \ + CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ + SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ + DOCKER_EXEC_RESULT=0 ; \ + echo "Build type: $(BUILD_TYPE)"; \ + docker run -d --name $$CONTAINER_NAME \ + --platform $(PLATFORM) \ + --entrypoint "/bin/bash" \ + -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ + -e BUILD_TYPE=$(BUILD_TYPE) \ + -e BUILD_THREADS=$$BUILD_THREADS \ + -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ + -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ + -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ + -e CTEST_OUTPUT_ON_FAILURE=$(CTEST_OUTPUT_ON_FAILURE) \ + -v $$(pwd)/../../../../kagome:/opt/kagome \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ + -v $(CACHE_DIR)/.cargo/git:/root/.cargo/git \ + -v $(CACHE_DIR)/.cargo/registry:/root/.cargo/registry \ + -v $(CACHE_DIR)/.hunter:/root/.hunter \ + -v $(CACHE_DIR)/.cache/ccache:/root/.cache/ccache \ + $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(MINIDEB_TAG) \ + -c "tail -f /dev/null"; \ + docker exec -t $$CONTAINER_NAME /bin/bash -c \ + "cd /opt/kagome && \ + git config --global --add safe.directory /opt/kagome && \ + git config --global --add safe.directory /root/.hunter/_Base/Cache/meta && \ + source /venv/bin/activate && \ + git submodule update --init && \ + echo \"Building in $$(pwd)\" && \ + cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ + cmake --build \"$(BUILD_DIR)\" --target kagome -- -j${BUILD_THREADS} && \ + mkdir -p /tmp/kagome && \ + cp /opt/kagome/$(BUILD_DIR)/node/kagome /tmp/kagome/kagome && \ + cd /opt/kagome/housekeeping/docker/kagome-dev && \ + ./build_apt_package.sh \ + \"$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)\" \ + $(PACKAGE_ARCHITECTURE) \ + kagome-dev \ + /tmp/kagome \ + 'Kagome Dev Debian Package' \ + '$(DEPENDENCIES)' ; \ + " || DOCKER_EXEC_RESULT=$$? ; \ + if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ + echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ + docker stop $$CONTAINER_NAME; \ + exit $$DOCKER_EXEC_RESULT; \ + fi; \ + docker stop $$CONTAINER_NAME + +upload_apt_package: + SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ + gcloud config set artifacts/location $(REGION); \ + gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev_$$(date +'%y.%m.%d')-$${SHORT_COMMIT_HASH}-$(BUILD_TYPE)_$(PACKAGE_ARCHITECTURE).deb + +runtime_cache: + CONTAINER_NAME=kagome_dev_runtime_cache_$$(openssl rand -hex 6); \ + RUNTIME_VERSION=$$(python3 get_wasmedge_version.py)-$$(date +'%y.%m.%d'); \ + DOCKER_EXEC_RESULT=0 ; \ + echo "Runtime version: $$RUNTIME_VERSION"; \ + docker run -d --name $$CONTAINER_NAME \ + --platform $(PLATFORM) \ + --entrypoint "/bin/bash" \ + -e RUNTIME_VERSION=$$RUNTIME_VERSION \ + -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.gcp/google_creds.json \ + -v $$(pwd)/../../../../kagome:/opt/kagome \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ + $(DOCKER_REGISTRY_PATH)zombie_tester:latest \ + -c "tail -f /dev/null"; \ + docker exec -t $$CONTAINER_NAME /bin/bash -c \ + "cd /opt/kagome/zombienet && \ + sed -i '1s/^#//' /etc/apt/sources.list.d/kagome.list && \ + install_packages kagome-dev && \ + ./precompile.sh && \ + cd /opt/kagome/housekeeping/docker/kagome-dev && \ + mkdir -p /tmp/kagome_runtime && \ + cp -r /tmp/kagome/runtimes-cache/* /tmp/kagome_runtime && \ + ./build_apt_package.sh \ + \"$${RUNTIME_VERSION}\" \ + $(PACKAGE_ARCHITECTURE) \ + kagome-dev-runtime \ + /tmp/kagome_runtime \ + 'Kagome Runtime Dev Debian Package' \ + 'kagome-dev' \ + /tmp/kagome/runtimes-cache/ ; \ + " || DOCKER_EXEC_RESULT=$$? ; \ + if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ + echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ + docker stop $$CONTAINER_NAME; \ + exit $$DOCKER_EXEC_RESULT; \ + fi; \ + docker stop $$CONTAINER_NAME + +upload_apt_package_runtime: + RUNTIME_VERSION=$$(python3 get_wasmedge_version.py)-$$(date +'%y.%m.%d'); \ + echo "Runtime version: $$RUNTIME_VERSION"; \ + gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ + gcloud config set artifacts/location $(REGION); \ + RUNTIME_PACKAGE_EXIST=$$(gcloud artifacts versions list --package=kagome-dev-runtime --format=json | jq -e ".[] | select(.name | endswith(\"${RUNTIME_VERSION}\"))" > /dev/null && echo "True" || echo "False"); \ + if [ "$$RUNTIME_PACKAGE_EXIST" = "True" ]; then \ + echo "Package with version $$RUNTIME_VERSION already exists"; \ + gcloud artifacts versions delete $$RUNTIME_VERSION --package=kagome-dev-runtime --quiet; \ + fi; \ + gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/kagome-dev-runtime_$${RUNTIME_VERSION}_$(PACKAGE_ARCHITECTURE).deb diff --git a/housekeeping/docker/kagome-dev/build_apt_package.sh b/housekeeping/docker/kagome-dev/build_apt_package.sh new file mode 100755 index 0000000000..9b776b904e --- /dev/null +++ b/housekeeping/docker/kagome-dev/build_apt_package.sh @@ -0,0 +1,101 @@ +#!/bin/bash + +set -euo pipefail + +usage() { + echo "Usage: $0 []" + echo "Example: $0 1.0 amd64 mypackage /path/to/artifacts 'libc6, libssl1.1' /usr/local/bin" + exit 1 +} + +log() { + local MESSAGE=$1 + echo "[INFO] $MESSAGE" +} + +if [ "$#" -lt 6 ] || [ "$#" -gt 7 ]; then + usage +fi + +VERSION=$1 +ARCHITECTURE=$2 +PACKAGE_NAME=$3 +ARTIFACTS_DIR=$4 +DESCRIPTION=$5 +DEPENDENCIES=$6 +BINARY_INSTALL_DIR=${7:-/usr/local/bin} + +MAINTAINER="Zak Fein " +HOMEPAGE="https://qdrvm.io" +DIR_NAME=${PACKAGE_NAME}_${VERSION}_${ARCHITECTURE} + +log "Validating parameters..." + +if [ -z "$VERSION" ]; then + echo "Error: Version is required." >&2 + exit 1 +fi + +if [ -z "$ARCHITECTURE" ]; then + echo "Error: Architecture is required." >&2 + exit 1 +fi + +if [ -z "$PACKAGE_NAME" ]; then + echo "Error: Package name is required." >&2 + exit 1 +fi + +if [ -z "$ARTIFACTS_DIR" ] || [ ! -d "$ARTIFACTS_DIR" ]; then + echo "Error: A valid artifacts directory is required." >&2 + exit 1 +fi + +if [ -z "$DESCRIPTION" ]; then + echo "Error: Description is required." >&2 + exit 1 +fi + +if [ -z "$DEPENDENCIES" ]; then + echo "Error: Dependencies are required." >&2 + exit 1 +fi + +log "Creating package directory structure..." +mkdir -p ./pkg/${DIR_NAME}/DEBIAN +mkdir -p ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR} + +log "Working directory: $(pwd)/pkg/" + +log "Copying artifacts..." +mv -f ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR}/ + +log "Package: ${PACKAGE_NAME}" +log "Version: ${VERSION}" +log "Maintainer: ${MAINTAINER}" +log "Depends: ${DEPENDENCIES}" +log "Architecture: ${ARCHITECTURE}" +log "Homepage: ${HOMEPAGE}" +log "Description: ${DESCRIPTION}" +log "Artifacts: ${ARTIFACTS_DIR}" +log "Binary Install Directory: ${BINARY_INSTALL_DIR}" +log "Directory: ./pkg/${DIR_NAME}${BINARY_INSTALL_DIR}/" + +log "Creating control file..." +cat < ./pkg/${DIR_NAME}/DEBIAN/control +Package: ${PACKAGE_NAME} +Version: ${VERSION} +Maintainer: ${MAINTAINER} +Depends: ${DEPENDENCIES} +Architecture: ${ARCHITECTURE} +Homepage: ${HOMEPAGE} +Description: ${DESCRIPTION} +EOF + +log "Building the package..." +dpkg --build ./pkg/${DIR_NAME} + +log "Displaying package info..." +dpkg-deb --info ./pkg/${DIR_NAME}.deb + +log "Package created successfully." diff --git a/housekeeping/docker/kagome-dev/get_wasmedge_version.py b/housekeeping/docker/kagome-dev/get_wasmedge_version.py new file mode 100644 index 0000000000..551a4427ba --- /dev/null +++ b/housekeeping/docker/kagome-dev/get_wasmedge_version.py @@ -0,0 +1,25 @@ +import re +import sys + +def extract_version_and_sha(config_file_path, package_name): + with open(config_file_path, 'r') as file: + content = file.read() + + version_pattern = re.compile(r'hunter_config\(\s*' + re.escape(package_name) + r'\s*.*?URL\s+.*?/tags/([^\s/]+)\.zip', re.DOTALL) + sha_pattern = re.compile(r'hunter_config\(\s*' + re.escape(package_name) + r'\s*.*?SHA1\s+(\w+)', re.DOTALL) + + version_match = version_pattern.search(content) + sha_match = sha_pattern.search(content) + + version = version_match.group(1) if version_match else None + sha = sha_match.group(1)[:7] if sha_match else None + + return version, sha + +if __name__ == "__main__": + config_file_path = '../../../cmake/Hunter/config.cmake' + package_name = 'WasmEdge' + + version, sha = extract_version_and_sha(config_file_path, package_name) + print(f"{version}-{sha}") + \ No newline at end of file diff --git a/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile new file mode 100644 index 0000000000..63be602aa3 --- /dev/null +++ b/housekeeping/docker/kagome-dev/kagome_builder_deb.Dockerfile @@ -0,0 +1,78 @@ +ARG AUTHOR="k.azovtsev@qdrvm.io " + +ARG BASE_IMAGE +ARG RUST_VERSION +ARG ARCHITECTURE=x86_64 + +FROM ${BASE_IMAGE} + +ARG AUTHOR +ENV AUTHOR=${AUTHOR} +LABEL org.opencontainers.image.authors="${AUTHOR}" +LABEL org.opencontainers.image.description="Kagome builder image" + +SHELL ["/bin/bash", "-c"] + +RUN install_packages \ + build-essential \ + ccache \ + clang-format-16 \ + clang-tidy-16 \ + curl \ + dpkg-dev \ + g++-12 \ + gcc-12 \ + gdb \ + gdbserver \ + git \ + gpg \ + gpg-agent \ + lcov \ + libclang-rt-16-dev \ + libgmp10 \ + libnsl-dev \ + libseccomp-dev \ + llvm-16-dev \ + make \ + mold \ + nano \ + python3 \ + python3-pip \ + python3-venv \ + software-properties-common \ + unzip \ + vim \ + wget \ + zlib1g-dev + +ARG RUST_VERSION +ENV RUST_VERSION=${RUST_VERSION} +ENV RUSTUP_HOME=/root/.rustup +ENV CARGO_HOME=/root/.cargo +ENV PATH="${CARGO_HOME}/bin:${PATH}" +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${RUST_VERSION} && \ + rustup default ${RUST_VERSION} + +RUN python3 -m venv /venv +RUN /venv/bin/python3 -m pip install --no-cache-dir --upgrade pip +RUN /venv/bin/pip install --no-cache-dir cmake==3.25 scikit-build requests gitpython gcovr pyyaml + +ENV HUNTER_PYTHON_LOCATION=/venv/bin/python3 + +ENV LLVM_ROOT=/usr/lib/llvm-16 +ENV LLVM_DIR=/usr/lib/llvm-16/lib/cmake/llvm/ +ENV PATH=${LLVM_ROOT}/bin:${LLVM_ROOT}/share/clang:${PATH} +ENV CC=gcc-12 +ENV CXX=g++-12 + +RUN update-alternatives --install /usr/bin/python python /venv/bin/python3 90 && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 80 && \ + \ + update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-16 50 && \ + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-16 50 && \ + update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-16/bin/clang-16 50 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 50 && \ + \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90 && \ + update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 90 diff --git a/housekeeping/docker/kagome-dev/minideb-testing.Dockerfile b/housekeeping/docker/kagome-dev/minideb-testing.Dockerfile new file mode 100644 index 0000000000..4908de1f09 --- /dev/null +++ b/housekeeping/docker/kagome-dev/minideb-testing.Dockerfile @@ -0,0 +1,85 @@ +# bookworm +ARG BASE_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16 +ARG RUST_VERSION=1.79.0 + +FROM ${BASE_IMAGE} + +SHELL ["/bin/bash", "-c"] + +ENV KAGOME_IN_DOCKER=1 + +RUN install_packages \ + gpg \ + gpg-agent \ + wget \ + vim \ + python3 \ + python3-pip \ + python3-venv \ + software-properties-common \ + gdb \ + gdbserver \ + curl + +RUN install_packages \ + mold \ + build-essential \ + gcc-12 \ + g++-12 \ + llvm-16-dev \ + clang-tidy-16 \ + clang-format-16 \ + libclang-rt-16-dev \ + make \ + git \ + ccache \ + lcov \ + zlib1g-dev \ + libgmp10 \ + libnsl-dev \ + libseccomp-dev \ + unzip + +ARG RUST_VERSION +ENV RUST_VERSION=${RUST_VERSION} +ENV RUSTUP_HOME=/root/.rustup +ENV CARGO_HOME=/root/.cargo +ENV PATH="${CARGO_HOME}/bin:${PATH}" +RUN curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${RUST_VERSION} && \ + rustup default ${RUST_VERSION} + +RUN python3 -m venv /venv +RUN /venv/bin/python3 -m pip install --no-cache-dir --upgrade pip +RUN /venv/bin/pip install --no-cache-dir cmake==3.25 scikit-build requests gitpython gcovr pyyaml + +ENV HUNTER_PYTHON_LOCATION=/venv/bin/python3 + +ENV LLVM_ROOT=/usr/lib/llvm-16 +ENV LLVM_DIR=/usr/lib/llvm-16/lib/cmake/llvm/ +ENV PATH=${LLVM_ROOT}/bin:${LLVM_ROOT}/share/clang:${PATH} +ENV CC=gcc-12 +ENV CXX=g++-12 + +RUN update-alternatives --install /usr/bin/python python /venv/bin/python3 90 && \ + update-alternatives --install /usr/bin/python python /usr/bin/python3 80 && \ + \ + update-alternatives --install /usr/bin/clang-tidy clang-tidy /usr/bin/clang-tidy-16 50 && \ + update-alternatives --install /usr/bin/clang-format clang-format /usr/bin/clang-format-16 50 && \ + update-alternatives --install /usr/bin/clang clang /usr/lib/llvm-16/bin/clang-16 50 && \ + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-16 50 && \ + \ + update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 90 && \ + update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-12 90 && \ + update-alternatives --install /usr/bin/gcov gcov /usr/bin/gcov-12 90 + +RUN install -m 0755 -d /etc/apt/keyrings && \ + curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc && \ + chmod a+r /etc/apt/keyrings/docker.asc && \ + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian \ + $(. /etc/os-release && echo "$VERSION_CODENAME") stable" | \ + tee /etc/apt/sources.list.d/docker.list > /dev/null && \ + apt-get update && \ + install_packages \ + docker-ce \ + docker-ce-cli \ + containerd.io diff --git a/zombienet/docker/.gitignore b/zombienet/docker/.gitignore index 72bb4cac3b..0bd9e91225 100644 --- a/zombienet/docker/.gitignore +++ b/zombienet/docker/.gitignore @@ -1,3 +1,9 @@ -*-version.txt +*-versions.txt +commit_hash.txt + cumulus -polkadot \ No newline at end of file +polkadot + +cargo/ +pkg/ +polkadot_binary/ diff --git a/zombienet/docker/Makefile b/zombienet/docker/Makefile index 704844af90..292317b0c8 100644 --- a/zombienet/docker/Makefile +++ b/zombienet/docker/Makefile @@ -1,99 +1,226 @@ -BRANCH_NAME ?= master -KAGOME_IMAGE ?= qdrvm/kagome:latest -ZOMBIE_TESTER_IMAGE ?= qdrvm/zombie-tester:latest -CONTAINER_NAME ?= zombienet-test +# Shared Environment Variables +# bookworm (Debian 12) +MINIDEB_IMAGE=bitnami/minideb@sha256:6cc3baf349947d587a9cd4971e81ff3ffc0d17382f2b5b6de63d6542bff10c16 +## DOCKER_REGISTRY_PATH empty for local usage +DOCKER_REGISTRY_PATH ?= +GOOGLE_APPLICATION_CREDENTIALS ?= +PLATFORM ?= linux/amd64 +ARCHITECTURE ?= amd64 + +# polkadot_builder Variables +RUST_IMAGE=rust:1.79-slim-bookworm +SCCACHE_VERSION ?= 0.7.4 + +# polkadot_binary Variables +SCCACHE_GCS_BUCKET ?= +CARGO_PACKETS=-p test-parachain-adder-collator -p polkadot-test-malus -p test-parachain-undying-collator +RESULT_BINARIES=./target/release/malus ./target/release/undying-collator ./target/release/adder-collator +RESULT_BIN_NAMES=malus undying-collator adder-collator +POLKADOT_BINARY_DEPENDENCIES=libstdc++6, zlib1g, libgcc-s1, libc6 + +# upload_apt_package Variables +ARTIFACTS_REPO ?= kagome-apt +REGION ?= europe-north1 + +# zombie_tester Variables +PROJECT_ID ?= +POLKADOT_BINARY_PACKAGE_VERSION ?= +# copy_logs_to_host Variables +# COPY_LOGS_TO_HOST: boolean flag to determine whether to copy logs to host COPY_LOGS_TO_HOST ?= true HOST_LOGS_PATH ?= /tmp/test_logs +CONTAINER_NAME ?= zombienet-test + +# tests Variables +ZOMBIE_TESTER_IMAGE ?= zombie_tester:latest +KAGOME_PACKAGE_VERSION ?= +WORKING_DIR := $(shell pwd)/../../../kagome +DELETE_IMAGE_AFTER_TEST ?= true + -CUMULUS_RELEASE ?= v0.9.420 -POLKADOT_RELEASE ?= v0.9.42 -POLKADOT_RELEASE_GLOBAL ?= v0.9.42 +export DOCKER_BUILDKIT=1 +# BUILDKIT_PROGRESS - auto, plain, tty, rawjson +export BUILDKIT_PROGRESS=auto -# run_test takes two arguments: + +# run_test takes two arguments: # (1) docker image name # (2) test file path -# COPY_LOGS_TO_HOST: boolean flag to determine whether to copy logs to host - define run_test @CONTAINER_NAME=$$(openssl rand -hex 6); \ - echo "Running test in container $$CONTAINER_NAME with params image:$(1), test:$(2)"; \ - docker run --name $$CONTAINER_NAME --platform linux/amd64 $(1) zombienet-linux-x64 test -p native $(2); \ + START_TIME=$$(date +%s); \ + echo "Running test in container $$CONTAINER_NAME with params \n image:$(1), test:$(2) and \n Kagome package version: $(KAGOME_PACKAGE_VERSION)"; \ + TEST_PATH=$$(echo $(2) | xargs); \ + docker run --name $$CONTAINER_NAME \ + --platform $(PLATFORM) \ + --entrypoint "/bin/bash" \ + -v $(WORKING_DIR):/home/nonroot/bin/kagome \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ + -e TEST_PATH=$$TEST_PATH \ + -e KAGOME_PACKAGE_VERSION=$(KAGOME_PACKAGE_VERSION) \ + -e GOOGLE_APPLICATION_CREDENTIALS=/root/.gcp/google_creds.json \ + $(1) \ + -c "sed -i '1s/^#//' /etc/apt/sources.list.d/kagome.list && \ + install_packages kagome-dev=$(KAGOME_PACKAGE_VERSION) kagome-dev-runtime && \ + zombienet-linux-x64 test -p native /home/nonroot/bin/$$TEST_PATH " ; \ TEST_EXIT_CODE=$$(docker inspect $$CONTAINER_NAME --format='{{.State.ExitCode}}'); \ if [ "$(COPY_LOGS_TO_HOST)" = "true" ]; then \ $(MAKE) copy_logs_to_host CONTAINER_NAME=$$CONTAINER_NAME; \ fi; \ + END_TIME=$$(date +%s); \ + DURATION=$$((END_TIME - START_TIME)); \ + MINUTES=$$((DURATION / 60)); \ + SECONDS=$$((DURATION % 60)); \ + echo "Test duration: $$MINUTES min $$SECONDS sec"; \ echo "Test finished with exit code $$TEST_EXIT_CODE"; \ + echo "Runtime cache directory content:"; \ + if [ "$(DELETE_IMAGE_AFTER_TEST)" = "true" ]; then \ + docker rm -f $$CONTAINER_NAME; \ + echo "Container $$CONTAINER_NAME removed"; \ + fi; \ exit $$TEST_EXIT_CODE endef -copy_logs_to_host: - @CONTAINER_NAME=$(CONTAINER_NAME); \ - FINISHED_CONTAINER_NAME=$$CONTAINER_NAME-finished; \ - FINISHED_IMAGE_NAME=$$CONTAINER_NAME-finished-image; \ - echo "Copying logs from container $$CONTAINER_NAME to host path $(HOST_LOGS_PATH)"; \ - docker commit $$CONTAINER_NAME $$FINISHED_IMAGE_NAME; \ - echo "Starting temporary container $$FINISHED_CONTAINER_NAME to copy logs"; \ - docker run -d --name $$FINISHED_CONTAINER_NAME --platform linux/amd64 --entrypoint "/bin/bash" $$FINISHED_IMAGE_NAME -c "tail -f /dev/null"; \ - mkdir -p $(HOST_LOGS_PATH); \ - DIRS_TO_COPY=$$(docker exec $$FINISHED_CONTAINER_NAME "/bin/bash" -c "find /tmp/ -type d -name 'zombie-*'"); \ - for DIR in $$DIRS_TO_COPY; do \ - docker cp "$$FINISHED_CONTAINER_NAME:$$DIR/logs" "$(HOST_LOGS_PATH)/$$(basename $$DIR)"; \ - done; \ - echo "Stop and removing container $$FINISHED_CONTAINER_NAME and image $$FINISHED_IMAGE_NAME"; \ - docker stop $$FINISHED_CONTAINER_NAME; \ - docker rm -f $$FINISHED_CONTAINER_NAME; \ - docker rmi $$FINISHED_IMAGE_NAME - -builder_last: - python version.py https://github.com/paritytech/polkadot-sdk && \ - python version.py https://github.com/paritytech/zombienet && \ - make fetch_and_run +get_versions: + python3 version.py https://github.com/paritytech/polkadot-sdk && \ + python3 version.py https://github.com/paritytech/zombienet + echo "full_commit_hash: `git rev-parse HEAD`" > commit_hash.txt + echo "short_commit_hash: `git rev-parse HEAD | head -c 7`" >> commit_hash.txt -fetch_and_run: +polkadot_builder: + $(MAKE) get_versions if [ -f polkadot-sdk-versions.txt ]; then \ POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL=$$(grep 'short_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE -t qdrvm/zombie-builder:latest \ - -f builder.Dockerfile \ - --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE \ - --build-arg POLKADOT_RELEASE_GLOBAL=$$POLKADOT_RELEASE_GLOBAL \ - --build-arg POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC . ; \ + docker build --platform $(PLATFORM) \ + -t $(DOCKER_REGISTRY_PATH)polkadot_builder:$$POLKADOT_SDK_RELEASE \ + -t $(DOCKER_REGISTRY_PATH)polkadot_builder:latest \ + -f polkadot_builder.Dockerfile \ + --build-arg RUST_IMAGE=$(RUST_IMAGE) \ + --build-arg SCCACHE_VERSION=$(SCCACHE_VERSION) \ + --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ else \ echo "One or more files are missing."; \ fi -builder_push: +polkadot_builder_push: if [ -f polkadot-sdk-versions.txt ]; then \ POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - docker push qdrvm/zombie-builder:$$POLKADOT_SDK_RELEASE ; \ - docker push qdrvm/zombie-builder:latest ; \ + docker push $(DOCKER_REGISTRY_PATH)polkadot_builder:$$POLKADOT_SDK_RELEASE ; \ + docker push $(DOCKER_REGISTRY_PATH)polkadot_builder:latest ; \ else \ echo "One or more files are missing."; \ - fi + fi -builder: - DOCKER_BUILDKIT=0 docker build -t qdrvm/zombie-builder:latest -f builder.Dockerfile --build-arg CUMULUS_RELEASE=$(CUMULUS_RELEASE) --build-arg POLKADOT_RELEASE=$(POLKADOT_RELEASE) --build-arg POLKADOT_RELEASE_GLOBAL=$(POLKADOT_RELEASE_GLOBAL) . +polkadot_binary: + if [ -f polkadot-sdk-versions.txt ]; then \ + CONTAINER_NAME=polkadot_build_$$(openssl rand -hex 6); \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + DOCKER_EXEC_RESULT=0 ; \ + mkdir -p ./polkadot_binary ./cargo/registry ./cargo/git ./pkg; \ + docker run -d --name $$CONTAINER_NAME \ + --platform $(PLATFORM) \ + --entrypoint "/bin/bash" \ + -e RUSTC_WRAPPER=sccache \ + -e SCCACHE_GCS_BUCKET=$(SCCACHE_GCS_BUCKET) \ + -e SCCACHE_GCS_KEY_PATH=/root/.gcp/google_creds.json \ + -e SCCACHE_GCS_KEY_PREFIX=polkadot_builder \ + -e SCCACHE_GCS_RW_MODE=READ_WRITE \ + -e SCCACHE_LOG=info \ + -e ARCHITECTURE=$(ARCHITECTURE) \ + -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ + -e POLKADOT_RELEASE_GLOBAL_NUMERIC=$$POLKADOT_RELEASE_GLOBAL_NUMERIC \ + -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ + -v ./polkadot_binary:/tmp/polkadot_binary \ + -v ./cargo/registry:/usr/local/cargo/registry/ \ + -v ./cargo/git:/usr/local/cargo/git/ \ + -v ./pkg:/home/nonroot/polkadot-sdk/pkg \ + -v ./build_apt_package.sh:/home/nonroot/polkadot-sdk/build_apt_package.sh \ + $(DOCKER_REGISTRY_PATH)polkadot_builder:$$POLKADOT_SDK_RELEASE \ + -c "tail -f /dev/null"; \ + docker exec -t $$CONTAINER_NAME /bin/bash -c \ + "cargo update $(CARGO_PACKETS) && \ + cargo build --release $(CARGO_PACKETS) && \ + cp $(RESULT_BINARIES) /tmp/polkadot_binary/ && \ + ./build_apt_package.sh \ + $$POLKADOT_RELEASE_GLOBAL_NUMERIC-$$SHORT_COMMIT_HASH \ + $(ARCHITECTURE) \ + polkadot-binary \ + /tmp/polkadot_binary \ + 'Polkadot binaries: $(RESULT_BIN_NAMES)' \ + '$(POLKADOT_BINARY_DEPENDENCIES)' ; \ + " || DOCKER_EXEC_RESULT=$$? ; \ + if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ + echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ + docker stop $$CONTAINER_NAME; \ + exit $$DOCKER_EXEC_RESULT; \ + fi; \ + docker stop $$CONTAINER_NAME; \ + else \ + echo "One or more files are missing."; \ + fi -tester: - python3 version.py https://github.com/paritytech/polkadot-sdk && \ +upload_apt_package: + SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ + POLKADOT_RELEASE_GLOBAL_NUMERIC=$$(grep 'numeric_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + gcloud config set artifacts/repository $(ARTIFACTS_REPO); \ + gcloud config set artifacts/location $(REGION); \ + gcloud artifacts versions delete $$POLKADOT_RELEASE_GLOBAL_NUMERIC-$$SHORT_COMMIT_HASH --package=polkadot-binary --quiet || true ; \ + gcloud artifacts apt upload $(ARTIFACTS_REPO) --source=./pkg/polkadot-binary_$$POLKADOT_RELEASE_GLOBAL_NUMERIC-$${SHORT_COMMIT_HASH}_$(ARCHITECTURE).deb + +zombie_tester: if [ -f polkadot-sdk-versions.txt ]; then \ + ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ - BRANCH_NAME="$(BRANCH_NAME)" \ - KAGOME_IMAGE="$(KAGOME_IMAGE)" \ - DOCKER_BUILDKIT=0 docker build --platform linux/amd64 --no-cache -t qdrvm/zombie-tester:latest -t zombie-tester:latest -f tester.Dockerfile \ - --build-arg BRANCH_NAME=$$BRANCH_NAME \ - --build-arg KAGOME_IMAGE=$$KAGOME_IMAGE \ - --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ + PROJECT_ID=$(PROJECT_ID) \ + docker build \ + --platform $(PLATFORM) \ + --no-cache \ + -t $(DOCKER_REGISTRY_PATH)zombie_tester:latest \ + -t $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} \ + --secret id=google_creds,src=$(GOOGLE_APPLICATION_CREDENTIALS) \ + -f zombie_tester.Dockerfile \ + --build-arg BASE_IMAGE=$(MINIDEB_IMAGE) \ + --build-arg PROJECT_ID=$$PROJECT_ID \ + --build-arg POLKADOT_BINARY_PACKAGE_VERSION="$(POLKADOT_BINARY_PACKAGE_VERSION)" \ + --build-arg ZOMBIENET_RELEASE=$$ZOMBIENET_RELEASE \ + --build-arg POLKADOT_SDK_RELEASE=$$POLKADOT_SDK_RELEASE . ; \ + else \ + echo "One or more files are missing."; \ + fi + +zombie_tester_push: + if [ -f polkadot-sdk-versions.txt ]; then \ + ZOMBIENET_RELEASE=$$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2); \ + POLKADOT_SDK_RELEASE=$$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2); \ + docker push $(DOCKER_REGISTRY_PATH)zombie_tester:$${POLKADOT_SDK_RELEASE}_$${ZOMBIENET_RELEASE} ; \ + docker push $(DOCKER_REGISTRY_PATH)zombie_tester:latest ; \ else \ echo "One or more files are missing."; \ fi -tests: - docker run --platform linux/amd64 -it qdrvm/zombie-tester:latest bash +copy_logs_to_host: + @CONTAINER_NAME=$(CONTAINER_NAME); \ + FINISHED_CONTAINER_NAME=$$CONTAINER_NAME-finished; \ + FINISHED_IMAGE_NAME=$$CONTAINER_NAME-finished-image; \ + echo "Copying logs from container $$CONTAINER_NAME to host path $(HOST_LOGS_PATH)"; \ + docker commit $$CONTAINER_NAME $$FINISHED_IMAGE_NAME; \ + echo "Starting temporary container $$FINISHED_CONTAINER_NAME to copy logs"; \ + docker run -d --name $$FINISHED_CONTAINER_NAME --platform $(PLATFORM) --entrypoint "/bin/bash" $$FINISHED_IMAGE_NAME -c "tail -f /dev/null"; \ + mkdir -p $(HOST_LOGS_PATH); \ + DIRS_TO_COPY=$$(docker exec $$FINISHED_CONTAINER_NAME "/bin/bash" -c "find /tmp/ -type d -name 'zombie-*'"); \ + for DIR in $$DIRS_TO_COPY; do \ + docker cp "$$FINISHED_CONTAINER_NAME:$$DIR/logs" "$(HOST_LOGS_PATH)/$$(basename $$DIR)"; \ + done; \ + echo "Logs copied to $(HOST_LOGS_PATH)"; \ + echo "Runtime cache directory content:"; \ + docker exec $$FINISHED_CONTAINER_NAME "/bin/bash" -c "ls -la /tmp/kagome/runtimes-cache/" ; \ + echo "Stop and removing container $$FINISHED_CONTAINER_NAME and image $$FINISHED_IMAGE_NAME"; \ + docker stop $$FINISHED_CONTAINER_NAME; \ + docker rm -f $$FINISHED_CONTAINER_NAME; \ + docker rmi $$FINISHED_IMAGE_NAME test-polkadot-functional-0001-parachains-pvf: $(call run_test, $(ZOMBIE_TESTER_IMAGE), "kagome/zombienet/polkadot/functional/0001-parachains-pvf.zndsl") diff --git a/zombienet/docker/build_apt_package.sh b/zombienet/docker/build_apt_package.sh new file mode 100755 index 0000000000..5c82325b2e --- /dev/null +++ b/zombienet/docker/build_apt_package.sh @@ -0,0 +1,93 @@ +#!/bin/bash + +set -euo pipefail + +# Function to display usage +usage() { + echo "Usage: $0 " + echo "Example: $0 1.0 amd64 mypackage /path/to/artifacts 'libc6, libssl1.1'" + exit 1 +} + +# Function to log messages +log() { + local MESSAGE=$1 + echo "[INFO] $MESSAGE" +} + +# Check for the correct number of arguments +if [ "$#" -ne 6 ]; then + usage +fi + +# Assign variables from arguments +VERSION=$1 +ARCHITECTURE=$2 +PACKAGE_NAME=$3 +ARTIFACTS_DIR=$4 +DESCRIPTION=$5 +DEPENDENCIES=$6 + +# Define maintainer and homepage variables +MAINTAINER="Zak Fein " +HOMEPAGE="https://qdrvm.io" +DIR_NAME=${PACKAGE_NAME}_${VERSION}_${ARCHITECTURE} + +# Validate parameters +log "Validating parameters..." + +if [ -z "$VERSION" ]; then + echo "Error: Version is required." >&2 + exit 1 +fi + +if [ -z "$ARCHITECTURE" ]; then + echo "Error: Architecture is required." >&2 + exit 1 +fi + +if [ -z "$PACKAGE_NAME" ]; then + echo "Error: Package name is required." >&2 + exit 1 +fi + +if [ -z "$ARTIFACTS_DIR" ] || [ ! -d "$ARTIFACTS_DIR" ]; then + echo "Error: A valid artifacts directory is required." >&2 + exit 1 +fi + +if [ -z "$DESCRIPTION" ]; then + echo "Error: Description is required." >&2 + exit 1 +fi + +if [ -z "$DEPENDENCIES" ]; then + echo "Error: Dependencies are required." >&2 + exit 1 +fi + +log "Creating package directory structure..." +mkdir -p ./pkg/${DIR_NAME}/DEBIAN +mkdir -p ./pkg/${DIR_NAME}/usr/local/bin + +log "Copying artifacts..." +mv -f ${ARTIFACTS_DIR}/* ./pkg/${DIR_NAME}/usr/local/bin/ + +log "Creating control file..." +cat < ./pkg/${DIR_NAME}/DEBIAN/control +Package: ${PACKAGE_NAME} +Version: ${VERSION} +Maintainer: ${MAINTAINER} +Depends: ${DEPENDENCIES} +Architecture: ${ARCHITECTURE} +Homepage: ${HOMEPAGE} +Description: ${DESCRIPTION} +EOF + +log "Building the package..." +dpkg --build ./pkg/${DIR_NAME} + +log "Displaying package info..." +dpkg-deb --info ./pkg/${DIR_NAME}.deb + +log "Package created successfully." diff --git a/zombienet/docker/builder.Dockerfile b/zombienet/docker/builder.Dockerfile deleted file mode 100644 index c4192892c4..0000000000 --- a/zombienet/docker/builder.Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -ARG POLKADOT_RELEASE_GLOBAL -ARG POLKADOT_RELEASE_GLOBAL_NUMERIC -ARG ZOMBIENET_RELEASE - -FROM node:bookworm-slim as zombie-builder -USER root - -RUN apt-get update && apt update && \ - apt-get install -y --no-install-recommends git ssh curl nano build-essential protobuf-compiler libprotobuf-dev time && \ - apt install -y --no-install-recommends clang ca-certificates openssl && \ - rm -rf /var/lib/apt/lists/* && \ - curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y -ENV RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/cargo PATH=/root/.cargo/bin:$PATH -RUN rustup default stable && rustup target add wasm32-unknown-unknown && rustup component add rust-src -RUN groupadd --gid 10001 nonroot && \ - useradd --home-dir /home/nonroot \ - --create-home \ - --shell /bin/bash \ - --gid nonroot \ - --groups nonroot \ - --uid 10000 nonroot -WORKDIR /home/nonroot/ - -FROM zombie-builder AS zombie-builder-polkadot-sdk -ARG POLKADOT_SDK_RELEASE -WORKDIR /home/nonroot/ -RUN git clone --depth 1 --branch $POLKADOT_SDK_RELEASE https://github.com/paritytech/polkadot-sdk.git - -FROM zombie-builder-polkadot-sdk AS zombie-builder-polkadot-sdk-bin -WORKDIR /home/nonroot/polkadot-sdk/ -RUN cargo update -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator -RUN cargo build --profile testnet -p test-parachain-adder-collator \ - -p polkadot-test-malus \ - -p test-parachain-undying-collator -RUN find /home/nonroot/polkadot-sdk/target/ -maxdepth 2 -print - -FROM docker.io/parity/polkadot:$POLKADOT_RELEASE_GLOBAL AS polkadot - -FROM docker.io/parity/polkadot-parachain:$POLKADOT_RELEASE_GLOBAL_NUMERIC AS polkadot-parachain - -FROM zombie-builder AS final - -RUN mkdir -p /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/malus /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/adder-collator /home/nonroot/bin -COPY --from=zombie-builder-polkadot-sdk-bin /home/nonroot/polkadot-sdk/target/testnet/undying-collator /home/nonroot/bin -COPY --from=polkadot /usr/bin/polkadot /home/nonroot/bin -COPY --from=polkadot /usr/lib/polkadot/polkadot-execute-worker /home/nonroot/bin -COPY --from=polkadot /usr/lib/polkadot/polkadot-prepare-worker /home/nonroot/bin -COPY --from=polkadot-parachain /usr/local/bin/polkadot-parachain /home/nonroot/bin -COPY polkadot-sdk-versions.txt /home/nonroot/polkadot-sdk-versions.txt -COPY zombienet-versions.txt /home/nonroot/zombienet-versions.txt -ENV PATH=/home/nonroot/bin:$PATH - -RUN apt-get update && \ - apt-get install -y --no-install-recommends software-properties-common curl gpg gpg-agent wget nano && \ - curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \ - echo \ - "deb http://deb.debian.org/debian/ experimental main" | tee -a /etc/apt/sources.list.d/docker.list > /dev/null && \ - wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - && \ - add-apt-repository -y "deb http://deb.debian.org/debian/ testing main" && \ - apt-get update && \ - apt-get install --no-install-recommends -y libstdc++6 libc6 libnsl2 && \ - rm -rf /var/lib/apt/lists/* - -ARG ZOMBIENET_RELEASE -RUN wget https://github.com/paritytech/zombienet/releases/download/$ZOMBIENET_RELEASE/zombienet-linux-x64 && \ - chmod +x zombienet-linux-x64 && \ - cp zombienet-linux-x64 /home/nonroot/bin && \ - rm -rf zombienet-linux-x64 - -RUN chown -R root. /home/nonroot diff --git a/zombienet/docker/polkadot_builder.Dockerfile b/zombienet/docker/polkadot_builder.Dockerfile new file mode 100644 index 0000000000..6f2dec11e0 --- /dev/null +++ b/zombienet/docker/polkadot_builder.Dockerfile @@ -0,0 +1,56 @@ +ARG AUTHOR="k.azovtsev@qdrvm.io " + +ARG ARCHITECTURE=x86_64 +ARG SCCACHE_VERSION + +ARG POLKADOT_SDK_RELEASE +ARG RUST_IMAGE + +FROM ${RUST_IMAGE} AS polkadot-sdk-builder + +ARG AUTHOR +ENV AUTHOR=${AUTHOR} +LABEL org.opencontainers.image.authors="${AUTHOR}" +LABEL org.opencontainers.image.description="Polkadot SDK builder image" + +WORKDIR /home/nonroot/ +SHELL ["/bin/bash", "-c"] + +ARG POLKADOT_SDK_RELEASE +ENV POLKADOT_SDK_RELEASE=$POLKADOT_SDK_RELEASE + +RUN apt-get update && \ + apt-get install --no-install-recommends --yes \ + git \ + wget \ + openssl \ + ca-certificates \ + build-essential \ + clang \ + protobuf-compiler \ + libprotobuf-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN rustup default stable && \ + rustup target add wasm32-unknown-unknown && \ + rustup component add rust-src + +RUN git clone --depth 1 --branch $POLKADOT_SDK_RELEASE https://github.com/paritytech/polkadot-sdk.git + +ARG SCCACHE_VERSION +ENV SCCACHE_VERSION=$SCCACHE_VERSION +ARG ARCHITECTURE +ENV ARCHITECTURE=$ARCHITECTURE + +# Version >0.7.4 has a bug - work with GCS is broken +RUN mkdir -p /tmp/download && \ + cd /tmp/download && \ + wget -q -O sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v${SCCACHE_VERSION}/sccache-v${SCCACHE_VERSION}-${ARCHITECTURE}-unknown-linux-musl.tar.gz && \ + tar xzf sccache.tar.gz && \ + ls -la && \ + mv sccache-v*/sccache /usr/local/bin/sccache && \ + chmod +x /usr/local/bin/sccache ; \ + rm -rf /tmp/download + +WORKDIR /home/nonroot/polkadot-sdk/ diff --git a/zombienet/docker/tester.Dockerfile b/zombienet/docker/tester.Dockerfile deleted file mode 100644 index 8626279e9c..0000000000 --- a/zombienet/docker/tester.Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -ARG POLKADOT_SDK_RELEASE -ARG BRANCH_NAME -ARG KAGOME_IMAGE - -FROM $KAGOME_IMAGE as kagome - -FROM qdrvm/zombie-builder:$POLKADOT_SDK_RELEASE AS tester -ARG BRANCH_NAME -ENV BRANCH_NAME=${BRANCH_NAME} - -COPY --from=kagome /usr/local/bin/kagome /home/nonroot/bin - -RUN echo "Cloning branch: ${BRANCH_NAME}" -RUN git clone https://github.com/qdrvm/kagome.git -b ${BRANCH_NAME} - -RUN mkdir /tmp/kagome -RUN cd kagome/zombienet && ./precompile.sh - -RUN mkdir /home/nonroot/.local && \ - chown nonroot:nonroot /home/nonroot/.local && \ - chown -R nonroot:nonroot /tmp -USER nonroot diff --git a/zombienet/docker/version.py b/zombienet/docker/version.py index eb405774e9..5703337927 100644 --- a/zombienet/docker/version.py +++ b/zombienet/docker/version.py @@ -3,7 +3,7 @@ def list_all_tags_for_remote_git_repo(repo_url): result = subprocess.run([ "git", "ls-remote", "--tags", repo_url - ], stdout=subprocess.PIPE, text=True) + ], stdout=subprocess.PIPE, text=True, timeout=60) output_lines = result.stdout.splitlines() tags = [ line.split("refs/tags/")[-1] for line in output_lines diff --git a/zombienet/docker/zombie_tester.Dockerfile b/zombienet/docker/zombie_tester.Dockerfile new file mode 100644 index 0000000000..407765756d --- /dev/null +++ b/zombienet/docker/zombie_tester.Dockerfile @@ -0,0 +1,130 @@ +ARG AUTHOR="k.azovtsev@qdrvm.io " + +ARG BASE_IMAGE +ARG RUST_VERSION=1.79.0 + +ARG PROJECT_ID +ARG POLKADOT_BINARY_PACKAGE_VERSION +ARG POLKADOT_SDK_RELEASE +ARG ZOMBIENET_RELEASE + +ARG REGION=europe-north1 +ARG ARCHITECTURE=x86_64 + + +FROM ${BASE_IMAGE} as zombie-tester + +ARG AUTHOR +ENV AUTHOR=${AUTHOR} +LABEL org.opencontainers.image.authors="${AUTHOR}" +LABEL org.opencontainers.image.description="Zombienet image" + +ARG ZOMBIENET_RELEASE +ENV ZOMBIENET_RELEASE=$ZOMBIENET_RELEASE +ARG POLKADOT_SDK_RELEASE +ENV POLKADOT_SDK_RELEASE=$POLKADOT_SDK_RELEASE + +RUN groupadd --gid 10000 nonroot && \ + useradd --home-dir /home/nonroot \ + --create-home \ + --shell /bin/bash \ + --gid nonroot \ + --groups nonroot \ + --uid 10000 nonroot + +WORKDIR /home/nonroot/ + +RUN install_packages \ + bash \ + wget \ + nano \ + ca-certificates \ + gnupg2 \ + curl + +SHELL ["/bin/bash", "-c"] + +RUN mkdir -p /home/nonroot/bin + +WORKDIR /home/nonroot/bin + +ENV PATH="/home/nonroot/bin:${PATH}" + +RUN wget -q https://github.com/paritytech/zombienet/releases/download/$ZOMBIENET_RELEASE/zombienet-linux-x64 && \ + chmod +x zombienet-linux-x64 && \ + ln -s /home/nonroot/bin/zombienet-linux-x64 /home/nonroot/bin/zombienet + +# Setup enterprise repository + +ARG REGION +ENV REGION=$REGION + +RUN curl -fsSL https://${REGION}-apt.pkg.dev/doc/repo-signing-key.gpg | \ + gpg --dearmor -o /usr/share/keyrings/${REGION}-apt-archive-keyring.gpg + +RUN curl -fsSL https://packages.cloud.google.com/apt/doc/apt-key.gpg | \ + gpg --dearmor -o /usr/share/keyrings/cloud-google-apt-archive-keyring.gpg + +RUN echo "deb [signed-by=/usr/share/keyrings/${REGION}-apt-archive-keyring.gpg] \ + http://packages.cloud.google.com/apt apt-transport-artifact-registry-stable main" | \ + tee -a /etc/apt/sources.list.d/artifact-registry.list + +RUN install_packages apt-transport-artifact-registry + +ARG PROJECT_ID +ENV PROJECT_ID=$PROJECT_ID + +RUN echo "deb [signed-by=/usr/share/keyrings/europe-north1-apt-archive-keyring.gpg] \ + ar+https://${REGION}-apt.pkg.dev/projects/${PROJECT_ID} kagome-apt main" | \ + tee -a /etc/apt/sources.list.d/kagome.list + +RUN sed -i 's|^\(\s*\)# *Service-Account-JSON ".*";|\1Service-Account-JSON "/root/.gcp/google_creds.json";|' \ + /etc/apt/apt.conf.d/90artifact-registry + +ARG POLKADOT_BINARY_PACKAGE_VERSION +ENV POLKADOT_BINARY_PACKAGE_VERSION=$POLKADOT_BINARY_PACKAGE_VERSION + +RUN mkdir -p /root/.gcp +RUN --mount=type=secret,id=google_creds cat /run/secrets/google_creds > /root/.gcp/google_creds.json && \ + install_packages polkadot-binary=${POLKADOT_BINARY_PACKAGE_VERSION} && \ + rm /root/.gcp/google_creds.json && sed -i '1s/^/#/' /etc/apt/sources.list.d/kagome.list + +# WARNING: Setup always returns 2, even when successful +RUN zombienet setup -y polkadot polkadot-parachain; \ + EXIT_CODE=$?; \ + if [ $EXIT_CODE -eq 2 ]; then \ + echo "Command exited with code 2, continuing..."; \ + else \ + echo "Command exited with code $EXIT_CODE"; \ + exit $EXIT_CODE; \ + fi; + +RUN ./polkadot --version && \ + ./polkadot-parachain --version && \ + ./zombienet version && \ + ./polkadot-execute-worker --version && \ + ./polkadot-prepare-worker --version && \ + malus --version && \ + adder-collator --version && \ + undying-collator --version + +RUN ln -s /home/nonroot/bin/zombienet-linux-x64 /usr/local/bin/zombienet && \ + ln -s /home/nonroot/bin/polkadot /usr/local/bin/polkadot && \ + ln -s /home/nonroot/bin/polkadot-parachain /usr/local/bin/polkadot-parachain && \ + ln -s /home/nonroot/bin/polkadot-execute-worker /usr/local/bin/polkadot-execute-worker && \ + ln -s /home/nonroot/bin/polkadot-prepare-worker /usr/local/bin/polkadot-prepare-worker + +RUN install_packages \ + curl \ + gpg \ + gpg-agent \ + git \ + time \ + software-properties-common \ + libstdc++6 \ + libc6 \ + libnsl2 \ + libtinfo6 \ + libseccomp2 \ + libatomic1 \ + ssh