update zombie tester build pipeline, install zombienet from npm

qdrvm · Feb 10, 2025 · 331fd22 · 331fd22
1 parent 2a898a5
commit 331fd22
Show file tree

Hide file tree

Showing 5 changed files with 203 additions and 107 deletions.
diff --git a/.github/workflows/zombie-builder.yml b/.github/workflows/zombie-builder.yml
@@ -37,30 +37,38 @@ on:
         description: 'Custom Polkadot SDK tag'
         required: false
         type: 'string'
-
-      # zombie_tester_image_rebuild:
-      #   description: 'Rebuild the zombie tester image'
-      #   required: false
-      #   default: false
-      #   type: 'boolean'
-      # polkadot_sdk_tag:
-      #   description: 'Custom Polkadot SDK tag'
-      #   required: false
-      #   type: 'string'
-      # zombie_tester_latest_tag:
-      #   description: 'Custom Zombie Tester tag'
-      #   default: 'latest'
-      #   required: false
-      #   type: 'string'
+      zombie_tester:
+        description: 'Build the zombie tester image'
+        required: false
+        default: 'false'
+        type: 'boolean'
+      zombie_tester_tag:
+        description: 'Custom Zombie Tester tag'
+        default: 'latest'
+        required: false
+        type: 'string'
+      zombienet_release:
+        description: 'Custom zombienet release'
+        default: '1.3.119'
+        required: false
+        type: 'string'
+      polkadot_package_version:
+        description: 'Custom Polkadot package version'
+        default: '2412-20250127'
+        required: false
+        type: 'string'
 
 env:
   DOCKER_REGISTRY_PATH: ${{ secrets.GCP_REGISTRY }}/${{ secrets.GCP_PROJECT }}/kagome-dev/
   GITHUB_HUNTER_USERNAME: ${{ secrets.HUNTER_USERNAME }}
   GITHUB_HUNTER_TOKEN: ${{ secrets.HUNTER_TOKEN }}
+  PROJECT_ID: ${{ secrets.GCP_PROJECT }}
   PLATFORM: ${{ github.event.inputs.platform || 'linux/amd64' }}
   POLKADOT_SDK_TAG: ${{ github.event.inputs.polkadot_sdk_tag }}
   BUILDER_LATEST_TAG: ${{ github.event.inputs.polkadot_builder_tag || 'latest' }}
-  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_latest_tag || 'latest' }}
+  TESTER_LATEST_TAG: ${{ github.event.inputs.zombie_tester_tag || 'latest' }}
+  ZOMBIENET_RELEASE: ${{ github.event.inputs.zombienet_release || '1.3.119' }}
+  POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH: ${{ github.event.inputs.polkadot_package_version || '2412-20250127' }}
   SCCACHE_GCS_BUCKET: ${{ secrets.SCCACHE_GCS_BUCKET }}
   CACHE_VERSION: v001
   CACHE_PATHS: ./zombienet/docker/build_docker/cargo
@@ -95,7 +103,8 @@ jobs:
 
       - name: Fetch master branch
         if: env.skip != 'true'
-        run: git fetch origin master:master
+        run: |
+          git fetch origin master:master
              
       - name : "Authenticate with Google Cloud"
         if: env.skip != 'true'
@@ -126,7 +135,8 @@ jobs:
       - name: "Build Polkadot Builder"
         working-directory: ./zombienet/docker
         if: env.exists == 'false' || env.skip != 'true'
-        run: make polkadot_builder && make polkadot_builder_push
+        run: |
+          make polkadot_builder && make polkadot_builder_push
         
       - name: "Versions Info"
         if: always() && env.skip != 'true'
@@ -154,7 +164,8 @@ jobs:
 
       - name: Fetch master branch
         if: env.skip != 'true'
-        run: git fetch origin master:master
+        run: |
+          git fetch origin master:master
              
       - name : "Authenticate with Google Cloud"
         if: env.skip != 'true'
@@ -185,7 +196,8 @@ jobs:
       - name: "Build Polkadot Builder"
         working-directory: ./zombienet/docker
         if: env.exists == 'false' || env.skip != 'true'
-        run: make polkadot_builder_push_manifest 
+        run: |
+          make polkadot_builder_push_manifest 
         
       - name: "Versions Info"
         if: always() && env.skip != 'true'
@@ -217,7 +229,8 @@ jobs:
           fetch-depth: 0
 
       - name: Fetch master branch
-        run: git fetch origin master:master
+        run: |
+          git fetch origin master:master
 
 #      - uses: actions/cache@v4
 #        with:
@@ -239,11 +252,13 @@ jobs:
           
       - name: "Build target"
         working-directory: ./zombienet/docker
-        run: make polkadot_binary
+        run: |
+          make polkadot_binary
           
       - name: "Push Polkadot APT Package"
         working-directory: ./zombienet/docker
-        run: make upload_apt_package
+        run: |
+          make upload_apt_package
 
       - name: "Debian package Info"
         if: always()
@@ -252,71 +267,141 @@ jobs:
           make polkadot_deb_package_info
 
           
-  # build_zombie_tester:
-  #   runs-on: ubuntu-latest
-  #   needs: [building_binaries]
-  #   timeout-minutes: 60
-  #   steps:
-  #     - name: "Checkout repository"
-  #       uses: actions/checkout@v4
+  build_zombie_tester:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - platform: linux/amd64
+            os: ubuntu-24.04
+          - platform: linux/arm64
+            os: ubuntu-24.04-arm
+    timeout-minutes: 60
+    env:
+      PLATFORM: ${{ matrix.platform }}
+    steps:
+      - name: "Сheck if zombie_tester is required"
+        run: |
+          if [ "${{ github.event.inputs.zombie_tester }}" != "true" ]; then
+            echo "Zombie Tester is not required"
+            echo "skip=true" >> $GITHUB_ENV
+          fi
+
+      - name: Checkout repository
+        if: env.skip != 'true'
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
 
-  #     - name : "Authenticate with Google Cloud"
-  #       uses: 'google-github-actions/auth@v2'
-  #       with:
-  #         credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
+      - name: Fetch master branch
+        if: env.skip != 'true'
+        run: |
+          git fetch origin master:master
+             
+      - name : "Authenticate with Google Cloud"
+        if: env.skip != 'true'
+        uses: 'google-github-actions/auth@v2'
+        with:
+          credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
 
-  #     - name: "Set up Cloud SDK"
-  #       uses: 'google-github-actions/setup-gcloud@v2'
+      - name: "Set up Cloud SDK"
+        if: env.skip != 'true'
+        uses: 'google-github-actions/setup-gcloud@v2'
 
-  #     - name: "Configure Docker for GCR"
-  #       run: |
-  #         gcloud auth configure-docker --quiet
-  #         gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
+      - name: "Configure Docker for GCR"
+        if: env.skip != 'true'
+        run: |
+          gcloud auth configure-docker --quiet
+          gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
+          
+      - name: "Check if image tag exists using Makefile"
+        if: env.skip != 'true'
+        id: check-tag
+        working-directory: ./zombienet/docker
+        run: |
+          make get_versions
+          exists=$(make zombie_tester_check_tag)
+          echo "Image Exists: $exists"
+          echo "exists=$exists" >> $GITHUB_ENV
 
-  #     - name: "Get Polkadot and Zombienet versions"
-  #       working-directory: ./zombienet/docker
-  #       run: make get_versions
+      - name: "Build Zombie Tester"
+        working-directory: ./zombienet/docker
+        if: env.exists == 'false' || env.skip != 'true'
+        run: | 
+          make zombie_tester && make zombie_tester_push
 
-  #     - name: "Download Variables Artifact"
-  #       uses: actions/download-artifact@v4
-  #       with:
-  #         name: variables-artifact
-  #         path: .
+      - name: "Build Zombie Tester"
+        working-directory: ./zombienet/docker
+        if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true'
+        run: |
+          make zombie_tester && 
+          make zombie_tester_push
+        
+      - name: "Versions Info"
+        if: always() && env.skip != 'true'
+        working-directory: ./zombienet/docker
+        run: |
+          make zombie_tester_image_info
+          
+  zombie_tester_manifest:
+    runs-on: ubuntu-24.04
+    timeout-minutes: 60
+    needs: [build_zombie_tester]
+    steps:
+      - name: "Сheck if zombie_tester is required"
+        run: |
+            if [ "${{ github.event.inputs.zombie_tester }}" != "true" ]; then
+            echo "Zombie Tester is not required"
+            echo "skip=true" >> $GITHUB_ENV
+            fi
+        
+      - name: Checkout repository
+        if: env.skip != 'true'
+        uses: actions/checkout@v4
+        with:
+            fetch-depth: 0
 
-  #     - name: "Load Variables"
-  #       id: load-variables
-  #       run: |
-  #         source variables.env
-  #         echo "POLKADOT_BINARY_PACKAGE_VERSION=${POLKADOT_BINARY_PACKAGE_VERSION}" >> $GITHUB_ENV
+      - name: Fetch master branch
+        if: env.skip != 'true'
+        run: |
+          git fetch origin master:master
+             
+      - name : "Authenticate with Google Cloud"
+        if: env.skip != 'true'
+        uses: 'google-github-actions/auth@v2'
+        with:
+            credentials_json: ${{ secrets.GCP_SERVICE_ACCOUNT_KEY }}
 
-  #     - name: "Check if image tag exists"
-  #       id: check-tag
-  #       working-directory: ./zombienet/docker
-  #       run: |
-  #         ZOMBIENET_RELEASE=$(grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2)
-  #         POLKADOT_SDK_RELEASE=$(grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2)
-  #         ZOMBIE_TESTER_IMAGE=${DOCKER_REGISTRY_PATH}zombie_tester:${POLKADOT_SDK_RELEASE}_${ZOMBIENET_RELEASE}
-  #         TAG_EXISTS=$(docker manifest inspect ${ZOMBIE_TESTER_IMAGE} > /dev/null 2>&1 && echo "true" || echo "false")
-  #         echo "exists=$TAG_EXISTS" >> $GITHUB_ENV
-  #         echo "zombie_tester_image=$ZOMBIE_TESTER_IMAGE" >> $GITHUB_OUTPUT
+      - name: "Set up Cloud SDK"
+        if: env.skip != 'true'
+        uses: 'google-github-actions/setup-gcloud@v2'
 
-  #     - name: "Build Zombie Tester"
-  #       working-directory: ./zombienet/docker
-  #       if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true'
-  #       run: make zombie_tester \
-  #         DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
-  #         GOOGLE_APPLICATION_CREDENTIALS=${GOOGLE_APPLICATION_CREDENTIALS} \
-  #         PROJECT_ID=${{ secrets.GCP_PROJECT }} \
-  #         TESTER_LATEST_TAG=${{ env.TESTER_LATEST_TAG }} \
-  #         POLKADOT_BINARY_PACKAGE_VERSION="${POLKADOT_BINARY_PACKAGE_VERSION}"
+      - name: "Configure Docker for GCR"
+        if: env.skip != 'true'
+        run: |
+            gcloud auth configure-docker --quiet
+            gcloud auth configure-docker ${{ secrets.GCP_REGISTRY }} --quiet
+        
+      - name: "Check if image tag exists using Makefile"
+        if: env.skip != 'true'
+        id: check-tag
+        working-directory: ./zombienet/docker
+        run: |
+            make get_versions
+            exists=$(make zombie_tester_check_tag)
+            echo "Image Exists: $exists"
+            echo "exists=$exists" >> $GITHUB_ENV
 
-  #     - name: "Push Zombie Tester"
-  #       working-directory: ./zombienet/docker
-  #       if: env.exists == 'false' || github.event.inputs.zombie_tester_image_rebuild == 'true'
-  #       run: make zombie_tester_push \
-  #         DOCKER_REGISTRY_PATH=${DOCKER_REGISTRY_PATH} \
-
-  #     - name: "Versions Info"
-  #       if: always()
-  #       run: |
-  #         echo "Zombie Tester Image: ${{ steps.check-tag.outputs.zombie_tester_image }}"
+      - name: "Build Zombie Tester"
+        working-directory: ./zombienet/docker
+        if: env.exists == 'false' || env.skip != 'true'
+        run: |
+          make zombie_tester_push_manifest
+        
+      - name: "Versions Info"
+        if: always() && env.skip != 'true'
+        working-directory: ./zombienet/docker
+        run: |
+            make zombie_tester_image_info
+      
diff --git a/zombienet/docker/Makefile.include/versions.mk b/zombienet/docker/Makefile.include/versions.mk
@@ -10,7 +10,7 @@ ifeq ($(POLKADOT_SDK_RELEASE),)
 	$(eval POLKADOT_SDK_RELEASE=$(shell grep 'polkadot_format_version:' polkadot-sdk-versions.txt | cut -d ' ' -f 2))
 endif
 ifeq ($(ZOMBIENET_RELEASE),)
-	$(eval ZOMBIENET_RELEASE=$(shell grep 'short_version:' zombienet-versions.txt | cut -d ' ' -f 2))
+	$(eval ZOMBIENET_RELEASE=$(shell grep 'numeric_version:' zombienet-versions.txt | cut -d ' ' -f 2))
 endif
 export POLKADOT_SDK_RELEASE
 POLKADOT_RELEASE_GLOBAL_NUMERIC=$(shell echo $(POLKADOT_SDK_RELEASE) | sed 's/[^0-9]//g' )
@@ -19,3 +19,7 @@ POLKADOT_DEB_PACKAGE_NAME=polkadot-binary_$(POLKADOT_RELEASE_GLOBAL_NUMERIC)-$(C
 export POLKADOT_DEB_PACKAGE_NAME
 POLKADOT_DEB_PACKAGE_VERSION=$(POLKADOT_RELEASE_GLOBAL_NUMERIC)-$(CURRENT_DATE)_$(ARCHITECTURE)
 export POLKADOT_DEB_PACKAGE_VERSION
+ifeq ($(POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH),)
+	POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH=$(POLKADOT_RELEASE_GLOBAL_NUMERIC)-$(CURRENT_DATE)
+endif
+export POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH
diff --git a/zombienet/docker/Makefile.include/zombie_tester.mk b/zombienet/docker/Makefile.include/zombie_tester.mk
@@ -11,7 +11,7 @@ zombie_tester: set_versions
 		--build-arg BASE_IMAGE_TAG=$(OS_IMAGE_TAG_WITH_HASH) \
 		--build-arg PROJECT_ID=$(PROJECT_ID) \
 		--build-arg RUST_VERSION=$(RUST_VERSION) \
-		--build-arg POLKADOT_BINARY_PACKAGE_VERSION="$(POLKADOT_BINARY_PACKAGE_VERSION)" \
+		--build-arg POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH="$(POLKADOT_DEB_PACKAGE_VERSION_NO_ARCH)" \
 		--build-arg ZOMBIENET_RELEASE=$(ZOMBIENET_RELEASE) \
 		--build-arg POLKADOT_SDK_RELEASE=$(POLKADOT_SDK_RELEASE) .
 

diff --git a/zombienet/docker/tester-entrypoint.sh b/zombienet/docker/tester-entrypoint.sh
@@ -2,10 +2,12 @@
 
 set -eo pipefail
 
+sed -i 's/^#//' /etc/apt/sources.list.d/kagome.list
 install_packages kagome-dev=$KAGOME_PACKAGE_VERSION kagome-dev-runtime
+sed -i '1s/^/#/' /etc/apt/sources.list.d/kagome.list
+
 echo "KAGOME_DEV_VERSION=`apt-cache policy kagome-dev | grep 'Installed:' | awk '{print $2}'`"
 echo "KAGOME_DEV_RUNTIME_VERSION=`apt-cache policy kagome-dev-runtime | grep 'Installed:' | awk '{print $2}'`"
 
-sed -i '1s/^/#/' /etc/apt/sources.list.d/kagome.list
 
 exec gosu $USER_NAME "$@"