[WIP] k8s workflow tests

.github/workflows/.gitignore

@@ -0,0 +1 @@
+terraform

.github/workflows/agglayer-regression-tests.yml

@@ -0,0 +1,95 @@
+name: Regression Tests - Agglayer
+'on':
+ workflow_dispatch:
+ inputs:
+ zkevm_agglayer_commit_id:
+ description: 0xPolygon/agglayer (commit id)
+ required: true
+ bake_time:
+ description: bake time (minutes)
+ required: false
+ default: 30
+ push:
+ branches:
+ - main
+ inputs:
+ zkevm_agglayer_commit_id:
+ description: 0xPolygon/agglayer commit id
+ required: false
+ default: '${{ github.event.before }}' # Set default value to commit ID being merged to main
+ bake_time:
+ description: bake time (minutes)
+ required: false
+ default: 20
+jobs:
+ deploy_devnet:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ ref: '${{ github.event.before }}'
+ - name: Set up Docker
+ uses: docker/setup-buildx-action@v1
+ - name: Clone and build agglayer image locally
+ run: |
+ git clone https://github.com/0xPolygon/agglayer.git
+ cd agglayer
+ git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+ docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+ - name: Install Foundry
+ uses: foundry-rs/foundry-toolchain@v1
+ - name: Clone internal kurtosis-cdk repo
+ run: |
+ git clone https://github.com/0xPolygon/kurtosis-cdk.git
+ cd kurtosis-cdk
+ git checkout dan/jit_containers
+ - name: Install kurtosis
+ run: |
+ echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+ sudo apt update
+ sudo apt install kurtosis-cli
+ - name: Run kurtosis agent in background
+ run: |
+ kurtosis gateway & # Run cmd in background
+ - name: Deploy CDK devnet on local github runner
+ run: |
+ cd kurtosis-cdk
+ kurtosis engine restart
+ kurtosis run --enclave cdk-v1 --args-file params.yml .
+ - name: Monitor and report any potential regressions to CI logs
+ run: |
+ bake_time="${{ github.event.inputs.bake_time }}"
+ end_minute=$(( $(date +'%M') + bake_time))
+
+ export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+ INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
+ incremented=false
+
+ while [ $(date +'%M') -lt $end_minute ]; do
+ # Attempt to connect to the service
+ if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+
+ # Check if STATUS has incremented
+ if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+ incremented=true
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+ exit 0
+ fi
+ else
+ echo "Failed to connect, waiting and retrying..."
+ sleep 60
+ continue
+ fi
+ sleep 60
+ done
+
+ if ! $incremented; then
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+ exit 1
+ fi
+ - name: Finally, remove all devnet resources locally
+ run: |
+ cd kurtosis-cdk
+ kurtosis clean -a

.github/workflows/cdk-kurtosis-gke-devnet.yml

@@ -0,0 +1,79 @@
+name: Regression Detector - Ephemeral CDK Devnet
+
+jobs:
+ deploy_devnet:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.before }}
+
+ - name: Set up Docker
+ uses: docker/setup-buildx-action@v1
+
+ - name: Setup Google Cloud SDK
+ uses: 'google-github-actions/setup-gcloud@v2'
+ with:
+ version: '>= 363.0.0'
+
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+ project_id: 'prj-polygonlabs-devtools-dev'
+
+ - name: Install kubectl and gcloud kubectl auth plugin
+ run: |
+ curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+ && sudo install -o root -g root -m 0755 kubectl /usr/local/bin/kubectl && gcloud components install gke-gcloud-auth-plugin
+
+ - name: Install tomlq for config file preparation
+ run: |
+ pip3 install tomlq
+
+ - name: Clone private kurtosis-cdk repo
+ run: |
+ eval `ssh-agent -s`
+ ssh-add - <<< '${{ secrets.SSH_PRIVATE_KEY }}'
+ git clone -b dan/contracts_running_state git@github.com:0xPolygon/kurtosis-cdk.git
+
+## add necessary manipulation steps here...
+ - name: Install kurtosis
+ run: |
+ echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+ sudo apt update
+ sudo apt install kurtosis-cli
+
+ - name: Setup necessary gke custom kurtosis netorking configs
+ run: | 
+ export KURTOSIS_CONFIG_PATH="$(kurtosis config path)"
+ cp /home/runner/work/bor/bor/kurtosis-config.yml $KURTOSIS_CONFIG_PATH
+
+ - name: Set kurtosis cloud env
+ run: | 
+ mkdir -p ~/.kube
+ echo "${{ secrets.GKE_CLUSTER_CREDENTIALS }}" > ~/.kube/config 
+ export KUBECONFIG=~/.kube/config && kurtosis cluster set cloud
+
+ - name: Run kurtosis agent in background
+ run: | 
+ kurtosis gateway & # Run cmd in background
+ sleep 10
+
+ - name: REMOVE, temp for CI testing remove all devnet resources
+ run: |
+ cd kurtosis-cdk
+ kurtosis clean -a
+ sleep 180
+
+ - name: Deploy CDK devnet to isolated GKE namespace
+ run: |
+ cd kurtosis-cdk
+ kurtosis engine restart
+ kurtosis run --enclave cdk-v1 --args-file params.yml .
+ sleep 600
+
+ - name: Finally, remove all devnet resources
+ run: |
+ cd kurtosis-cdk
+ kurtosis clean -a

.github/workflows/cdk-superuser-regression-tests.yml

@@ -0,0 +1,110 @@
+name: Regression Tests - Superuser
+'on':
+ workflow_dispatch:
+ inputs:
+ zkevm_agglayer_commit_id:
+ description: 0xPolygon/agglayer (commit id)
+ required: true
+ zkevm_bridge_service_commit_id:
+ description: 0xPolygonHermez/zkevm-bridge-service (commit id)
+ required: true
+ zkevm_bridge_ui_commit_id:
+ description: 0xPolygonHermez/zkevm-bridge-ui (commit id)
+ required: true
+ zkevm_dac_commit_id:
+ description: 0xPolygon/cdk-data-availability (commit id)
+ required: true
+ zkevm_node_commit_id:
+ description: 0xPolygon/cdk-validium-node (commit id)
+ required: true
+ bake_time:
+ description: bake time (minutes)
+ required: false
+ default: 30
+jobs:
+ deploy_devnet:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout repository
+ uses: actions/checkout@v2
+ - name: Set up Docker
+ uses: docker/setup-buildx-action@v1
+ - name: Clone and build agglayer
+ run: |
+ git clone https://github.com/0xPolygon/agglayer.git
+ cd agglayer
+ git checkout "${{ github.event.inputs.zkevm_agglayer_commit_id }}"
+ docker compose -f docker/docker-compose.yaml build --no-cache agglayer
+ - name: Clone and build zkevm-bridge-service
+ run: |
+ git clone https://github.com/0xPolygonHermez/zkevm-bridge-service.git
+ cd zkevm-bridge-service
+ git checkout "${{ github.event.inputs.zkevm_bridge_service_commit_id }}"
+ docker build -t zkevm-bridge-service:local -f ./Dockerfile .
+ - name: Clone and build zkevm-bridge-ui
+ run: |
+ git clone https://github.com/0xPolygonHermez/zkevm-bridge-ui.git
+ cd zkevm-bridge-ui
+ git checkout "${{ github.event.inputs.zkevm_bridge_ui_commit_id }}"
+ docker build -t zkevm-bridge-ui:local -f ./Dockerfile .
+ - name: Clone and build cdk-data-availability
+ run: |
+ git clone https://github.com/0xPolygon/cdk-data-availability.git
+ cd cdk-data-availability
+ git checkout "${{ github.event.inputs.zkevm_dac_commit_id }}"
+ docker build -t cdk-data-availability:local -f ./Dockerfile . 
+ - name: Clone and build cdk-validium-node
+ run: |
+ git clone https://github.com/0xPolygon/cdk-validium-node.git
+ cd cdk-validium-node
+ git checkout "${{ github.event.inputs.zkevm_node_commit_id }}"
+ docker build -t cdk-validium-node:local -f ./Dockerfile .
+ - name: Install Foundry
+ uses: foundry-rs/foundry-toolchain@v1
+ - name: Clone internal kurtosis-cdk repo
+ run: |
+ git clone https://github.com/0xPolygon/kurtosis-cdk.git
+ cd kurtosis-cdk
+ git checkout dan/jit_containers_superusers
+ - name: Install kurtosis
+ run: |
+ echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+ sudo apt update
+ sudo apt install kurtosis-cli
+ kurtosis analytics disable
+ - name: Deploy CDK devnet on local github runner
+ run: |
+ cd kurtosis-cdk
+ kurtosis run --enclave cdk-v1 --args-file params.yml .
+ - name: Monitor and report any potential regressions to CI logs
+ run: |
+ bake_time="${{ github.event.inputs.bake_time }}"
+ end_minute=$(( $(date +'%M') + bake_time))
+
+ export ETH_RPC_URL="$(kurtosis port print cdk-v1 zkevm-node-rpc-001 http-rpc)"
+ INITIAL_STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null)
+ incremented=false
+
+ while [ $(date +'%M') -lt $end_minute ]; do
+ # Attempt to connect to the service
+ if STATUS=$(cast rpc zkevm_verifiedBatchNumber 2>/dev/null); then
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER: $STATUS"
+
+ # Check if STATUS has incremented
+ if [ "$STATUS" != "$INITIAL_STATUS" ]; then
+ incremented=true
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER successfully incremented to $STATUS. Exiting..."
+ exit 0
+ fi
+ else
+ echo "Failed to connect, waiting and retrying..."
+ sleep 60
+ continue
+ fi
+ sleep 60
+ done
+
+ if ! $incremented; then
+ echo "ZKEVM_VERIFIED_BATCH_NUMBER did not increment. This may indicate chain experienced a regression. Please investigate."
+ exit 1
+ fi

.github/workflows/kind-config.yaml.sample

@@ -0,0 +1,18 @@
+apiVersion: kind.x-k8s.io/v1alpha4
+kind: Cluster
+networking:
+ apiServerAddress: "0.0.0.0"
+
+kubeadmConfigPatchesJSON6902:
+ - group: kubeadm.k8s.io
+ version: v1beta2
+ kind: ClusterConfiguration
+ patch: |
+ - op: add
+ path: /apiServer/certSANs/-
+ value: docker
+nodes:
+- role: control-plane
+ extraPortMappings:
+ - containerPort: 11111
+ hostPort: 11111

.github/workflows/make-k8s-images.yml

@@ -0,0 +1,39 @@
+name: Image Sync
+
+on:
+ schedule:
+ - cron: '*/30 * * * *'
+
+jobs:
+ sync-images:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v2
+ - name: Set up Docker
+ uses: docker/setup-buildx-action@v1
+ - uses: 'google-github-actions/auth@v2'
+ with:
+ credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
+ - name: 'Set up Cloud SDK'
+ uses: 'google-github-actions/setup-gcloud@v2'
+ with:
+ version: '>= 363.0.0'
+ - name: Fetch repos, rebuild for k8s, push to internal artifact registry
+ run: |
+ gcloud auth configure-docker europe-west2-docker.pkg.dev -q
+ while IFS= read -r image_path; do
+ latest_tag=$(curl -s "https://registry.hub.docker.com/v2/repositories/${image_path}/tags/" | jq -r '.results[].name | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' | sort -V | tail -n 1)
+ if [ -n "$latest_tag" ]; then
+ # Check if the image version already exists in GCP Container Registry
+ if ! gcloud container images list-tags europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path} --filter="tags=${latest_tag}" --format="value(tags)" | grep -q "${latest_tag}"; then
+ docker pull "${image_path}:${latest_tag}"
+ docker tag "${image_path}:${latest_tag}" "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+ sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:${latest_tag}"
+ # also tag image as 'latest' for internal tracking
+ sudo -u $USER docker push "europe-west2-docker.pkg.dev/prj-polygonlabs-devtools-dev/container/${image_path}:latest"
+ else
+ echo "Image version already exists for ${image_path}:${latest_tag} in GCR. Skipping upload."
+ fi
+ fi
+ done < .github/workflows/repo-list.txt.sample

.github/workflows/repo-list.txt.sample

@@ -0,0 +1,3 @@
+0xpolygon/bor
+0xpolygon/heimdall
+thorax/erigon

kurtosis-config.yml

@@ -0,0 +1,17 @@
+config-version: 2
+should-send-metrics: true
+kurtosis-clusters:
+ docker:
+ type: "docker"
+ minikube:
+ type: "kubernetes"
+ config:
+ kubernetes-cluster-name: "minikube"
+ storage-class: "standard"
+ enclave-size-in-megabytes: 400
+ cloud:
+ type: "kubernetes"
+ config:
+ kubernetes-cluster-name: "gke_prj-polygonlabs-devtools-dev_europe-west2_ci-cluster-1"
+ storage-class: "standard"
+ enclave-size-in-megabytes: 400