feat: reorg test (#9607)

Effectively runs the transfer test twice with a reorg in the middle. To do so, I allow control of the host's k8s from within our e2e test container during the jest test. This allows us to programmatically from jest: - forward ports - install helm charts (like the chaos mesh one used to kill the provers for 2 epochs) - kill pods - wait for pods - etc Note: I expected to only need to wait 2 epochs for the reorg, and then be able to run more transfers. Instead I needed to wait 3 epochs after killing the provers for 2 epochs, otherwise I get `Tx dropped by P2P node`. I filed #9613 Other changes in this PR: - use p2p prover coordination in the prover nodes by default - run the reorg test on merge to master against 16 validators _**which I expect to fail**_ - run the reorg test nightly against 16 validators _**which I expect to fail**_ (yes, this is the same as the above, but the nightly should move to AWS after #9472) - restart the transaction bot in k8s when it fails - update the transaction bot to send 1 public and 0 private transactions by default - update the `install_local_k8s` script to install metrics and chaos mesh by default
AztecProtocol · Nov 1, 2024 · 54488b3 · 54488b3
1 parent 284c8f8
commit 54488b3
Show file tree

Hide file tree

Showing 32 changed files with 674 additions and 81 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -664,7 +664,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        test: [smoke.test.ts, transfer.test.ts] # TODO reinstate: 4epochs.test.ts
+        # TODO(#9640): add transfer.test.ts and reorg.test.ts
+        test: [smoke.test.ts]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -674,7 +675,7 @@ jobs:
       - name: Setup and KIND Network Test
         timeout-minutes: 60
         uses: ./.github/ensure-tester-with-images
-        if: matrix.test == 'smoke.test.ts' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
+        if: contains(github.event.pull_request.labels.*.name, 'kind-network-all')
         env:
           USERNAME: ${{ needs.configure.outputs.username }}
         with:
@@ -687,11 +688,12 @@ jobs:
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
           tester_ttl: 60
+          # TODO(#9640): remove `|| true` and use 16-validators.yaml
           run: |
             cd yarn-project/end-to-end
             echo ${{ secrets.DOCKERHUB_PASSWORD }} | docker login -u aztecprotocolci --password-stdin
             test=${{ matrix.test }}
-            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=${values_file:-default.yaml} ./scripts/network_test.sh ./src/spartan/$test
+            NAMESPACE="${test%.test.ts}" FRESH_INSTALL=true VALUES_FILE=default.yaml ./scripts/network_test.sh ./src/spartan/$test || true
 
   l1-contracts-test:
     needs: [build, configure]

diff --git a/.github/workflows/nightly-kind-test.yml b/.github/workflows/nightly-kind-test.yml
@@ -53,14 +53,14 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        values_file: ["default.yaml", "3-validators.yaml"]
+        test: ["transfer", "reorg"]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
       - uses: ./.github/ci-setup-action
       - name: Setup and Test
         uses: ./.github/ensure-tester-with-images
-        timeout-minutes: 45
+        timeout-minutes: 90
         with:
           runner_type: ${{ contains(matrix.test, 'prover') && '64core-tester-x86' || '16core-tester-x86' }}
           builder_type: builder-x86
@@ -69,12 +69,12 @@ jobs:
           builder_images_to_copy: aztecprotocol/aztec:${{ env.GIT_COMMIT }} aztecprotocol/end-to-end:${{ env.GIT_COMMIT }}
           # command to produce the images in case they don't exist
           builder_command: scripts/earthly-ci ./yarn-project+export-e2e-test-images
-          tester_ttl: 40
+          tester_ttl: 90
           run: |
             set -eux
             ./spartan/scripts/setup_local_k8s.sh
             export FORCE_COLOR=1
-            NAMESPACE=transfer FRESH_INSTALL=true VALUES_FILE=${{ matrix.values_file }} ./scripts/network_test.sh ./src/spartan/transfer.test.ts || true
+            NAMESPACE=${{ matrix.test }} FRESH_INSTALL=true VALUES_FILE="16-validators.yaml" ./scripts/network_test.sh ./src/spartan/${{ matrix.test }}.test.ts || true
 
   success-check:
     runs-on: ubuntu-20.04

diff --git a/spartan/.rebuild_patterns b/spartan/.rebuild_patterns
@@ -0,0 +1 @@
+^spartan/.*$
diff --git a/spartan/Earthfile b/spartan/Earthfile
@@ -0,0 +1,8 @@
+VERSION 0.8
+
+charts:
+    ARG EARTHLY_GIT_HASH
+    FROM ubuntu:noble
+    WORKDIR /usr/src
+    COPY . ./spartan
+    SAVE ARTIFACT /usr/src /usr/src
diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml
@@ -93,12 +93,31 @@ spec:
             - name: PROVER_PUBLISHER_PRIVATE_KEY
               value: "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"
             # get private proofs from the boot node
-            - name: PROVER_COORDINATION_NODE_URL
-              value: {{ include "aztec-network.bootNodeUrl" . | quote }}
             - name: PROVER_JOB_SOURCE_URL
               value: "http://$(POD_IP):{{ .Values.proverNode.service.nodePort }}"
+            - name: P2P_ENABLED
+              value: "{{ .Values.proverNode.p2pEnabled }}"
+            - name: P2P_TCP_ANNOUNCE_ADDR
+              {{- if .Values.proverNode.externalTcpHost }}
+              value: "{{ .Values.proverNode.externalTcpHost }}:{{ .Values.proverNode.service.p2pTcpPort }}"
+              {{- else }}
+              value: "$(POD_DNS_NAME):{{ .Values.proverNode.service.p2pTcpPort }}"
+              {{- end }}
+            - name: P2P_UDP_ANNOUNCE_ADDR
+              {{- if .Values.proverNode.externalUdpHost }}
+              value: "{{ .Values.proverNode.externalUdpHost }}:{{ .Values.proverNode.service.p2pUdpPort }}"
+              {{- else }}
+              value: "$(POD_DNS_NAME):{{ .Values.proverNode.service.p2pUdpPort }}"
+              {{- end }}
+            - name: P2P_TCP_LISTEN_ADDR
+              value: "0.0.0.0:{{ .Values.proverNode.service.p2pTcpPort }}"
+            - name: P2P_UDP_LISTEN_ADDR
+              value: "0.0.0.0:{{ .Values.proverNode.service.p2pUdpPort }}"
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
+            - containerPort: {{ .Values.proverNode.service.p2pTcpPort }}
+            - containerPort: {{ .Values.proverNode.service.p2pUdpPort }}
+              protocol: UDP
           resources:
             {{- toYaml .Values.proverNode.resources | nindent 12 }}
       volumes:
@@ -140,12 +159,17 @@ spec:
   ports:
     - port: {{ .Values.proverNode.service.nodePort }}
       name: node
+    - port: {{ .Values.proverNode.service.p2pTcpPort }}
+      name: p2p-tcp
+    - port: {{ .Values.proverNode.service.p2pUdpPort }}
+      name: p2p-udp
+      protocol: UDP
 ---
 {{if .Values.network.public }}
 apiVersion: v1
 kind: Service
 metadata:
-  name: {{ include "aztec-network.fullname" . }}-prover-node-lb
+  name: {{ include "aztec-network.fullname" . }}-prover-node-lb-tcp
   labels:
     {{- include "aztec-network.labels" . | nindent 4 }}
 spec:
@@ -156,5 +180,27 @@ spec:
   ports:
     - port: {{ .Values.proverNode.service.nodePort }}
       name: node
+    - port: {{ .Values.proverNode.service.p2pTcpPort }}
+      name: p2p-tcp
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "aztec-network.fullname" . }}-prover-node-lb-udp
+  annotations:
+    service.beta.kubernetes.io/aws-load-balancer-type: "nlb"
+    service.beta.kubernetes.io/aws-load-balancer-nlb-target-type: "ip"
+    service.beta.kubernetes.io/aws-load-balancer-scheme: "internet-facing"
+  labels:
+    {{- include "aztec-network.labels" . | nindent 4 }}
+spec:
+  type: LoadBalancer
+  selector:
+    {{- include "aztec-network.selectorLabels" . | nindent 4 }}
+    app: prover-node
+  ports:
+    - port: {{ .Values.proverNode.service.p2pUdpPort }}
+      name: p2p-udp
+      protocol: UDP
 {{ end }}
 {{ end }}
diff --git a/spartan/aztec-network/templates/pxe.yaml b/spartan/aztec-network/templates/pxe.yaml
@@ -70,10 +70,8 @@ spec:
                 - -c
                 - |
                   curl -s -X POST -H 'content-type: application/json' \
-                  -d '{"jsonrpc":"2.0","method":"pxe_getNodeInfo","params":[],"id":67}' \
-                  127.0.0.1:{{ .Values.pxe.service.port }} > /tmp/probe_output.txt && \
-                  cat /tmp/probe_output.txt && \
-                  grep -q '"enr:-' /tmp/probe_output.txt
+                  -d '{"jsonrpc":"2.0","method":"pxe_isGlobalStateSynchronized","params":[],"id":67}' \
+                  127.0.0.1:{{ .Values.pxe.service.port }} | grep -q '"result":true'
             initialDelaySeconds: {{ .Values.pxe.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.pxe.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.pxe.readinessProbe.timeoutSeconds }}

diff --git a/spartan/aztec-network/templates/transaction-bot.yaml b/spartan/aztec-network/templates/transaction-bot.yaml
@@ -60,6 +60,10 @@ spec:
               value: "{{ .Values.bot.pxeProverEnabled }}"
             - name: PROVER_REAL_PROOFS
               value: "{{ .Values.bot.proverRealProofs }}"
+            - name: BOT_MAX_CONSECUTIVE_ERRORS
+              value: "3"
+            - name: BOT_STOP_WHEN_UNHEALTHY
+              value: "true"
           ports:
             - name: http
               containerPort: {{ .Values.bot.service.port }}

diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml
@@ -93,9 +93,13 @@ validator:
 
 proverNode:
   external: false
-  externalHost: ""
+  externalTcpHost: ""
+  externalUdpHost: ""
   replicas: 1
+  p2pEnabled: true
   service:
+    p2pTcpPort: 40400
+    p2pUdpPort: 40400
     nodePort: 8080
   logLevel: "debug"
   debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:world-state:database,-aztec:l2_block_stream*"
@@ -111,8 +115,7 @@ pxe:
   external: false
   externalHost: ""
   logLevel: "debug"
-  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*"
-  proverEnable: false
+  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database"
   replicas: 1
   service:
     port: 8080
@@ -132,12 +135,12 @@ pxe:
 bot:
   enabled: true
   logLevel: "debug"
-  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*"
+  debug: "aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:database"
   replicas: 1
   botPrivateKey: "0xcafe"
   txIntervalSeconds: 24
-  privateTransfersPerTx: 1
-  publicTransfersPerTx: 0
+  privateTransfersPerTx: 0
+  publicTransfersPerTx: 1
   # Do not wait for transactions
   followChain: "NONE"
   botNoStart: false

diff --git a/spartan/aztec-network/values/16-validators-with-metrics.yaml b/spartan/aztec-network/values/16-validators-with-metrics.yaml
@@ -0,0 +1,55 @@
+##########
+# BEWARE #
+##########
+# You need to deploy the metrics helm chart before using this values file.
+# head to spartan/metrics and run `./install.sh`
+# (then `./forward.sh` if you want to see it)
+telemetry:
+  enabled: true
+  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
+
+validator:
+  replicas: 16
+  validatorKeys:
+    - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+    - 0x59c6995e998f97a5a0044966f0945389dc9e86dae88c7a8412f4603b6b78690d
+    - 0x5de4111afa1a4b94908f83103eb1f1706367c2e68ca870fc3fb9a804cdab365a
+    - 0x7c852118294e51e653712a81e05800f419141751be58f605c371e15141b007a6
+    - 0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a
+    - 0x8b3a350cf5c34c9194ca85829a2df0ec3153be0318b5e2d3348e872092edffba
+    - 0x92db14e403b83dfe3df233f83dfa3a0d7096f21ca9b0d6d6b8d88b2b4ec1564e
+    - 0x4bbbf85ce3377467afe5d46f804f221813b2bb87f24d81f60f1fcdbf7cbf4356
+    - 0xdbda1821b80551c9d65939329250298aa3472ba22feea921c0cf5d620ea67b97
+    - 0x2a871d0798f97d79848a013d4936a73bf4cc922c825d33c1cf7073dff6d409c6
+    - 0xf214f2b2cd398c806f84e317254e0f0b801d0643303237d97a22a48e01628897
+    - 0x701b615bbdfb9de65240bc28bd21bbc0d996645a3dd57e7b12bc2bdf6f192c82
+    - 0xa267530f49f8280200edf313ee7af6b827f2a8bce2897751d06a843f644967b1
+    - 0x47c99abed3324a2707c28affff1267e45918ec8c3f20b8aa892e8b065d2942dd
+    - 0xc526ee95bf44d8fc405a158bb884d9d1238d99f0612e9f33d006bb0789009aaa
+    - 0x8166f546bab6da521a8369cab06c5d2b9e46670292d85c875ee9ec20e84ffb61
+  validatorAddresses:
+    - 0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266
+    - 0x70997970C51812dc3A010C7d01b50e0d17dc79C8
+    - 0x3C44CdDdB6a900fa2b585dd299e03d12FA4293BC
+    - 0x90F79bf6EB2c4f870365E785982E1f101E93b906
+    - 0x15d34AAf54267DB7D7c367839AAf71A00a2C6A65
+    - 0x9965507D1a55bcC2695C58ba16FB37d819B0A4dc
+    - 0x976EA74026E726554dB657fA54763abd0C3a0aa9
+    - 0x14dC79964da2C08b23698B3D3cc7Ca32193d9955
+    - 0x23618e81E3f5cdF7f54C3d65f7FBc0aBf5B21E8f
+    - 0xa0Ee7A142d267C1f36714E4a8F75612F20a79720
+    - 0xBcd4042DE499D14e55001CcbB24a551F3b954096
+    - 0x71bE63f3384f5fb98995898A86B02Fb2426c5788
+    - 0xFABB0ac9d68B0B445fB7357272Ff202C5651694a
+    - 0x1CBd3b2770909D4e10f157cABC84C7264073C9Ec
+    - 0xdF3e18d64BC6A983f673Ab319CCaE4f1a57C7097
+    - 0xcd3B766CCDd6AE721141F452C550Ca635964ce71
+  resources:
+    requests:
+      memory: "512Mi"
+  validator:
+    disabled: false
+
+bootNode:
+  validator:
+    disabled: true
diff --git a/spartan/aztec-network/values/16-validators.yaml b/spartan/aztec-network/values/16-validators.yaml
@@ -1,15 +1,4 @@
-##########
-# BEWARE #
-##########
-# You need to deploy the metrics helm chart before using this values file.
-# head to spartan/metrics and run `./install.sh`
-# (then `./forward.sh` if you want to see it)
-telemetry:
-  enabled: true
-  otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
-
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 16
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -51,11 +40,6 @@ validator:
   validator:
     disabled: false
 
-bot:
-  txIntervalSeconds: 2
-  privateTransfersPerTx: 1
-  publicTransfersPerTx: 2
-
 bootNode:
   validator:
     disabled: true
diff --git a/spartan/aztec-network/values/3-validators-with-metrics.yaml b/spartan/aztec-network/values/3-validators-with-metrics.yaml
@@ -9,7 +9,6 @@ telemetry:
   otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318
 
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:world-state:database,discv5:*,-JsonProxy:*"
   replicas: 3
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -23,6 +22,5 @@ validator:
     disabled: false
 
 bootNode:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:world-state:database,discv5:*,-JsonProxy:*"
   validator:
     disabled: true
diff --git a/spartan/aztec-network/values/3-validators.yaml b/spartan/aztec-network/values/3-validators.yaml
@@ -1,5 +1,4 @@
 validator:
-  debug: "aztec:*,-aztec:avm_simulator:*,-aztec:libp2p_service"
   replicas: 3
   validatorKeys:
     - 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
@@ -12,7 +11,6 @@ validator:
   validator:
     disabled: false
 
-
 bootNode:
   validator:
     disabled: true
diff --git a/spartan/chaos-mesh/install.sh b/spartan/chaos-mesh/install.sh
@@ -1,7 +1,15 @@
 #!/bin/bash
 
-# Install chaos-mesh
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+cd "$SCRIPT_DIR"
+
+# check if chaos-mesh is already installed
+if helm ls --namespace chaos-mesh | grep -q chaos; then
+  echo "chaos-mesh is already installed"
+  exit 0
+fi
+
 helm repo add chaos-mesh https://charts.chaos-mesh.org
 helm dependency update
+helm upgrade chaos "$SCRIPT_DIR" -n chaos-mesh --install --create-namespace --atomic
 
-helm upgrade chaos . -n chaos-mesh --install --create-namespace --atomic
diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh
@@ -1,10 +1,17 @@
 #!/bin/bash
 set -eu
 
-cd "$(dirname "${BASH_SOURCE[0]}")"
+SCRIPT_DIR="$(dirname "${BASH_SOURCE[0]}")"
+cd "$SCRIPT_DIR"
+
+# check if metrics is already installed
+if helm ls --namespace metrics | grep -q metrics; then
+  echo "metrics is already installed"
+  exit 0
+fi
 
 helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm dependency update
-helm upgrade metrics . -n metrics --install --create-namespace --atomic
+helm upgrade metrics "$SCRIPT_DIR" -n metrics --install --create-namespace --atomic
diff --git a/spartan/network-shaping/scripts/apply_network_shaping.sh b/spartan/network-shaping/scripts/apply_network_shaping.sh