From e08989f409a5460f419edacb162ebec7f5fa5ff0 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 17:41:39 +0000
Subject: [PATCH 01/39] x

---
 build-system/s3-cache-scripts/earthly-s3-cache.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build-system/s3-cache-scripts/earthly-s3-cache.sh b/build-system/s3-cache-scripts/earthly-s3-cache.sh
index 571eb210450..cb45ee4a1fe 100755
--- a/build-system/s3-cache-scripts/earthly-s3-cache.sh
+++ b/build-system/s3-cache-scripts/earthly-s3-cache.sh
@@ -5,7 +5,7 @@
 # - $prefix
 # - $command
 # The rest of the env variables are injected as secrets (e.g. aws creds and s3 modes)
-set -eu
+set -eux
 
 # definitions
 FILE="$prefix-$(cat .content-hash).tar.gz"

From 03ff27b661730ae7a04e9eb030b568de71918356 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 17:43:49 +0000
Subject: [PATCH 02/39] x

---
 build-system/s3-cache-scripts/earthly-s3-cache.sh | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/build-system/s3-cache-scripts/earthly-s3-cache.sh b/build-system/s3-cache-scripts/earthly-s3-cache.sh
index cb45ee4a1fe..e946b489b5c 100755
--- a/build-system/s3-cache-scripts/earthly-s3-cache.sh
+++ b/build-system/s3-cache-scripts/earthly-s3-cache.sh
@@ -19,7 +19,7 @@ function s3_upload() {
   if [ "${S3_BUILD_CACHE_UPLOAD:-true}" = "false" ] || [ "${AWS_ACCESS_KEY_ID}" == "" ] ; then
     return 0 # exit silently
   fi
-  /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts
+  /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts || echo "WARNING: S3 upload failed!" >&2
 }
 function minio_download() {
   if [ -z "$S3_BUILD_CACHE_MINIO_URL" ] ; then
@@ -35,7 +35,7 @@ function minio_upload() {
   fi
   # minio is S3-compatible
   S3_BUILD_CACHE_AWS_PARAMS="--endpoint-url $S3_BUILD_CACHE_MINIO_URL" AWS_SECRET_ACCESS_KEY=minioadmin AWS_ACCESS_KEY_ID=minioadmin \
-    /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts
+    /usr/src/build-system/s3-cache-scripts/cache-upload.sh "$FILE" $build_artifacts || echo "WARNING Minio upload failed!" >&2
 }
 
 # commands
@@ -53,5 +53,5 @@ if ! bash -c "$command" ; then
   exit 1 # we have failed to build, don't continue
 fi
 
-minio_upload || echo "Minio upload failed!"
-s3_upload || echo "S3 upload failed!"
+minio_upload
+s3_upload

From fc40b0dc55a4f08acb66b9b84ab36c2dcf1a8059 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 19:12:53 +0000
Subject: [PATCH 03/39] small ci fix; get load balancer url helper function ;
 allow otel injection in scripts

---
 .github/ensure-tester/action.yml                    |  3 +--
 spartan/scripts/test_spartan.sh                     | 13 ++++++++++---
 .../end-to-end/scripts/native-network/boot-node.sh  |  4 ++--
 .../end-to-end/scripts/native-network/validator.sh  |  2 ++
 4 files changed, 15 insertions(+), 7 deletions(-)

diff --git a/.github/ensure-tester/action.yml b/.github/ensure-tester/action.yml
index ca85b7da853..71f29151406 100644
--- a/.github/ensure-tester/action.yml
+++ b/.github/ensure-tester/action.yml
@@ -8,7 +8,6 @@ inputs:
   run:
     # command to run once tester available
     required: false
-
   ttl:
     required: false
     description: "Time to live for the tester instance in minutes"
@@ -31,7 +30,7 @@ runs:
         echo "ttl=${{ inputs.ttl }}" >> $GITHUB_OUTPUT
         SIZE=large
         if [[ $TYPE == 4core-* ]]; then
-          SIZE=large
+          SIZE=xlarge
         elif [[ $TYPE == 8core-* ]]; then
           SIZE=2xlarge
         elif [[ $TYPE == 16core-* ]]; then
diff --git a/spartan/scripts/test_spartan.sh b/spartan/scripts/test_spartan.sh
index f6b3bbe2e9a..f362a6dbcbf 100755
--- a/spartan/scripts/test_spartan.sh
+++ b/spartan/scripts/test_spartan.sh
@@ -16,10 +16,17 @@ fi
 
 echo "Note: Repo should be bootstrapped with ./bootstrap.sh fast."
 
+# Helper function to get load balancer URL based on namespace and service name
+function get_load_balancer_url() {
+  local namespace=$1
+  local service_name=$2
+  kubectl get svc -n $namespace -o jsonpath="{.items[?(@.metadata.name=='$service_name')].status.loadBalancer.ingress[0].hostname}"
+}
+
 # Fetch the service URLs based on the namespace for injection in the test-transfer.sh
-export BOOTNODE_URL=http://$(kubectl get svc -n $NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='$NAMESPACE-aztec-network-boot-node-lb-tcp')].status.loadBalancer.ingress[0].hostname}"):8080
-export PXE_URL=http://$(kubectl get svc -n $NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='$NAMESPACE-aztec-network-pxe-lb')].status.loadBalancer.ingress[0].hostname}"):8080
-export ETHEREUM_HOST=http://$(kubectl get svc -n $NAMESPACE -o jsonpath="{.items[?(@.metadata.name=='$NAMESPACE-aztec-network-ethereum-lb')].status.loadBalancer.ingress[0].hostname}"):8545
+export BOOTNODE_URL=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-boot-node-lb-tcp"):8080
+export PXE_URL=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-pxe-lb"):8080
+export ETHEREUM_HOST=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-ethereum-lb"):8545
 
 echo "BOOTNODE_URL: $BOOTNODE_URL"
 echo "PXE_URL: $PXE_URL"
diff --git a/yarn-project/end-to-end/scripts/native-network/boot-node.sh b/yarn-project/end-to-end/scripts/native-network/boot-node.sh
index 5ee8e8e0f98..9d8b4ed3b69 100755
--- a/yarn-project/end-to-end/scripts/native-network/boot-node.sh
+++ b/yarn-project/end-to-end/scripts/native-network/boot-node.sh
@@ -22,8 +22,8 @@ export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:40400"
 export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:40400"
 export P2P_TCP_LISTEN_ADDR="0.0.0.0:40400"
 export P2P_UDP_LISTEN_ADDR="0.0.0.0:40400"
-export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=""
-export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=""
+export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
 export VALIDATOR_PRIVATE_KEY="0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a"
 REPO=$(git rev-parse --show-toplevel)
 
diff --git a/yarn-project/end-to-end/scripts/native-network/validator.sh b/yarn-project/end-to-end/scripts/native-network/validator.sh
index 67750247ebd..f7952af763a 100755
--- a/yarn-project/end-to-end/scripts/native-network/validator.sh
+++ b/yarn-project/end-to-end/scripts/native-network/validator.sh
@@ -54,6 +54,8 @@ export P2P_TCP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT"
 export P2P_UDP_ANNOUNCE_ADDR="127.0.0.1:$P2P_PORT"
 export P2P_TCP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
 export P2P_UDP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
+export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
 
 # Add L1 validator
 # this may fail, so try 3 times

From 380713d46dfe4a3c3b0fdc4890d5575fca914c17 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 20:27:10 +0000
Subject: [PATCH 04/39] .

---
 yarn-project/package.json                                 | 2 +-
 .../protocol-contracts/src/protocol_contract_data.ts      | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/yarn-project/package.json b/yarn-project/package.json
index 747d2467f82..65f1af37513 100644
--- a/yarn-project/package.json
+++ b/yarn-project/package.json
@@ -12,7 +12,7 @@
     "format": "yarn prettier --cache -w .",
     "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/prover-client -p -v run test && yarn workspaces foreach --include @aztec/end-to-end -p -v run test:unit",
     "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build",
-    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd .. && yarn generate && tsc -b",
+    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-contracts && yarn generate && cd .. && yarn generate && tsc -b",
     "build:dev": "./watch.sh",
     "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate",
     "clean": "yarn workspaces foreach -p -v run clean"
diff --git a/yarn-project/protocol-contracts/src/protocol_contract_data.ts b/yarn-project/protocol-contracts/src/protocol_contract_data.ts
index c5e4b30fb85..11810f96b37 100644
--- a/yarn-project/protocol-contracts/src/protocol_contract_data.ts
+++ b/yarn-project/protocol-contracts/src/protocol_contract_data.ts
@@ -50,14 +50,14 @@ export const ProtocolContractAddress: Record<ProtocolContractName, AztecAddress>
 };
 
 export const ProtocolContractLeaf = {
-  AuthRegistry: Fr.fromString('0x0f482caa8a420dd90facb7763f742e2cb456e89d0a85eb55b840569b61ed2083'),
+  AuthRegistry: Fr.fromString('0x305c48d9d087f24b1618a7cc92c5081db4672f6855816af7ac89ea7e873245cd'),
   ContractInstanceDeployer: Fr.fromString('0x04a661c9d4d295fc485a7e0f3de40c09b35366343bce8ad229106a8ef4076fe5'),
   ContractClassRegisterer: Fr.fromString('0x147ba3294403576dbad10f86d3ffd4eb83fb230ffbcd5c8b153dd02942d0611f'),
   MultiCallEntrypoint: Fr.fromString('0x154b701b41d6cf6da7204fef36b2ee9578b449d21b3792a9287bf45eba48fd26'),
-  FeeJuice: Fr.fromString('0x02f0c22c8c8f8bcc9707383fbb2ca57278c6554af0b3e2a7ca046cd6f956fb62'),
-  Router: Fr.fromString('0x299546b7c3804dfc5842d3713c7f238f7075af008e4828405f9a1eb4897ac9c2'),
+  FeeJuice: Fr.fromString('0x21c9ab2e339c9b3394e4e1ff3b7cf37be4e0fc0bc177a192d287d98963b9b254'),
+  Router: Fr.fromString('0x2779d7e4ccba389da358a0c9362364d0c65e14cd4d9df929e6722187e808e068'),
 };
 
 export const protocolContractTreeRoot = Fr.fromString(
-  '0x120eb84242cae2f2eb6bdbe86ddd2787a2b9e043da0b80f4a5b7f504e3d03ffb',
+  '0x00724e4de088411c873c3d6975491eb48889bfa51bc854744a4fcc307ee9abd8',
 );

From 11b41b7bd9d28057bb569fe49a23380f03289928 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 20:36:11 +0000
Subject: [PATCH 05/39] .

---
 yarn-project/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/package.json b/yarn-project/package.json
index 65f1af37513..a0f0c56a374 100644
--- a/yarn-project/package.json
+++ b/yarn-project/package.json
@@ -12,7 +12,7 @@
     "format": "yarn prettier --cache -w .",
     "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/prover-client -p -v run test && yarn workspaces foreach --include @aztec/end-to-end -p -v run test:unit",
     "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build",
-    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-contracts && yarn generate && cd .. && yarn generate && tsc -b",
+    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-artifacts && yarn generate && cd .. && yarn generate && tsc -b",
     "build:dev": "./watch.sh",
     "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate",
     "clean": "yarn workspaces foreach -p -v run clean"

From c6ddaf36985121c0847b6de7562b1e0911fa9dc1 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 20:57:40 +0000
Subject: [PATCH 06/39] .

---
 yarn-project/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/package.json b/yarn-project/package.json
index a0f0c56a374..ad6d95ba38a 100644
--- a/yarn-project/package.json
+++ b/yarn-project/package.json
@@ -12,7 +12,7 @@
     "format": "yarn prettier --cache -w .",
     "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/prover-client -p -v run test && yarn workspaces foreach --include @aztec/end-to-end -p -v run test:unit",
     "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build",
-    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-artifacts && yarn generate && cd .. && yarn generate && tsc -b",
+    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-artifacts && yarn generate && yarn build && cd .. && yarn generate && tsc -b",
     "build:dev": "./watch.sh",
     "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate",
     "clean": "yarn workspaces foreach -p -v run clean"

From 92d1e4bc04617079ad68eb298195916f2f300fe2 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Fri, 25 Oct 2024 21:08:33 +0000
Subject: [PATCH 07/39] -

---
 yarn-project/package.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/package.json b/yarn-project/package.json
index ad6d95ba38a..531a20ddadd 100644
--- a/yarn-project/package.json
+++ b/yarn-project/package.json
@@ -12,7 +12,7 @@
     "format": "yarn prettier --cache -w .",
     "test": "FORCE_COLOR=true yarn workspaces foreach --exclude @aztec/aztec3-packages --exclude @aztec/end-to-end --exclude @aztec/prover-client -p -v run test && yarn workspaces foreach --include @aztec/end-to-end -p -v run test:unit",
     "build": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose --exclude @aztec/aztec3-packages --exclude @aztec/docs run build",
-    "build:fast": "cd foundation && yarn build && cd ../circuits.js && yarn build && cd ../l1-artifacts && yarn generate && yarn build && cd .. && yarn generate && tsc -b",
+    "build:fast": "cd ../l1-artifacts && yarn generate && yarn build && cd foundation && yarn build && cd ../circuits.js && yarn build && cd .. && yarn generate && tsc -b",
     "build:dev": "./watch.sh",
     "generate": "FORCE_COLOR=true yarn workspaces foreach --parallel --topological-dev --verbose run generate",
     "clean": "yarn workspaces foreach -p -v run clean"

From 1ebfe9b8f1b90ee7e5c271f5747263d8a00aec3b Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 13:58:37 +0000
Subject: [PATCH 08/39] lead

---
 docker-compose.yml | 132 ++-------------------------------------------
 1 file changed, 5 insertions(+), 127 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 18ca3c48335..9faae02a3ad 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,118 +1,5 @@
-name: aztec-devnet
+name: otel
 services:
-  pxe:
-    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
-    # need to run bb for proofs and bb is only built for x86
-    platform: linux/amd64
-    environment:
-      LOG_LEVEL: ${LOG_LEVEL:-info}
-      DEBUG: ${DEBUG:-aztec:*,-json-rpc:*,-aztec:circuits:artifact_hash,-aztec:randomness_singleton}
-      DEBUG_COLORS: 1
-      L1_CHAIN_ID: 31337
-      VERSION: 1
-      PXE_PROVER_ENABLED: ${PXE_PROVER_ENABLED:-1}
-      PXE_DATA_DIRECTORY: /var/lib/aztec/pxe
-      NODE_NO_WARNINGS: 1
-      AZTEC_PORT: 8080
-      MNEMONIC: "test test test test test test test test test test test junk"
-    entrypoint: >
-      sh -c '
-        export AZTEC_NODE_URL=$$(cat /var/run/secrets/aztec-node-url);
-        while ! curl --head --silent $AZTEC_NODE_URL > /dev/null; do echo "Node $$AZTEC_NODE_URL not up. Retrying after 1s"; sleep 1; done;
-        node /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe
-      '
-    secrets:
-      - aztec-node-url
-    extra_hosts:
-      - "host.docker.internal:host-gateway"
-    volumes:
-      - aztec:/var/lib/aztec
-    ports:
-      - 8080:8080/tcp
-    profiles:
-      - pxe
-
-  node:
-    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
-    # need to run bb for proofs and bb is only built for x86
-    platform: linux/amd64
-    environment:
-      LOG_LEVEL: ${LOG_LEVEL:-info}
-      DEBUG: ${DEBUG:-aztec:*,-json-rpc:*,-aztec:circuits:artifact_hash,-aztec:randomness_singleton,-aztec:avm_simulator:*}
-      DEBUG_COLORS: 1
-      L1_CHAIN_ID: 31337
-      VERSION: 1
-      NODE_NO_WARNINGS: 1
-      PROVER_REAL_PROOFS: ${PROVER_REAL_PROOFS:-1}
-      DATA_DIRECTORY: /var/lib/aztec
-      PROVER_AGENT_ENABLED: 0
-      ROLLUP_CONTRACT_ADDRESS: "0x4ef7ca89e724a75ccca5dde7fb98075a388c2c91"
-      REGISTRY_CONTRACT_ADDRESS: "0x589a9634c1d00c62e47b3b7a790c8dc986b3d40d"
-      INBOX_CONTRACT_ADDRESS: "0x12d9b5effc69bf5c0c29c8258c6b6fa95a08de74"
-      OUTBOX_CONTRACT_ADDRESS: "0x3ec4b6c68a8c2ce4c78cdd465b3019b11a568d1d"
-      FEE_JUICE_CONTRACT_ADDRESS: "0x73c43b919973711e096bfc04c9d4b3be511ffc0b"
-      FEE_JUICE_PORTAL_CONTRACT_ADDRESS: "0xdf25b0a34dbee9f25518f7a4d63bab8b3bb3e496"
-      ETHEREUM_HOST:
-      P2P_TCP_LISTEN_ADDR: "0.0.0.0:9000"
-      P2P_UDP_LISTEN_ADDR: "0.0.0.0:9001"
-      P2P_TCP_ANNOUNCE_ADDR: ":9000"
-      P2P_UDP_ANNOUNCE_ADDR: ":9001"
-      P2P_QUERY_FOR_IP: true
-      P2P_ENABLED: true
-      PEER_ID_PRIVATE_KEY:
-      AZTEC_PORT: 8999
-      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics}
-      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces}
-    secrets:
-      - ethereum-host
-      - p2p-boot-node
-    entrypoint: |
-      /bin/sh -c '
-        export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host)
-        export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node)
-
-        test -z "$$PEER_ID_PRIVATE_KEY" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d" " -f 3 | tee /var/lib/aztec/p2p-private-key || echo "Re-using existing P2P private key"
-        test -z "$$PEER_ID_PRIVATE_KEY" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key)
-
-        # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node
-        i=0
-        max=3
-        while ! curl --head --silent $$OTEL_EXPORTER_OTLP_METRICS_ENDPOINT > /dev/null; do
-          echo "OpenTelemetry collector not up. Retrying after 1s";
-          sleep 1;
-          i=$$((i+1));
-          if [ $$i -eq $$max ]; then
-            echo "OpenTelemetry collector at $$OTEL_EXPORTER_METRICS_ENDPOINT not up after $${max}s. Running without metrics";
-            unset OTEL_EXPORTER_METRICS_ENDPOINT;
-            unset OTEL_EXPORTER_TRACES_ENDPOINT;
-            break
-          fi;
-        done;
-
-        node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver
-      '
-    volumes:
-      - aztec:/var/lib/aztec
-    profiles:
-      - node
-    ports:
-      # the RPC endpoint - expose it on a different port to avoid conflicting with PXE
-      - 8999:8999/tcp
-      - 9000:9000/tcp
-      - 9001:9001/udp
-
-  cli:
-    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
-    # run the same image as pxe
-    platform: linux/amd64
-    environment:
-      PXE_URL: http://pxe:8080
-      NODE_NO_WARNINGS: 1
-      SECRET_KEY:
-      ETHEREUM_HOST:
-    profiles:
-      - cli
-
   otel-collector:
     image: otel/opentelemetry-collector-contrib
     configs:
@@ -121,12 +8,15 @@ services:
     profiles:
       - metrics
     ports:
+      - 4317:4317
       - 4318:4318
 
   prometheus:
     image: prom/prometheus
     profiles:
       - metrics
+    ports:
+      - 2112:2112
     configs:
       - source: prometheus-config
         target: /etc/prometheus/prometheus.yml
@@ -151,10 +41,6 @@ services:
     profiles:
       - metrics
 
-volumes:
-  aztec:
-  grafana:
-
 configs:
   grafana-sources:
     content: |
@@ -210,12 +96,4 @@ configs:
           metrics:
             receivers: [otlp]
             processors: [batch]
-            exporters: [prometheus]
-
-secrets:
-  aztec-node-url:
-    environment: AZTEC_NODE_URL
-  ethereum-host:
-    environment: ETHEREUM_HOST
-  p2p-boot-node:
-    environment: BOOTSTRAP_NODES
+            exporters: [prometheus]
\ No newline at end of file

From 0738e890d287a8ba5d21a557977bab68b9c4f0a6 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 19:47:57 +0000
Subject: [PATCH 09/39] deploy_spartan.sh prod changes

---
 .../s3-cache-scripts/earthly-s3-cache.sh      |   2 +-
 spartan/aztec-network/templates/_helpers.tpl  |   7 ++
 .../aztec-network/templates/boot-node.yaml    |   2 +
 .../aztec-network/templates/validator.yaml    |   2 +
 spartan/metrics/install.sh                    |   4 +-
 spartan/metrics/values.yaml                   |   2 +-
 spartan/scripts/deploy_spartan.sh             |  13 +-
 .../scripts/native-network/boot-node.sh       |   1 +
 yarn-project/foundation/src/config/env_var.ts |   1 +
 yarn-project/telemetry-client/package.json    |   1 +
 yarn-project/telemetry-client/src/config.ts   |   9 +-
 yarn-project/yarn.lock                        | 118 +++++++++++++++++-
 12 files changed, 149 insertions(+), 13 deletions(-)

diff --git a/build-system/s3-cache-scripts/earthly-s3-cache.sh b/build-system/s3-cache-scripts/earthly-s3-cache.sh
index e946b489b5c..01267639be0 100755
--- a/build-system/s3-cache-scripts/earthly-s3-cache.sh
+++ b/build-system/s3-cache-scripts/earthly-s3-cache.sh
@@ -5,7 +5,7 @@
 # - $prefix
 # - $command
 # The rest of the env variables are injected as secrets (e.g. aws creds and s3 modes)
-set -eux
+set -eu
 
 # definitions
 FILE="$prefix-$(cat .content-hash).tar.gz"
diff --git a/spartan/aztec-network/templates/_helpers.tpl b/spartan/aztec-network/templates/_helpers.tpl
index 2fcc4b7b974..f9be2d9ecaa 100644
--- a/spartan/aztec-network/templates/_helpers.tpl
+++ b/spartan/aztec-network/templates/_helpers.tpl
@@ -102,6 +102,13 @@ http://{{ include "aztec-network.fullname" . }}-metrics.{{ .Release.Namespace }}
 {{- end -}}
 {{- end -}}
 
+{{- define "aztec-network.otelCollectorLogsEndpoint" -}}
+{{- if .Values.telemetry.enabled -}}
+{{- if .Values.telemetry.otelCollectorEndpoint -}}
+{{- .Values.telemetry.otelCollectorEndpoint -}}/v1/logs
+{{- end -}}
+{{- end -}}
+{{- end -}}
 
 {{- define "helpers.flag" -}}
 {{- $name := index . 0 -}}
diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 3467935296b..6195582fa14 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -141,6 +141,8 @@ spec:
               value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
             - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
               value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
+            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
+              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
           ports:
             - containerPort: {{ .Values.bootNode.service.nodePort }}
             - containerPort: {{ .Values.bootNode.service.p2pTcpPort }}
diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index acd2eb9b937..6c264c28075 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -130,6 +130,8 @@ spec:
               value: {{ include "aztec-network.otelCollectorMetricsEndpoint" . | quote }}
             - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
               value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
+            - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
+              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
           ports:
             - containerPort: {{ .Values.validator.service.nodePort }}
             - containerPort: {{ .Values.validator.service.p2pTcpPort }}
diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh
index fda4203601f..1f163218699 100755
--- a/spartan/metrics/install.sh
+++ b/spartan/metrics/install.sh
@@ -1,8 +1,10 @@
 #!/bin/bash
 set -eu
 
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
 helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm-charts
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm dependency update
-helm upgrade metrics . -n metrics --install --create-namespace --atomic
+helm template metrics . -n metrics  #--install --create-namespace --atomic
diff --git a/spartan/metrics/values.yaml b/spartan/metrics/values.yaml
index bc969d3c1a6..9b1d9139273 100644
--- a/spartan/metrics/values.yaml
+++ b/spartan/metrics/values.yaml
@@ -1,5 +1,5 @@
 opentelemetry-collector:
-  mode: daemonset
+  mode: deployment
 
   service:
     enabled: true
diff --git a/spartan/scripts/deploy_spartan.sh b/spartan/scripts/deploy_spartan.sh
index 85d7b12e1a8..5b00baba860 100755
--- a/spartan/scripts/deploy_spartan.sh
+++ b/spartan/scripts/deploy_spartan.sh
@@ -5,6 +5,11 @@ set -o pipefail
 TAG=$1
 VALUES=$2
 NAMESPACE=${3:-spartan}
+PROD=${4:-true}
+PROD_ARGS=""
+if [ "$PROD" = "true" ] ; then
+  PROD_ARGS="--set network.public=true --set telemetry.enabled=true --set telemetry.otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318"
+fi
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 
 if [ -z "$TAG" ]; then
@@ -46,16 +51,14 @@ function upgrade() {
     helm template $NAMESPACE $SCRIPT_DIR/../aztec-network \
           --namespace $NAMESPACE \
           --create-namespace \
-          --values $SCRIPT_DIR/../aztec-network/values/$VALUES.yaml \
-          --set images.aztec.image="$IMAGE" \
-          --set network.public=true
+          --values $SCRIPT_DIR/../aztec-network/values/$VALUES.yaml $PROD_ARGS \
+          --set images.aztec.image="$IMAGE"
   else
     helm upgrade --install $NAMESPACE $SCRIPT_DIR/../aztec-network \
           --namespace $NAMESPACE \
           --create-namespace \
-          --values $SCRIPT_DIR/../aztec-network/values/$VALUES.yaml \
+          --values $SCRIPT_DIR/../aztec-network/values/$VALUES.yaml $PROD_ARGS \
           --set images.aztec.image="$IMAGE" \
-          --set network.public=true \
           --wait \
           --wait-for-jobs=true \
           --timeout=30m 2>&1
diff --git a/yarn-project/end-to-end/scripts/native-network/boot-node.sh b/yarn-project/end-to-end/scripts/native-network/boot-node.sh
index 9d8b4ed3b69..a266fe458d2 100755
--- a/yarn-project/end-to-end/scripts/native-network/boot-node.sh
+++ b/yarn-project/end-to-end/scripts/native-network/boot-node.sh
@@ -24,6 +24,7 @@ export P2P_TCP_LISTEN_ADDR="0.0.0.0:40400"
 export P2P_UDP_LISTEN_ADDR="0.0.0.0:40400"
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-}"
 export VALIDATOR_PRIVATE_KEY="0x47e179ec197488593b187f80a00eb0da91f1b9d0b13f8733639f19c30a34926a"
 REPO=$(git rev-parse --show-toplevel)
 
diff --git a/yarn-project/foundation/src/config/env_var.ts b/yarn-project/foundation/src/config/env_var.ts
index 06a7745487b..67b703b5c9a 100644
--- a/yarn-project/foundation/src/config/env_var.ts
+++ b/yarn-project/foundation/src/config/env_var.ts
@@ -58,6 +58,7 @@ export type EnvVar =
   | 'NOMISMATOKOPIO_CONTRACT_ADDRESS'
   | 'OTEL_EXPORTER_OTLP_METRICS_ENDPOINT'
   | 'OTEL_EXPORTER_OTLP_TRACES_ENDPOINT'
+  | 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT'
   | 'OTEL_SERVICE_NAME'
   | 'OUTBOX_CONTRACT_ADDRESS'
   | 'P2P_BLOCK_CHECK_INTERVAL_MS'
diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json
index ecbc5444c34..0c854eea28a 100644
--- a/yarn-project/telemetry-client/package.json
+++ b/yarn-project/telemetry-client/package.json
@@ -31,6 +31,7 @@
     "@opentelemetry/exporter-metrics-otlp-http": "^0.52.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.52.0",
     "@opentelemetry/host-metrics": "^0.35.2",
+    "@opentelemetry/instrumentation-winston": "^0.41.0",
     "@opentelemetry/resource-detector-aws": "^1.5.2",
     "@opentelemetry/resources": "^1.25.0",
     "@opentelemetry/sdk-metrics": "^1.25.0",
diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts
index c7789ba05bc..1745b5a4e53 100644
--- a/yarn-project/telemetry-client/src/config.ts
+++ b/yarn-project/telemetry-client/src/config.ts
@@ -1,8 +1,10 @@
 import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
 
+
 export interface TelemetryClientConfig {
   metricsCollectorUrl?: URL;
   tracesCollectorUrl?: URL;
+  logsCollectorUrl?: URL;
   serviceName: string;
   networkName: string;
 }
@@ -18,6 +20,11 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
     description: 'The URL of the telemetry collector for traces',
     parseEnv: (val: string) => new URL(val),
   },
+  logsCollectorUrl: {
+    env: 'OTEL_EXPORTER_OTLP_LOGS_ENDPOINT',
+    description: 'The URL of the telemetry collector for logs',
+    parseEnv: (val: string) => new URL(val),
+  },
   serviceName: {
     env: 'OTEL_SERVICE_NAME',
     description: 'The URL of the telemetry collector',
@@ -32,4 +39,4 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
 
 export function getConfigEnvVars(): TelemetryClientConfig {
   return getConfigFromMappings<TelemetryClientConfig>(telemetryClientConfigMappings);
-}
+}
\ No newline at end of file
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index 3a409f0ab9c..847fa3fdd49 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -1151,6 +1151,7 @@ __metadata:
     "@opentelemetry/exporter-metrics-otlp-http": ^0.52.0
     "@opentelemetry/exporter-trace-otlp-http": ^0.52.0
     "@opentelemetry/host-metrics": ^0.35.2
+    "@opentelemetry/instrumentation-winston": ^0.41.0
     "@opentelemetry/resource-detector-aws": ^1.5.2
     "@opentelemetry/resources": ^1.25.0
     "@opentelemetry/sdk-metrics": ^1.25.0
@@ -3442,7 +3443,16 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.9.0":
+"@opentelemetry/api-logs@npm:0.54.0, @opentelemetry/api-logs@npm:^0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/api-logs@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: 5fc91054a290663844049cd9eb66419ea06d191b82220f2513147acdbd82579d1d3703a7e09f58a0014118d52b96d8b6340f9b43dd33a2c4469a31f13b3abc62
+  languageName: node
+  linkType: hard
+
+"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.3.0, @opentelemetry/api@npm:^1.9.0":
   version: 1.9.0
   resolution: "@opentelemetry/api@npm:1.9.0"
   checksum: 9e88e59d53ced668f3daaecfd721071c5b85a67dd386f1c6f051d1be54375d850016c881f656ffbe9a03bedae85f7e89c2f2b635313f9c9b195ad033cdc31020
@@ -3522,6 +3532,34 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/instrumentation-winston@npm:^0.41.0":
+  version: 0.41.0
+  resolution: "@opentelemetry/instrumentation-winston@npm:0.41.0"
+  dependencies:
+    "@opentelemetry/api-logs": ^0.54.0
+    "@opentelemetry/instrumentation": ^0.54.0
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: 10d2769925ae9a9982c8da9bb81c6e121f9c1c3437ac99216eb8c68fddea8a3527b5f9acfb09275646e3bc05183bf5bd2b719576756fa40ea3b98a6950a7533f
+  languageName: node
+  linkType: hard
+
+"@opentelemetry/instrumentation@npm:^0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/instrumentation@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/api-logs": 0.54.0
+    "@types/shimmer": ^1.2.0
+    import-in-the-middle: ^1.8.1
+    require-in-the-middle: ^7.1.1
+    semver: ^7.5.2
+    shimmer: ^1.2.1
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: b1fea61c2debea78573a728a3c9065eb6f502e98e6bbc61e98b77492734e86a6fbb79d27694f190bc192429f62a7f97a5025c0bcdff6f30c1c91c9f0cc6d931d
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/otlp-exporter-base@npm:0.52.0":
   version: 0.52.0
   resolution: "@opentelemetry/otlp-exporter-base@npm:0.52.0"
@@ -4748,6 +4786,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@types/shimmer@npm:^1.2.0":
+  version: 1.2.0
+  resolution: "@types/shimmer@npm:1.2.0"
+  checksum: f081a31d826ce7bfe8cc7ba8129d2b1dffae44fd580eba4fcf741237646c4c2494ae6de2cada4b7713d138f35f4bc512dbf01311d813dee82020f97d7d8c491c
+  languageName: node
+  linkType: hard
+
 "@types/sinon@npm:^10.0.15":
   version: 10.0.20
   resolution: "@types/sinon@npm:10.0.20"
@@ -5357,6 +5402,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"acorn-import-attributes@npm:^1.9.5":
+  version: 1.9.5
+  resolution: "acorn-import-attributes@npm:1.9.5"
+  peerDependencies:
+    acorn: ^8
+  checksum: 1c0c49b6a244503964ae46ae850baccf306e84caf99bc2010ed6103c69a423987b07b520a6c619f075d215388bd4923eccac995886a54309eda049ab78a4be95
+  languageName: node
+  linkType: hard
+
 "acorn-jsx@npm:^5.3.2":
   version: 5.3.2
   resolution: "acorn-jsx@npm:5.3.2"
@@ -6524,6 +6578,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"cjs-module-lexer@npm:^1.2.2":
+  version: 1.4.1
+  resolution: "cjs-module-lexer@npm:1.4.1"
+  checksum: 2556807a99aec1f9daac60741af96cd613a707f343174ae7967da46402c91dced411bf830d209f2e93be4cecea46fc75cecf1f17c799d7d8a9e1dd6204bfcd22
+  languageName: node
+  linkType: hard
+
 "clean-stack@npm:^2.0.0":
   version: 2.2.0
   resolution: "clean-stack@npm:2.2.0"
@@ -7375,6 +7436,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"debug@npm:^4.3.5":
+  version: 4.3.7
+  resolution: "debug@npm:4.3.7"
+  dependencies:
+    ms: ^2.1.3
+  peerDependenciesMeta:
+    supports-color:
+      optional: true
+  checksum: 822d74e209cd910ef0802d261b150314bbcf36c582ccdbb3e70f0894823c17e49a50d3e66d96b633524263975ca16b6a833f3e3b7e030c157169a5fabac63160
+  languageName: node
+  linkType: hard
+
 "decamelize-keys@npm:^1.1.0":
   version: 1.1.1
   resolution: "decamelize-keys@npm:1.1.1"
@@ -9747,6 +9820,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"import-in-the-middle@npm:^1.8.1":
+  version: 1.11.2
+  resolution: "import-in-the-middle@npm:1.11.2"
+  dependencies:
+    acorn: ^8.8.2
+    acorn-import-attributes: ^1.9.5
+    cjs-module-lexer: ^1.2.2
+    module-details-from-path: ^1.0.3
+  checksum: 06fb73100a918e00778779713119236cc8d3d4656aae9076a18159cfcd28eb0cc26e0a5040d11da309c5f8f8915c143b8d74e73c0734d3f5549b1813d1008bb9
+  languageName: node
+  linkType: hard
+
 "import-local@npm:^3.0.2":
   version: 3.1.0
   resolution: "import-local@npm:3.1.0"
@@ -12345,6 +12430,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"module-details-from-path@npm:^1.0.3":
+  version: 1.0.3
+  resolution: "module-details-from-path@npm:1.0.3"
+  checksum: 378a8a26013889aa3086bfb0776b7860c5bb957336253e1ba5d779c2f239a218930b145ca76e52c1dd7c8079d52b2af64b8eec30822f81ffdb0dfa27d6fe6f33
+  languageName: node
+  linkType: hard
+
 "module-lookup-amd@npm:^7.0.1":
   version: 7.0.1
   resolution: "module-lookup-amd@npm:7.0.1"
@@ -12392,7 +12484,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ms@npm:^2.1.1":
+"ms@npm:^2.1.1, ms@npm:^2.1.3":
   version: 2.1.3
   resolution: "ms@npm:2.1.3"
   checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d
@@ -13910,6 +14002,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"require-in-the-middle@npm:^7.1.1":
+  version: 7.4.0
+  resolution: "require-in-the-middle@npm:7.4.0"
+  dependencies:
+    debug: ^4.3.5
+    module-details-from-path: ^1.0.3
+    resolve: ^1.22.8
+  checksum: 80a3fdf25ef3f7826486469bfebb01365be87316945143f89607d4777b2019e5ac71bf627f1dcd5e2ee6e91a6e49c76c5d452d5a317153531a2907ccb1bc018b
+  languageName: node
+  linkType: hard
+
 "requirejs-config-file@npm:^4.0.0":
   version: 4.0.0
   resolution: "requirejs-config-file@npm:4.0.0"
@@ -14002,7 +14105,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"resolve@npm:^1.1.4, resolve@npm:^1.10.0, resolve@npm:^1.17.0, resolve@npm:^1.20.0, resolve@npm:^1.21.0, resolve@npm:^1.22.4, resolve@npm:^1.4.0":
+"resolve@npm:^1.1.4, resolve@npm:^1.10.0, resolve@npm:^1.17.0, resolve@npm:^1.20.0, resolve@npm:^1.21.0, resolve@npm:^1.22.4, resolve@npm:^1.22.8, resolve@npm:^1.4.0":
   version: 1.22.8
   resolution: "resolve@npm:1.22.8"
   dependencies:
@@ -14025,7 +14128,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"resolve@patch:resolve@^1.1.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.10.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.17.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.20.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.21.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.4.0#~builtin<compat/resolve>":
+"resolve@patch:resolve@^1.1.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.10.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.17.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.20.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.21.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.8#~builtin<compat/resolve>, resolve@patch:resolve@^1.4.0#~builtin<compat/resolve>":
   version: 1.22.8
   resolution: "resolve@patch:resolve@npm%3A1.22.8#~builtin<compat/resolve>::version=1.22.8&hash=c3c19d"
   dependencies:
@@ -14405,6 +14508,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"shimmer@npm:^1.2.1":
+  version: 1.2.1
+  resolution: "shimmer@npm:1.2.1"
+  checksum: aa0d6252ad1c682a4fdfda69e541be987f7a265ac7b00b1208e5e48cc68dc55f293955346ea4c71a169b7324b82c70f8400b3d3d2d60b2a7519f0a3522423250
+  languageName: node
+  linkType: hard
+
 "side-channel@npm:^1.0.4, side-channel@npm:^1.0.6":
   version: 1.0.6
   resolution: "side-channel@npm:1.0.6"

From 35e2a98b044fbd5f6d795dbc5e7ad979a433ee49 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 19:55:28 +0000
Subject: [PATCH 10/39] validator.sh

---
 yarn-project/end-to-end/scripts/native-network/validator.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/yarn-project/end-to-end/scripts/native-network/validator.sh b/yarn-project/end-to-end/scripts/native-network/validator.sh
index f7952af763a..53faa16e920 100755
--- a/yarn-project/end-to-end/scripts/native-network/validator.sh
+++ b/yarn-project/end-to-end/scripts/native-network/validator.sh
@@ -56,6 +56,7 @@ export P2P_TCP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
 export P2P_UDP_LISTEN_ADDR="0.0.0.0:$P2P_PORT"
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT="${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-}"
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT="${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-}"
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT="${OTEL_EXPORTER_OTLP_LOGS_ENDPOINT:-}"
 
 # Add L1 validator
 # this may fail, so try 3 times

From f477cd314d39440735f82ab70d72e621ed9a0cc9 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 20:01:22 +0000
Subject: [PATCH 11/39] telemetry for winston

---
 yarn-project/telemetry-client/package.json |   4 +-
 yarn-project/yarn.lock                     | 176 ++++++++-------------
 2 files changed, 72 insertions(+), 108 deletions(-)

diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json
index 0c854eea28a..18821f0e39b 100644
--- a/yarn-project/telemetry-client/package.json
+++ b/yarn-project/telemetry-client/package.json
@@ -31,12 +31,12 @@
     "@opentelemetry/exporter-metrics-otlp-http": "^0.52.0",
     "@opentelemetry/exporter-trace-otlp-http": "^0.52.0",
     "@opentelemetry/host-metrics": "^0.35.2",
-    "@opentelemetry/instrumentation-winston": "^0.41.0",
     "@opentelemetry/resource-detector-aws": "^1.5.2",
     "@opentelemetry/resources": "^1.25.0",
     "@opentelemetry/sdk-metrics": "^1.25.0",
     "@opentelemetry/sdk-trace-node": "^1.25.0",
-    "@opentelemetry/semantic-conventions": "^1.25.0"
+    "@opentelemetry/semantic-conventions": "^1.25.0",
+    "@opentelemetry/winston-transport": "^0.7.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index 847fa3fdd49..820c1c77037 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -1151,12 +1151,12 @@ __metadata:
     "@opentelemetry/exporter-metrics-otlp-http": ^0.52.0
     "@opentelemetry/exporter-trace-otlp-http": ^0.52.0
     "@opentelemetry/host-metrics": ^0.35.2
-    "@opentelemetry/instrumentation-winston": ^0.41.0
     "@opentelemetry/resource-detector-aws": ^1.5.2
     "@opentelemetry/resources": ^1.25.0
     "@opentelemetry/sdk-metrics": ^1.25.0
     "@opentelemetry/sdk-trace-node": ^1.25.0
     "@opentelemetry/semantic-conventions": ^1.25.0
+    "@opentelemetry/winston-transport": ^0.7.0
     "@types/jest": ^29.5.0
     jest: ^29.5.0
     ts-node: ^10.9.1
@@ -3443,7 +3443,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/api-logs@npm:0.54.0, @opentelemetry/api-logs@npm:^0.54.0":
+"@opentelemetry/api-logs@npm:^0.54.0":
   version: 0.54.0
   resolution: "@opentelemetry/api-logs@npm:0.54.0"
   dependencies:
@@ -3532,34 +3532,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/instrumentation-winston@npm:^0.41.0":
-  version: 0.41.0
-  resolution: "@opentelemetry/instrumentation-winston@npm:0.41.0"
-  dependencies:
-    "@opentelemetry/api-logs": ^0.54.0
-    "@opentelemetry/instrumentation": ^0.54.0
-  peerDependencies:
-    "@opentelemetry/api": ^1.3.0
-  checksum: 10d2769925ae9a9982c8da9bb81c6e121f9c1c3437ac99216eb8c68fddea8a3527b5f9acfb09275646e3bc05183bf5bd2b719576756fa40ea3b98a6950a7533f
-  languageName: node
-  linkType: hard
-
-"@opentelemetry/instrumentation@npm:^0.54.0":
-  version: 0.54.0
-  resolution: "@opentelemetry/instrumentation@npm:0.54.0"
-  dependencies:
-    "@opentelemetry/api-logs": 0.54.0
-    "@types/shimmer": ^1.2.0
-    import-in-the-middle: ^1.8.1
-    require-in-the-middle: ^7.1.1
-    semver: ^7.5.2
-    shimmer: ^1.2.1
-  peerDependencies:
-    "@opentelemetry/api": ^1.3.0
-  checksum: b1fea61c2debea78573a728a3c9065eb6f502e98e6bbc61e98b77492734e86a6fbb79d27694f190bc192429f62a7f97a5025c0bcdff6f30c1c91c9f0cc6d931d
-  languageName: node
-  linkType: hard
-
 "@opentelemetry/otlp-exporter-base@npm:0.52.0":
   version: 0.52.0
   resolution: "@opentelemetry/otlp-exporter-base@npm:0.52.0"
@@ -3717,6 +3689,16 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/winston-transport@npm:^0.7.0":
+  version: 0.7.0
+  resolution: "@opentelemetry/winston-transport@npm:0.7.0"
+  dependencies:
+    "@opentelemetry/api-logs": ^0.54.0
+    winston-transport: 4.*
+  checksum: a75d1915e90ab9beaec842fe2f2ce053ea2b43001d8be7cfd47945fa6e1dee6e1d1b5850becb72c9553edb6904844b685df838a1a2cbea0f2f6edf6ce85dc3bb
+  languageName: node
+  linkType: hard
+
 "@pkgjs/parseargs@npm:^0.11.0":
   version: 0.11.0
   resolution: "@pkgjs/parseargs@npm:0.11.0"
@@ -4786,13 +4768,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@types/shimmer@npm:^1.2.0":
-  version: 1.2.0
-  resolution: "@types/shimmer@npm:1.2.0"
-  checksum: f081a31d826ce7bfe8cc7ba8129d2b1dffae44fd580eba4fcf741237646c4c2494ae6de2cada4b7713d138f35f4bc512dbf01311d813dee82020f97d7d8c491c
-  languageName: node
-  linkType: hard
-
 "@types/sinon@npm:^10.0.15":
   version: 10.0.20
   resolution: "@types/sinon@npm:10.0.20"
@@ -5359,6 +5334,15 @@ __metadata:
   languageName: node
   linkType: hard
 
+"abort-controller@npm:^3.0.0":
+  version: 3.0.0
+  resolution: "abort-controller@npm:3.0.0"
+  dependencies:
+    event-target-shim: ^5.0.0
+  checksum: 170bdba9b47b7e65906a28c8ce4f38a7a369d78e2271706f020849c1bfe0ee2067d4261df8bbb66eb84f79208fd5b710df759d64191db58cfba7ce8ef9c54b75
+  languageName: node
+  linkType: hard
+
 "abortable-iterator@npm:^5.0.1":
   version: 5.0.1
   resolution: "abortable-iterator@npm:5.0.1"
@@ -5402,15 +5386,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"acorn-import-attributes@npm:^1.9.5":
-  version: 1.9.5
-  resolution: "acorn-import-attributes@npm:1.9.5"
-  peerDependencies:
-    acorn: ^8
-  checksum: 1c0c49b6a244503964ae46ae850baccf306e84caf99bc2010ed6103c69a423987b07b520a6c619f075d215388bd4923eccac995886a54309eda049ab78a4be95
-  languageName: node
-  linkType: hard
-
 "acorn-jsx@npm:^5.3.2":
   version: 5.3.2
   resolution: "acorn-jsx@npm:5.3.2"
@@ -6578,13 +6553,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"cjs-module-lexer@npm:^1.2.2":
-  version: 1.4.1
-  resolution: "cjs-module-lexer@npm:1.4.1"
-  checksum: 2556807a99aec1f9daac60741af96cd613a707f343174ae7967da46402c91dced411bf830d209f2e93be4cecea46fc75cecf1f17c799d7d8a9e1dd6204bfcd22
-  languageName: node
-  linkType: hard
-
 "clean-stack@npm:^2.0.0":
   version: 2.2.0
   resolution: "clean-stack@npm:2.2.0"
@@ -7436,18 +7404,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"debug@npm:^4.3.5":
-  version: 4.3.7
-  resolution: "debug@npm:4.3.7"
-  dependencies:
-    ms: ^2.1.3
-  peerDependenciesMeta:
-    supports-color:
-      optional: true
-  checksum: 822d74e209cd910ef0802d261b150314bbcf36c582ccdbb3e70f0894823c17e49a50d3e66d96b633524263975ca16b6a833f3e3b7e030c157169a5fabac63160
-  languageName: node
-  linkType: hard
-
 "decamelize-keys@npm:^1.1.0":
   version: 1.1.1
   resolution: "decamelize-keys@npm:1.1.1"
@@ -8682,6 +8638,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"event-target-shim@npm:^5.0.0":
+  version: 5.0.1
+  resolution: "event-target-shim@npm:5.0.1"
+  checksum: 1ffe3bb22a6d51bdeb6bf6f7cf97d2ff4a74b017ad12284cc9e6a279e727dc30a5de6bb613e5596ff4dc3e517841339ad09a7eec44266eccb1aa201a30448166
+  languageName: node
+  linkType: hard
+
 "eventemitter3@npm:^4.0.0":
   version: 4.0.7
   resolution: "eventemitter3@npm:4.0.7"
@@ -9820,18 +9783,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"import-in-the-middle@npm:^1.8.1":
-  version: 1.11.2
-  resolution: "import-in-the-middle@npm:1.11.2"
-  dependencies:
-    acorn: ^8.8.2
-    acorn-import-attributes: ^1.9.5
-    cjs-module-lexer: ^1.2.2
-    module-details-from-path: ^1.0.3
-  checksum: 06fb73100a918e00778779713119236cc8d3d4656aae9076a18159cfcd28eb0cc26e0a5040d11da309c5f8f8915c143b8d74e73c0734d3f5549b1813d1008bb9
-  languageName: node
-  linkType: hard
-
 "import-local@npm:^3.0.2":
   version: 3.1.0
   resolution: "import-local@npm:3.1.0"
@@ -11861,6 +11812,20 @@ __metadata:
   languageName: node
   linkType: hard
 
+"logform@npm:^2.6.1":
+  version: 2.6.1
+  resolution: "logform@npm:2.6.1"
+  dependencies:
+    "@colors/colors": 1.6.0
+    "@types/triple-beam": ^1.3.2
+    fecha: ^4.2.0
+    ms: ^2.1.1
+    safe-stable-stringify: ^2.3.1
+    triple-beam: ^1.3.0
+  checksum: 0c6b95fa8350ccc33c7c33d77de2a9920205399706fc1b125151c857b61eb90873f4670d9e0e58e58c165b68a363206ae670d6da8b714527c838da3c84449605
+  languageName: node
+  linkType: hard
+
 "long@npm:^5.0.0":
   version: 5.2.3
   resolution: "long@npm:5.2.3"
@@ -12430,13 +12395,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"module-details-from-path@npm:^1.0.3":
-  version: 1.0.3
-  resolution: "module-details-from-path@npm:1.0.3"
-  checksum: 378a8a26013889aa3086bfb0776b7860c5bb957336253e1ba5d779c2f239a218930b145ca76e52c1dd7c8079d52b2af64b8eec30822f81ffdb0dfa27d6fe6f33
-  languageName: node
-  linkType: hard
-
 "module-lookup-amd@npm:^7.0.1":
   version: 7.0.1
   resolution: "module-lookup-amd@npm:7.0.1"
@@ -12484,7 +12442,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"ms@npm:^2.1.1, ms@npm:^2.1.3":
+"ms@npm:^2.1.1":
   version: 2.1.3
   resolution: "ms@npm:2.1.3"
   checksum: aa92de608021b242401676e35cfa5aa42dd70cbdc082b916da7fb925c542173e36bce97ea3e804923fe92c0ad991434e4a38327e15a1b5b5f945d66df615ae6d
@@ -13927,6 +13885,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"readable-stream@npm:^4.5.2":
+  version: 4.5.2
+  resolution: "readable-stream@npm:4.5.2"
+  dependencies:
+    abort-controller: ^3.0.0
+    buffer: ^6.0.3
+    events: ^3.3.0
+    process: ^0.11.10
+    string_decoder: ^1.3.0
+  checksum: c4030ccff010b83e4f33289c535f7830190773e274b3fcb6e2541475070bdfd69c98001c3b0cb78763fc00c8b62f514d96c2b10a8bd35d5ce45203a25fa1d33a
+  languageName: node
+  linkType: hard
+
 "receptacle@npm:^1.3.2":
   version: 1.3.2
   resolution: "receptacle@npm:1.3.2"
@@ -14002,17 +13973,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"require-in-the-middle@npm:^7.1.1":
-  version: 7.4.0
-  resolution: "require-in-the-middle@npm:7.4.0"
-  dependencies:
-    debug: ^4.3.5
-    module-details-from-path: ^1.0.3
-    resolve: ^1.22.8
-  checksum: 80a3fdf25ef3f7826486469bfebb01365be87316945143f89607d4777b2019e5ac71bf627f1dcd5e2ee6e91a6e49c76c5d452d5a317153531a2907ccb1bc018b
-  languageName: node
-  linkType: hard
-
 "requirejs-config-file@npm:^4.0.0":
   version: 4.0.0
   resolution: "requirejs-config-file@npm:4.0.0"
@@ -14105,7 +14065,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"resolve@npm:^1.1.4, resolve@npm:^1.10.0, resolve@npm:^1.17.0, resolve@npm:^1.20.0, resolve@npm:^1.21.0, resolve@npm:^1.22.4, resolve@npm:^1.22.8, resolve@npm:^1.4.0":
+"resolve@npm:^1.1.4, resolve@npm:^1.10.0, resolve@npm:^1.17.0, resolve@npm:^1.20.0, resolve@npm:^1.21.0, resolve@npm:^1.22.4, resolve@npm:^1.4.0":
   version: 1.22.8
   resolution: "resolve@npm:1.22.8"
   dependencies:
@@ -14128,7 +14088,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"resolve@patch:resolve@^1.1.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.10.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.17.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.20.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.21.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.8#~builtin<compat/resolve>, resolve@patch:resolve@^1.4.0#~builtin<compat/resolve>":
+"resolve@patch:resolve@^1.1.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.10.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.17.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.20.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.21.0#~builtin<compat/resolve>, resolve@patch:resolve@^1.22.4#~builtin<compat/resolve>, resolve@patch:resolve@^1.4.0#~builtin<compat/resolve>":
   version: 1.22.8
   resolution: "resolve@patch:resolve@npm%3A1.22.8#~builtin<compat/resolve>::version=1.22.8&hash=c3c19d"
   dependencies:
@@ -14508,13 +14468,6 @@ __metadata:
   languageName: node
   linkType: hard
 
-"shimmer@npm:^1.2.1":
-  version: 1.2.1
-  resolution: "shimmer@npm:1.2.1"
-  checksum: aa0d6252ad1c682a4fdfda69e541be987f7a265ac7b00b1208e5e48cc68dc55f293955346ea4c71a169b7324b82c70f8400b3d3d2d60b2a7519f0a3522423250
-  languageName: node
-  linkType: hard
-
 "side-channel@npm:^1.0.4, side-channel@npm:^1.0.6":
   version: 1.0.6
   resolution: "side-channel@npm:1.0.6"
@@ -14995,7 +14948,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"string_decoder@npm:^1.1.1":
+"string_decoder@npm:^1.1.1, string_decoder@npm:^1.3.0":
   version: 1.3.0
   resolution: "string_decoder@npm:1.3.0"
   dependencies:
@@ -16447,6 +16400,17 @@ __metadata:
   languageName: node
   linkType: hard
 
+"winston-transport@npm:4.*":
+  version: 4.8.0
+  resolution: "winston-transport@npm:4.8.0"
+  dependencies:
+    logform: ^2.6.1
+    readable-stream: ^4.5.2
+    triple-beam: ^1.3.0
+  checksum: f84092188176d49a6f4f75321ba3e50107ac0942a51a6d7e36b80af19dafb22b57258aaa6d8220763044ea23e30bffd597d3280d2a2298e6a491fe424896bac7
+  languageName: node
+  linkType: hard
+
 "winston-transport@npm:^4.4.0, winston-transport@npm:^4.7.0":
   version: 4.7.0
   resolution: "winston-transport@npm:4.7.0"

From f83ed70aacdd1c3ad8631de3acdd4d1708adf699 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 20:45:09 +0000
Subject: [PATCH 12/39] negative patterns in core logger

---
 yarn-project/aztec/src/logging.ts             | 26 +-----
 yarn-project/foundation/src/log/logger.ts     | 85 ++++++++++++++++---
 .../validator-client/src/validator.ts         |  4 +-
 3 files changed, 75 insertions(+), 40 deletions(-)

diff --git a/yarn-project/aztec/src/logging.ts b/yarn-project/aztec/src/logging.ts
index d59cd9b33cd..9c71fa54205 100644
--- a/yarn-project/aztec/src/logging.ts
+++ b/yarn-project/aztec/src/logging.ts
@@ -30,35 +30,13 @@ function createWinstonLocalFileLogger() {
   });
 }
 
-function extractNegativePatterns(debugString: string): string[] {
-  return (
-    debugString
-      .split(',')
-      .filter(p => p.startsWith('-'))
-      // Remove the leading '-' from the pattern
-      .map(p => p.slice(1))
-      // Remove any '*' from the pattern
-      .map(p => p.replace('*', ''))
-  );
-}
-
 /** Creates a winston logger that logs everything to stdout in json format */
-function createWinstonJsonStdoutLogger(
-  debugString: string = process.env.DEBUG ??
-    'aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*',
-) {
-  const ignorePatterns = extractNegativePatterns(debugString);
-  const ignoreAztecPattern = format(info => {
-    if (ignorePatterns.some(pattern => info.module.startsWith(pattern))) {
-      return false; // Skip logging this message
-    }
-    return info;
-  });
+function createWinstonJsonStdoutLogger() {
   return winston.createLogger({
     level: currentLevel,
     transports: [
       new winston.transports.Console({
-        format: format.combine(format.timestamp(), ignoreAztecPattern(), format.json()),
+        format: format.combine(format.timestamp(), format.json()),
       }),
     ],
   });
diff --git a/yarn-project/foundation/src/log/logger.ts b/yarn-project/foundation/src/log/logger.ts
index b3fe598e897..ed4305687c8 100644
--- a/yarn-project/foundation/src/log/logger.ts
+++ b/yarn-project/foundation/src/log/logger.ts
@@ -19,8 +19,24 @@ function getLogLevel() {
 
 export let currentLevel = getLogLevel();
 
+function filterNegativePatterns(debugString: string): string {
+  return debugString
+    .split(',')
+    .filter(p => !p.startsWith('-'))
+    .join(',');
+}
+function extractNegativePatterns(debugString: string): string[] {
+  return (
+    debugString
+      .split(',')
+      .filter(p => p.startsWith('-'))
+      // Remove the leading '-' from the pattern
+      .map(p => p.slice(1))
+  );
+}
+
 const namespaces = process.env.DEBUG ?? 'aztec:*';
-debug.enable(namespaces);
+debug.enable(filterNegativePatterns(namespaces));
 
 /** Log function that accepts an exception object */
 type ErrorLogFn = (msg: string, err?: Error | unknown, data?: LogData) => void;
@@ -42,30 +58,71 @@ export type DebugLogger = Logger;
  * Uses npm debug for debug level and console.error for other levels.
  * @param name - Name of the module.
  * @param fixedLogData - Additional data to include in the log message.
- * @usage createDebugLogger('aztec:validator', {validatorAddress: '0x1234...'});
+ * @usage createDebugLogger('aztec:validator');
  * // will always add the validator address to the log labels
  * @returns A debug logger.
  */
 
-export function createDebugLogger(name: string, fixedLogData?: LogData): DebugLogger {
+export function createDebugLogger(name: string): DebugLogger {
   const debugLogger = debug(name);
 
-  const attatchFixedLogData = (data?: LogData) => ({ ...fixedLogData, ...data });
+  const negativePatterns = extractNegativePatterns(namespaces);
+  const accepted = () => {
+    return !negativePatterns.some(pattern => name.match(pattern));
+  };
+  const log = (level: LogLevel, msg: string, data?: LogData) => {
+    if (accepted()) {
+      logWithDebug(debugLogger, level, msg, data);
+    }
+  };
+  const logger = {
+    silent: () => {},
+    error: (msg: string, err?: unknown, data?: LogData) => log('error', fmtErr(msg, err), data),
+    warn: (msg: string, data?: LogData) => log('warn', msg, data),
+    info: (msg: string, data?: LogData) => log('info', msg, data),
+    verbose: (msg: string, data?: LogData) => log('verbose', msg, data),
+    debug: (msg: string, data?: LogData) => log('debug', msg, data),
+  };
+  return Object.assign((msg: string, data?: LogData) => log('debug', msg, data), logger);
+}
 
+/**
+ * A function to create a logger that automatically includes fixed data in each log entry.
+ * @param debugLogger - The base DebugLogger instance to which we attach fixed log data.
+ * @param fixedLogData - The data to be included in every log entry.
+ * @returns A DebugLogger with log level methods (error, warn, info, verbose, debug) that
+ * automatically attach `fixedLogData` to every log message.
+ */
+export function attachedFixedDataToLogger(debugLogger: DebugLogger, fixedLogData: LogData): DebugLogger {
+  // Helper function to merge fixed data with additional data passed to log entries.
+  const attach = (data?: LogData) => ({ ...fixedLogData, ...data });
+
+  // Define the logger with all the necessary log level methods.
   const logger = {
+    // Silent log level does nothing.
     silent: () => {},
-    error: (msg: string, err?: unknown, data?: LogData) =>
-      logWithDebug(debugLogger, 'error', fmtErr(msg, err), attatchFixedLogData(data)),
-    warn: (msg: string, data?: LogData) => logWithDebug(debugLogger, 'warn', msg, attatchFixedLogData(data)),
-    info: (msg: string, data?: LogData) => logWithDebug(debugLogger, 'info', msg, attatchFixedLogData(data)),
-    verbose: (msg: string, data?: LogData) => logWithDebug(debugLogger, 'verbose', msg, attatchFixedLogData(data)),
-    debug: (msg: string, data?: LogData) => logWithDebug(debugLogger, 'debug', msg, attatchFixedLogData(data)),
+
+    // Error log level includes a message, error details, and additional data.
+    error: (msg: string, err?: unknown, data?: LogData) => debugLogger.error(fmtErr(msg, err), attach(data)),
+
+    // Warning log level includes a message and additional data.
+    warn: (msg: string, data?: LogData) => debugLogger.warn(msg, attach(data)),
+
+    // Info log level includes a message and additional data.
+    info: (msg: string, data?: LogData) => debugLogger.info(msg, attach(data)),
+
+    // Verbose log level includes a message and additional data.
+    verbose: (msg: string, data?: LogData) => debugLogger.verbose(msg, attach(data)),
+
+    // Debug log level includes a message and additional data.
+    debug: (msg: string, data?: LogData) => debugLogger.debug(msg, attach(data)),
   };
-  return Object.assign(
-    (msg: string, data?: LogData) => logWithDebug(debugLogger, 'debug', msg, attatchFixedLogData(data)),
-    logger,
-  );
+
+  // Return the logger as a function with `debug` as the default log level.
+  // The function will act as a shortcut to the `debug` log level.
+  return Object.assign((msg: string, data?: LogData) => debugLogger.debug(msg, attach(data)), logger);
 }
+
 /** A callback to capture all logs. */
 export type LogHandler = (level: LogLevel, namespace: string, msg: string, data?: LogData) => void;
 
diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts
index 1c516b195dd..7c11d474f6d 100644
--- a/yarn-project/validator-client/src/validator.ts
+++ b/yarn-project/validator-client/src/validator.ts
@@ -2,7 +2,7 @@ import { type BlockAttestation, type BlockProposal, type TxHash } from '@aztec/c
 import { type Header } from '@aztec/circuits.js';
 import { Buffer32 } from '@aztec/foundation/buffer';
 import { type Fr } from '@aztec/foundation/fields';
-import { createDebugLogger } from '@aztec/foundation/log';
+import { attachedFixedDataToLogger, createDebugLogger } from '@aztec/foundation/log';
 import { sleep } from '@aztec/foundation/sleep';
 import { type P2P } from '@aztec/p2p';
 import { type TelemetryClient, WithTracer } from '@aztec/telemetry-client';
@@ -40,7 +40,7 @@ export class ValidatorClient extends WithTracer implements Validator {
     private attestationPoolingIntervalMs: number,
     private attestationWaitTimeoutMs: number,
     telemetry: TelemetryClient,
-    private log = createDebugLogger('aztec:validator', { validatorAddress: keyStore.getAddress().toString() }),
+    private log = attachedFixedDataToLogger(createDebugLogger('aztec:validator'), { validatorAddress: keyStore.getAddress().toString() }),
   ) {
     // Instantiate tracer
     super(telemetry, 'Validator');

From 739acc7067c8111619578b75fca8ac129e0be6ec Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 21:07:46 +0000
Subject: [PATCH 13/39] negative patterns in core logger

---
 yarn-project/telemetry-client/package.json |  3 +-
 yarn-project/telemetry-client/src/otel.ts  | 49 +++++++++++++---------
 yarn-project/telemetry-client/src/start.ts |  7 +++-
 yarn-project/yarn.lock                     |  1 +
 4 files changed, 38 insertions(+), 22 deletions(-)

diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json
index 18821f0e39b..1455ddb44c4 100644
--- a/yarn-project/telemetry-client/package.json
+++ b/yarn-project/telemetry-client/package.json
@@ -36,7 +36,8 @@
     "@opentelemetry/sdk-metrics": "^1.25.0",
     "@opentelemetry/sdk-trace-node": "^1.25.0",
     "@opentelemetry/semantic-conventions": "^1.25.0",
-    "@opentelemetry/winston-transport": "^0.7.0"
+    "@opentelemetry/winston-transport": "^0.7.0",
+    "winston": "^3.10.0"
   },
   "devDependencies": {
     "@jest/globals": "^29.5.0",
diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts
index 7e6ae406233..fca2caf8717 100644
--- a/yarn-project/telemetry-client/src/otel.ts
+++ b/yarn-project/telemetry-client/src/otel.ts
@@ -1,31 +1,41 @@
-import { type DebugLogger } from '@aztec/foundation/log';
-
-import {
-  DiagConsoleLogger,
-  DiagLogLevel,
-  type Meter,
-  type Tracer,
-  type TracerProvider,
-  diag,
-} from '@opentelemetry/api';
+import { type DebugLogger, currentLevel, onLog } from '@aztec/foundation/log';
+
+
+
+import { DiagConsoleLogger, DiagLogLevel, type Meter, type Tracer, type TracerProvider, diag } from '@opentelemetry/api';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { HostMetrics } from '@opentelemetry/host-metrics';
 import { awsEc2Detector, awsEcsDetector } from '@opentelemetry/resource-detector-aws';
-import {
-  type IResource,
-  detectResourcesSync,
-  envDetectorSync,
-  osDetectorSync,
-  processDetectorSync,
-  serviceInstanceIdDetectorSync,
-} from '@opentelemetry/resources';
+import { type IResource, detectResourcesSync, envDetectorSync, osDetectorSync, processDetectorSync, serviceInstanceIdDetectorSync } from '@opentelemetry/resources';
 import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+import * as winston from 'winston';
 
 import { aztecDetector } from './aztec_resource_detector.js';
 import { type Gauge, type TelemetryClient } from './telemetry.js';
+import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
+
+
+export function setupConsoleJsonLog() {
+  const logger = createWinstonJsonStdoutLogger();
+  onLog((level, module, message, data) => {
+    logger.log({ ...data, level, module, message });
+  });
+}
+
+function createWinstonJsonStdoutLogger() {
+  const { format } = winston;
+  return winston.createLogger({
+    level: currentLevel,
+    transports: [
+      new OpenTelemetryTransportV3({
+        format: format.combine(format.timestamp(), format.json()),
+      }),
+    ],
+  });
+}
 
 export class OpenTelemetryClient implements TelemetryClient {
   hostMetrics: HostMetrics | undefined;
@@ -77,6 +87,7 @@ export class OpenTelemetryClient implements TelemetryClient {
   public static async createAndStart(
     metricsCollector: URL,
     tracesCollector: URL | undefined,
+    logsCollector: URL | undefined,
     log: DebugLogger,
   ): Promise<OpenTelemetryClient> {
     const resource = detectResourcesSync({
@@ -122,4 +133,4 @@ export class OpenTelemetryClient implements TelemetryClient {
 
     return service;
   }
-}
+}
\ No newline at end of file
diff --git a/yarn-project/telemetry-client/src/start.ts b/yarn-project/telemetry-client/src/start.ts
index eb07a4a431b..39ee91b1bdf 100644
--- a/yarn-project/telemetry-client/src/start.ts
+++ b/yarn-project/telemetry-client/src/start.ts
@@ -1,19 +1,22 @@
 import { createDebugLogger } from '@aztec/foundation/log';
 
+
+
 import { type TelemetryClientConfig } from './config.js';
 import { NoopTelemetryClient } from './noop.js';
 import { OpenTelemetryClient } from './otel.js';
 import { type TelemetryClient } from './telemetry.js';
 
+
 export * from './config.js';
 
 export async function createAndStartTelemetryClient(config: TelemetryClientConfig): Promise<TelemetryClient> {
   const log = createDebugLogger('aztec:telemetry-client');
   if (config.metricsCollectorUrl) {
     log.info('Using OpenTelemetry client');
-    return await OpenTelemetryClient.createAndStart(config.metricsCollectorUrl, config.tracesCollectorUrl, log);
+    return await OpenTelemetryClient.createAndStart(config.metricsCollectorUrl, config.tracesCollectorUrl, config.logsCollectorUrl, log);
   } else {
     log.info('Using NoopTelemetryClient');
     return new NoopTelemetryClient();
   }
-}
+}
\ No newline at end of file
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index 820c1c77037..be33295e880 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -1161,6 +1161,7 @@ __metadata:
     jest: ^29.5.0
     ts-node: ^10.9.1
     typescript: ^5.0.4
+    winston: ^3.10.0
   languageName: unknown
   linkType: soft
 

From 45be95d8581ebcc64121bf3517e94359b8b0e75d Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 21:11:09 +0000
Subject: [PATCH 14/39] yarn project working

---
 yarn-project/protocol-contracts/src/protocol_contract_data.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/yarn-project/protocol-contracts/src/protocol_contract_data.ts b/yarn-project/protocol-contracts/src/protocol_contract_data.ts
index 11810f96b37..8512d948c85 100644
--- a/yarn-project/protocol-contracts/src/protocol_contract_data.ts
+++ b/yarn-project/protocol-contracts/src/protocol_contract_data.ts
@@ -55,9 +55,9 @@ export const ProtocolContractLeaf = {
   ContractClassRegisterer: Fr.fromString('0x147ba3294403576dbad10f86d3ffd4eb83fb230ffbcd5c8b153dd02942d0611f'),
   MultiCallEntrypoint: Fr.fromString('0x154b701b41d6cf6da7204fef36b2ee9578b449d21b3792a9287bf45eba48fd26'),
   FeeJuice: Fr.fromString('0x21c9ab2e339c9b3394e4e1ff3b7cf37be4e0fc0bc177a192d287d98963b9b254'),
-  Router: Fr.fromString('0x2779d7e4ccba389da358a0c9362364d0c65e14cd4d9df929e6722187e808e068'),
+  Router: Fr.fromString('0x2b0b558e92b7a13cde0a2ecc7570c181a6fbae2bdc6f966cacfc39a784635394'),
 };
 
 export const protocolContractTreeRoot = Fr.fromString(
-  '0x00724e4de088411c873c3d6975491eb48889bfa51bc854744a4fcc307ee9abd8',
+  '0x0d560ad12f14dd5026070bc037ac343535db339212f0904dfc96c4aea4dcc8ab',
 );

From 137b3246ba7f20936d55e02fe00ef2b6199f4c16 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 22:02:45 +0000
Subject: [PATCH 15/39] yarn project working

---
 yarn-project/telemetry-client/package.json |   5 +-
 yarn-project/telemetry-client/src/otel.ts  |  56 ++++-----
 yarn-project/yarn.lock                     | 139 +++++++++++++++++++--
 3 files changed, 156 insertions(+), 44 deletions(-)

diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json
index 1455ddb44c4..8ebe92df672 100644
--- a/yarn-project/telemetry-client/package.json
+++ b/yarn-project/telemetry-client/package.json
@@ -28,11 +28,14 @@
   "dependencies": {
     "@aztec/foundation": "workspace:^",
     "@opentelemetry/api": "^1.9.0",
+    "@opentelemetry/api-logs": "^0.54.0",
+    "@opentelemetry/exporter-logs-otlp-http": "^0.54.0",
     "@opentelemetry/exporter-metrics-otlp-http": "^0.52.0",
-    "@opentelemetry/exporter-trace-otlp-http": "^0.52.0",
+    "@opentelemetry/exporter-trace-otlp-http": "^0.54.0",
     "@opentelemetry/host-metrics": "^0.35.2",
     "@opentelemetry/resource-detector-aws": "^1.5.2",
     "@opentelemetry/resources": "^1.25.0",
+    "@opentelemetry/sdk-logs": "^0.54.0",
     "@opentelemetry/sdk-metrics": "^1.25.0",
     "@opentelemetry/sdk-trace-node": "^1.25.0",
     "@opentelemetry/semantic-conventions": "^1.25.0",
diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts
index fca2caf8717..61f2444d3e9 100644
--- a/yarn-project/telemetry-client/src/otel.ts
+++ b/yarn-project/telemetry-client/src/otel.ts
@@ -1,41 +1,33 @@
-import { type DebugLogger, currentLevel, onLog } from '@aztec/foundation/log';
-
-
-
-import { DiagConsoleLogger, DiagLogLevel, type Meter, type Tracer, type TracerProvider, diag } from '@opentelemetry/api';
+import { type DebugLogger } from '@aztec/foundation/log';
+
+import {
+  DiagConsoleLogger,
+  DiagLogLevel,
+  type Meter,
+  type Tracer,
+  type TracerProvider,
+  diag,
+} from '@opentelemetry/api';
 import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
 import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
 import { HostMetrics } from '@opentelemetry/host-metrics';
 import { awsEc2Detector, awsEcsDetector } from '@opentelemetry/resource-detector-aws';
-import { type IResource, detectResourcesSync, envDetectorSync, osDetectorSync, processDetectorSync, serviceInstanceIdDetectorSync } from '@opentelemetry/resources';
+import {
+  type IResource,
+  detectResourcesSync,
+  envDetectorSync,
+  osDetectorSync,
+  processDetectorSync,
+  serviceInstanceIdDetectorSync,
+} from '@opentelemetry/resources';
+import { type LoggerProvider } from '@opentelemetry/sdk-logs';
 import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics';
 import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
 import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
-import * as winston from 'winston';
 
 import { aztecDetector } from './aztec_resource_detector.js';
+import { registerOtelLoggerProvider } from './otelLoggerProvider.js';
 import { type Gauge, type TelemetryClient } from './telemetry.js';
-import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
-
-
-export function setupConsoleJsonLog() {
-  const logger = createWinstonJsonStdoutLogger();
-  onLog((level, module, message, data) => {
-    logger.log({ ...data, level, module, message });
-  });
-}
-
-function createWinstonJsonStdoutLogger() {
-  const { format } = winston;
-  return winston.createLogger({
-    level: currentLevel,
-    transports: [
-      new OpenTelemetryTransportV3({
-        format: format.combine(format.timestamp(), format.json()),
-      }),
-    ],
-  });
-}
 
 export class OpenTelemetryClient implements TelemetryClient {
   hostMetrics: HostMetrics | undefined;
@@ -45,6 +37,7 @@ export class OpenTelemetryClient implements TelemetryClient {
     private resource: IResource,
     private meterProvider: MeterProvider,
     private traceProvider: TracerProvider,
+    private loggerProvider: LoggerProvider,
     private log: DebugLogger,
   ) {}
 
@@ -81,7 +74,7 @@ export class OpenTelemetryClient implements TelemetryClient {
   }
 
   public async stop() {
-    await Promise.all([this.meterProvider.shutdown()]);
+    await Promise.all([this.meterProvider.shutdown(), this.loggerProvider.shutdown()]);
   }
 
   public static async createAndStart(
@@ -127,10 +120,11 @@ export class OpenTelemetryClient implements TelemetryClient {
         }),
       ],
     });
+    const loggerProvider = registerOtelLoggerProvider(resource, logsCollector);
 
-    const service = new OpenTelemetryClient(resource, meterProvider, tracerProvider, log);
+    const service = new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
     service.start();
 
     return service;
   }
-}
\ No newline at end of file
+}
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index be33295e880..b2f5cf6cf8e 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -1148,11 +1148,14 @@ __metadata:
     "@aztec/foundation": "workspace:^"
     "@jest/globals": ^29.5.0
     "@opentelemetry/api": ^1.9.0
+    "@opentelemetry/api-logs": ^0.54.0
+    "@opentelemetry/exporter-logs-otlp-http": ^0.54.0
     "@opentelemetry/exporter-metrics-otlp-http": ^0.52.0
-    "@opentelemetry/exporter-trace-otlp-http": ^0.52.0
+    "@opentelemetry/exporter-trace-otlp-http": ^0.54.0
     "@opentelemetry/host-metrics": ^0.35.2
     "@opentelemetry/resource-detector-aws": ^1.5.2
     "@opentelemetry/resources": ^1.25.0
+    "@opentelemetry/sdk-logs": ^0.54.0
     "@opentelemetry/sdk-metrics": ^1.25.0
     "@opentelemetry/sdk-trace-node": ^1.25.0
     "@opentelemetry/semantic-conventions": ^1.25.0
@@ -3444,7 +3447,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/api-logs@npm:^0.54.0":
+"@opentelemetry/api-logs@npm:0.54.0, @opentelemetry/api-logs@npm:^0.54.0":
   version: 0.54.0
   resolution: "@opentelemetry/api-logs@npm:0.54.0"
   dependencies:
@@ -3491,6 +3494,32 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/core@npm:1.27.0":
+  version: 1.27.0
+  resolution: "@opentelemetry/core@npm:1.27.0"
+  dependencies:
+    "@opentelemetry/semantic-conventions": 1.27.0
+  peerDependencies:
+    "@opentelemetry/api": ">=1.0.0 <1.10.0"
+  checksum: 33ff551f89f0bb95830c9f9464c43b11adf88882ec1d3a03a5b9afcc89d2aafab33c36cb5047f18667d7929d6ab40ed0121649c42d0105f1cb33ffdca48f8b13
+  languageName: node
+  linkType: hard
+
+"@opentelemetry/exporter-logs-otlp-http@npm:^0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/exporter-logs-otlp-http@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/api-logs": 0.54.0
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/otlp-exporter-base": 0.54.0
+    "@opentelemetry/otlp-transformer": 0.54.0
+    "@opentelemetry/sdk-logs": 0.54.0
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: 407cde2dd930aa19c0c826147d15aba84f94a58f1afbf86cfa1c41576be4492b689e1e9c7971a92805b051851cd6fab063bf24f29160b14c2d3b2cf1fded2bec
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/exporter-metrics-otlp-http@npm:^0.52.0":
   version: 0.52.0
   resolution: "@opentelemetry/exporter-metrics-otlp-http@npm:0.52.0"
@@ -3506,18 +3535,18 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@opentelemetry/exporter-trace-otlp-http@npm:^0.52.0":
-  version: 0.52.0
-  resolution: "@opentelemetry/exporter-trace-otlp-http@npm:0.52.0"
+"@opentelemetry/exporter-trace-otlp-http@npm:^0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/exporter-trace-otlp-http@npm:0.54.0"
   dependencies:
-    "@opentelemetry/core": 1.25.0
-    "@opentelemetry/otlp-exporter-base": 0.52.0
-    "@opentelemetry/otlp-transformer": 0.52.0
-    "@opentelemetry/resources": 1.25.0
-    "@opentelemetry/sdk-trace-base": 1.25.0
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/otlp-exporter-base": 0.54.0
+    "@opentelemetry/otlp-transformer": 0.54.0
+    "@opentelemetry/resources": 1.27.0
+    "@opentelemetry/sdk-trace-base": 1.27.0
   peerDependencies:
-    "@opentelemetry/api": ^1.0.0
-  checksum: bed18523289c579b8108b1c3fcb2b74361bed2d7f3016270feb080a047fa422fc9dfb0678ff1b726cb1e0fa9413cead5824e7f97d1d781467aa983a87fe1ee93
+    "@opentelemetry/api": ^1.3.0
+  checksum: e53263c3ffcfe62d7d299efac9515a977d284aabc4c89a961cec60853095f24e439abae348c326c7bd88911a85d08dd57833a64769e20254d02df8ac73e9b277
   languageName: node
   linkType: hard
 
@@ -3545,6 +3574,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/otlp-exporter-base@npm:0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/otlp-exporter-base@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/otlp-transformer": 0.54.0
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: ded78325f22cd98314971216eb18d8f021a6cf7f3b1f69d08b0d257880deb2d409d598bfc3a6016b0557a1ec3b0c50527ba9acf09d4e3902f48d003f763441c0
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/otlp-transformer@npm:0.52.0":
   version: 0.52.0
   resolution: "@opentelemetry/otlp-transformer@npm:0.52.0"
@@ -3562,6 +3603,23 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/otlp-transformer@npm:0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/otlp-transformer@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/api-logs": 0.54.0
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/resources": 1.27.0
+    "@opentelemetry/sdk-logs": 0.54.0
+    "@opentelemetry/sdk-metrics": 1.27.0
+    "@opentelemetry/sdk-trace-base": 1.27.0
+    protobufjs: ^7.3.0
+  peerDependencies:
+    "@opentelemetry/api": ^1.3.0
+  checksum: 69451290ec2c65ee27f35b29d41a1b961d169ff928d231805c2694cbc4b4bda788027cf8149a6a1325da7c3bc2ca20dc939ef91a4f3e2af481ed187653386610
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/propagator-b3@npm:1.25.0":
   version: 1.25.0
   resolution: "@opentelemetry/propagator-b3@npm:1.25.0"
@@ -3609,6 +3667,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/resources@npm:1.27.0":
+  version: 1.27.0
+  resolution: "@opentelemetry/resources@npm:1.27.0"
+  dependencies:
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/semantic-conventions": 1.27.0
+  peerDependencies:
+    "@opentelemetry/api": ">=1.0.0 <1.10.0"
+  checksum: 43d298afea7daf7524e6b98c1441bcce9fa73b76aecf17e36cabb1a4cfaae6818acf9759d3e42706b1fd91243644076d2291e78c3ed81641d3b351fcff6cb9a9
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/resources@npm:^1.0.0":
   version: 1.25.1
   resolution: "@opentelemetry/resources@npm:1.25.1"
@@ -3634,6 +3704,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/sdk-logs@npm:0.54.0, @opentelemetry/sdk-logs@npm:^0.54.0":
+  version: 0.54.0
+  resolution: "@opentelemetry/sdk-logs@npm:0.54.0"
+  dependencies:
+    "@opentelemetry/api-logs": 0.54.0
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/resources": 1.27.0
+  peerDependencies:
+    "@opentelemetry/api": ">=1.4.0 <1.10.0"
+  checksum: fd6db65af6d7afdb454eac1df8a4029d3d287d37e9289a4d128bea07995e8843b7b1e5d1f39aa39538397ce1b6bf624cc2548f40dc18324ba3bbaec86dd845b9
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/sdk-metrics@npm:1.25.0, @opentelemetry/sdk-metrics@npm:^1.25.0, @opentelemetry/sdk-metrics@npm:^1.8.0":
   version: 1.25.0
   resolution: "@opentelemetry/sdk-metrics@npm:1.25.0"
@@ -3647,6 +3730,18 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/sdk-metrics@npm:1.27.0":
+  version: 1.27.0
+  resolution: "@opentelemetry/sdk-metrics@npm:1.27.0"
+  dependencies:
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/resources": 1.27.0
+  peerDependencies:
+    "@opentelemetry/api": ">=1.3.0 <1.10.0"
+  checksum: c8776577063a3a5199d5717247270daf5820ce6636530b5ea4b5a8d6b40170cec9bb6b56dacb5c118d2e90588af83d0ebbb13f4d370c7efe50f69d22e5d13463
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/sdk-trace-base@npm:1.25.0":
   version: 1.25.0
   resolution: "@opentelemetry/sdk-trace-base@npm:1.25.0"
@@ -3660,6 +3755,19 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/sdk-trace-base@npm:1.27.0":
+  version: 1.27.0
+  resolution: "@opentelemetry/sdk-trace-base@npm:1.27.0"
+  dependencies:
+    "@opentelemetry/core": 1.27.0
+    "@opentelemetry/resources": 1.27.0
+    "@opentelemetry/semantic-conventions": 1.27.0
+  peerDependencies:
+    "@opentelemetry/api": ">=1.0.0 <1.10.0"
+  checksum: d28c36724aeaf4884f7957e2ab138d9a0ca715a68b2ad23e2935ff0e39cd438c57fd0c8cc85fd5e280464857ede1ae8f9c8e40a37088a1e34d2e625e77276fee
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/sdk-trace-node@npm:^1.25.0":
   version: 1.25.0
   resolution: "@opentelemetry/sdk-trace-node@npm:1.25.0"
@@ -3690,6 +3798,13 @@ __metadata:
   languageName: node
   linkType: hard
 
+"@opentelemetry/semantic-conventions@npm:1.27.0":
+  version: 1.27.0
+  resolution: "@opentelemetry/semantic-conventions@npm:1.27.0"
+  checksum: 26d85f8d13c8c64024f7a84528cff41d56afc9829e7ff8a654576404f8b2c1a9c264adcc6fa5a9551bacdd938a4a464041fa9493e0a722e5605f2c2ae6752398
+  languageName: node
+  linkType: hard
+
 "@opentelemetry/winston-transport@npm:^0.7.0":
   version: 0.7.0
   resolution: "@opentelemetry/winston-transport@npm:0.7.0"

From 8fe411cceb8f57feb2439a56221fb666a82948ae Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Mon, 28 Oct 2024 23:47:40 +0000
Subject: [PATCH 16/39] metrics

---
 metrics/otel/otel.yml                          | 3 +++
 spartan/aztec-network/templates/boot-node.yaml | 2 +-
 spartan/aztec-network/templates/validator.yaml | 2 +-
 spartan/metrics/install-prod.sh                | 4 +++-
 spartan/scripts/deploy_spartan.sh              | 4 ++--
 5 files changed, 10 insertions(+), 5 deletions(-)

diff --git a/metrics/otel/otel.yml b/metrics/otel/otel.yml
index b9e70117034..675e6579132 100644
--- a/metrics/otel/otel.yml
+++ b/metrics/otel/otel.yml
@@ -27,3 +27,6 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [prometheus, debug]
+    logs:
+      receivers: [otlp]
+      exporters: [logging]
diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 6195582fa14..59f99ba4a7f 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -142,7 +142,7 @@ spec:
             - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
               value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
             - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
+              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           ports:
             - containerPort: {{ .Values.bootNode.service.nodePort }}
             - containerPort: {{ .Values.bootNode.service.p2pTcpPort }}
diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index 6c264c28075..963e778d745 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -131,7 +131,7 @@ spec:
             - name: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT
               value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
             - name: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT
-              value: {{ include "aztec-network.otelCollectorTracesEndpoint" . | quote }}
+              value: {{ include "aztec-network.otelCollectorLogsEndpoint" . | quote }}
           ports:
             - containerPort: {{ .Values.validator.service.nodePort }}
             - containerPort: {{ .Values.validator.service.p2pTcpPort }}
diff --git a/spartan/metrics/install-prod.sh b/spartan/metrics/install-prod.sh
index cd62570853d..849dbcd1cc9 100755
--- a/spartan/metrics/install-prod.sh
+++ b/spartan/metrics/install-prod.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
 set -eu
 
-helm upgrade metrics . -n metrics --values "./values/prod.yaml" --install --create-namespace --atomic
+cd "$(dirname "${BASH_SOURCE[0]}")"
+
+helm upgrade metrics . -n metrics --values "./values/prod.yaml" --install --create-namespace --atomic $@
diff --git a/spartan/scripts/deploy_spartan.sh b/spartan/scripts/deploy_spartan.sh
index 5b00baba860..ee78d504cf3 100755
--- a/spartan/scripts/deploy_spartan.sh
+++ b/spartan/scripts/deploy_spartan.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -eux
+set -eu
 set -o pipefail
 
 TAG=$1
@@ -8,7 +8,7 @@ NAMESPACE=${3:-spartan}
 PROD=${4:-true}
 PROD_ARGS=""
 if [ "$PROD" = "true" ] ; then
-  PROD_ARGS="--set network.public=true --set telemetry.enabled=true --set telemetry.otelCollectorEndpoint: http://metrics-opentelemetry-collector.metrics:4318"
+  PROD_ARGS="--set network.public=true --set telemetry.enabled=true --set telemetry.otelCollectorEndpoint=http://metrics-opentelemetry-collector.metrics:4318"
 fi
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 

From 50b9c7ac70bc27d0383d8825505af98008823618 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Tue, 29 Oct 2024 13:12:44 +0000
Subject: [PATCH 17/39] revert

---
 docker-compose.yml | 132 +++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 127 insertions(+), 5 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 9faae02a3ad..18ca3c48335 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,118 @@
-name: otel
+name: aztec-devnet
 services:
+  pxe:
+    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
+    # need to run bb for proofs and bb is only built for x86
+    platform: linux/amd64
+    environment:
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      DEBUG: ${DEBUG:-aztec:*,-json-rpc:*,-aztec:circuits:artifact_hash,-aztec:randomness_singleton}
+      DEBUG_COLORS: 1
+      L1_CHAIN_ID: 31337
+      VERSION: 1
+      PXE_PROVER_ENABLED: ${PXE_PROVER_ENABLED:-1}
+      PXE_DATA_DIRECTORY: /var/lib/aztec/pxe
+      NODE_NO_WARNINGS: 1
+      AZTEC_PORT: 8080
+      MNEMONIC: "test test test test test test test test test test test junk"
+    entrypoint: >
+      sh -c '
+        export AZTEC_NODE_URL=$$(cat /var/run/secrets/aztec-node-url);
+        while ! curl --head --silent $AZTEC_NODE_URL > /dev/null; do echo "Node $$AZTEC_NODE_URL not up. Retrying after 1s"; sleep 1; done;
+        node /usr/src/yarn-project/aztec/dest/bin/index.js start --pxe
+      '
+    secrets:
+      - aztec-node-url
+    extra_hosts:
+      - "host.docker.internal:host-gateway"
+    volumes:
+      - aztec:/var/lib/aztec
+    ports:
+      - 8080:8080/tcp
+    profiles:
+      - pxe
+
+  node:
+    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
+    # need to run bb for proofs and bb is only built for x86
+    platform: linux/amd64
+    environment:
+      LOG_LEVEL: ${LOG_LEVEL:-info}
+      DEBUG: ${DEBUG:-aztec:*,-json-rpc:*,-aztec:circuits:artifact_hash,-aztec:randomness_singleton,-aztec:avm_simulator:*}
+      DEBUG_COLORS: 1
+      L1_CHAIN_ID: 31337
+      VERSION: 1
+      NODE_NO_WARNINGS: 1
+      PROVER_REAL_PROOFS: ${PROVER_REAL_PROOFS:-1}
+      DATA_DIRECTORY: /var/lib/aztec
+      PROVER_AGENT_ENABLED: 0
+      ROLLUP_CONTRACT_ADDRESS: "0x4ef7ca89e724a75ccca5dde7fb98075a388c2c91"
+      REGISTRY_CONTRACT_ADDRESS: "0x589a9634c1d00c62e47b3b7a790c8dc986b3d40d"
+      INBOX_CONTRACT_ADDRESS: "0x12d9b5effc69bf5c0c29c8258c6b6fa95a08de74"
+      OUTBOX_CONTRACT_ADDRESS: "0x3ec4b6c68a8c2ce4c78cdd465b3019b11a568d1d"
+      FEE_JUICE_CONTRACT_ADDRESS: "0x73c43b919973711e096bfc04c9d4b3be511ffc0b"
+      FEE_JUICE_PORTAL_CONTRACT_ADDRESS: "0xdf25b0a34dbee9f25518f7a4d63bab8b3bb3e496"
+      ETHEREUM_HOST:
+      P2P_TCP_LISTEN_ADDR: "0.0.0.0:9000"
+      P2P_UDP_LISTEN_ADDR: "0.0.0.0:9001"
+      P2P_TCP_ANNOUNCE_ADDR: ":9000"
+      P2P_UDP_ANNOUNCE_ADDR: ":9001"
+      P2P_QUERY_FOR_IP: true
+      P2P_ENABLED: true
+      PEER_ID_PRIVATE_KEY:
+      AZTEC_PORT: 8999
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT: ${OTEL_EXPORTER_OTLP_METRICS_ENDPOINT:-http://otel-collector:4318/v1/metrics}
+      OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: ${OTEL_EXPORTER_OTLP_TRACES_ENDPOINT:-http://otel-collector:4318/v1/traces}
+    secrets:
+      - ethereum-host
+      - p2p-boot-node
+    entrypoint: |
+      /bin/sh -c '
+        export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host)
+        export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node)
+
+        test -z "$$PEER_ID_PRIVATE_KEY" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d" " -f 3 | tee /var/lib/aztec/p2p-private-key || echo "Re-using existing P2P private key"
+        test -z "$$PEER_ID_PRIVATE_KEY" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key)
+
+        # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node
+        i=0
+        max=3
+        while ! curl --head --silent $$OTEL_EXPORTER_OTLP_METRICS_ENDPOINT > /dev/null; do
+          echo "OpenTelemetry collector not up. Retrying after 1s";
+          sleep 1;
+          i=$$((i+1));
+          if [ $$i -eq $$max ]; then
+            echo "OpenTelemetry collector at $$OTEL_EXPORTER_METRICS_ENDPOINT not up after $${max}s. Running without metrics";
+            unset OTEL_EXPORTER_METRICS_ENDPOINT;
+            unset OTEL_EXPORTER_TRACES_ENDPOINT;
+            break
+          fi;
+        done;
+
+        node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver
+      '
+    volumes:
+      - aztec:/var/lib/aztec
+    profiles:
+      - node
+    ports:
+      # the RPC endpoint - expose it on a different port to avoid conflicting with PXE
+      - 8999:8999/tcp
+      - 9000:9000/tcp
+      - 9001:9001/udp
+
+  cli:
+    image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70}
+    # run the same image as pxe
+    platform: linux/amd64
+    environment:
+      PXE_URL: http://pxe:8080
+      NODE_NO_WARNINGS: 1
+      SECRET_KEY:
+      ETHEREUM_HOST:
+    profiles:
+      - cli
+
   otel-collector:
     image: otel/opentelemetry-collector-contrib
     configs:
@@ -8,15 +121,12 @@ services:
     profiles:
       - metrics
     ports:
-      - 4317:4317
       - 4318:4318
 
   prometheus:
     image: prom/prometheus
     profiles:
       - metrics
-    ports:
-      - 2112:2112
     configs:
       - source: prometheus-config
         target: /etc/prometheus/prometheus.yml
@@ -41,6 +151,10 @@ services:
     profiles:
       - metrics
 
+volumes:
+  aztec:
+  grafana:
+
 configs:
   grafana-sources:
     content: |
@@ -96,4 +210,12 @@ configs:
           metrics:
             receivers: [otlp]
             processors: [batch]
-            exporters: [prometheus]
\ No newline at end of file
+            exporters: [prometheus]
+
+secrets:
+  aztec-node-url:
+    environment: AZTEC_NODE_URL
+  ethereum-host:
+    environment: ETHEREUM_HOST
+  p2p-boot-node:
+    environment: BOOTSTRAP_NODES

From 48067cf6b0c55a2f6b77e76bbd53c37696df4f23 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Tue, 29 Oct 2024 13:35:31 +0000
Subject: [PATCH 18/39] missing file

---
 .../src/otelLoggerProvider.ts                 | 83 +++++++++++++++++++
 1 file changed, 83 insertions(+)
 create mode 100644 yarn-project/telemetry-client/src/otelLoggerProvider.ts

diff --git a/yarn-project/telemetry-client/src/otelLoggerProvider.ts b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
new file mode 100644
index 00000000000..8a50754a137
--- /dev/null
+++ b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
@@ -0,0 +1,83 @@
+// Based on https://github.com/open-telemetry/opentelemetry-js-contrib/blob/9a20e15547669450987b2bb7cab193f17e04ebb7/packages/winston-transport/src/utils.ts
+// Not wanting to pull in the entire winston transport
+import { type LogData, type LogLevel, onLog } from '@aztec/foundation/log';
+
+import {
+  type LogAttributes as OtelLogAttributes,
+  type LogRecord as OtelLogRecord,
+  type Logger as OtelLogger,
+  SeverityNumber as OtelSeverity,
+} from '@opentelemetry/api-logs';
+import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
+import { type IResource } from '@opentelemetry/resources';
+import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
+import { SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
+
+const npmLevels: Record<string, number> = {
+  error: OtelSeverity.ERROR,
+  warn: OtelSeverity.WARN,
+  info: OtelSeverity.INFO,
+  http: OtelSeverity.DEBUG3,
+  verbose: OtelSeverity.DEBUG2,
+  debug: OtelSeverity.DEBUG,
+  silly: OtelSeverity.TRACE,
+};
+
+const sysLoglevels: Record<string, number> = {
+  emerg: OtelSeverity.FATAL3,
+  alert: OtelSeverity.FATAL2,
+  crit: OtelSeverity.FATAL,
+  error: OtelSeverity.ERROR,
+  warning: OtelSeverity.WARN,
+  notice: OtelSeverity.INFO2,
+  info: OtelSeverity.INFO,
+  debug: OtelSeverity.DEBUG,
+};
+
+const cliLevels: Record<string, number> = {
+  error: OtelSeverity.ERROR,
+  warn: OtelSeverity.WARN,
+  help: OtelSeverity.INFO3,
+  data: OtelSeverity.INFO2,
+  info: OtelSeverity.INFO,
+  debug: OtelSeverity.DEBUG,
+  prompt: OtelSeverity.TRACE4,
+  verbose: OtelSeverity.TRACE3,
+  input: OtelSeverity.TRACE2,
+  silly: OtelSeverity.TRACE,
+};
+
+function getOtelSeverity(level: string): OtelSeverity | undefined {
+  return npmLevels[level] ?? sysLoglevels[level] ?? cliLevels[level];
+}
+
+export function emitLogRecord(logger: OtelLogger, level: LogLevel, message: string, data?: LogData): void {
+  const logRecord: OtelLogRecord = {
+    severityNumber: getOtelSeverity(level),
+    severityText: level,
+    body: message,
+    attributes: (data || {}) as OtelLogAttributes,
+  };
+  (console as any).log("LOG", logRecord);
+  logger.emit(logRecord);
+}
+
+export function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: URL) {
+  const loggerProvider = new LoggerProvider();
+  if (!otelLogsUrl) {
+    // If no URL provided, return it disconnected.
+    return loggerProvider;
+  }
+  const logExporter = new OTLPLogExporter({
+    url: otelLogsUrl.toString(),
+  });
+  // Add a processor to the logger provider
+  loggerProvider.addLogRecordProcessor(new SimpleLogRecordProcessor(logExporter));
+
+  const loggers: { [module: string]: OtelLogger } = {};
+  onLog((level, module, message, data) => {
+    loggers[module] = loggers[module] || loggerProvider.getLogger(module, resource.attributes[SEMRESATTRS_SERVICE_VERSION] as string);
+    emitLogRecord(loggers[module], level, message, data);
+  });
+  return loggerProvider;
+}

From 9b7215de41235937a9d47b635effd2416cd49ad5 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Tue, 29 Oct 2024 13:35:40 +0000
Subject: [PATCH 19/39] missing file

---
 scripts/run_native_testnet_with_metrics.sh | 13 +++++++++++++
 1 file changed, 13 insertions(+)
 create mode 100755 scripts/run_native_testnet_with_metrics.sh

diff --git a/scripts/run_native_testnet_with_metrics.sh b/scripts/run_native_testnet_with_metrics.sh
new file mode 100755
index 00000000000..ff3a3dd00ee
--- /dev/null
+++ b/scripts/run_native_testnet_with_metrics.sh
@@ -0,0 +1,13 @@
+#!/bin/bash
+set -eu
+
+echo "Trying to port forward. NOTE: Must be using a k8s context with metrics chart installed"
+kubectl port-forward svc/metrics-opentelemetry-collector 4318:4318 -n metrics &
+
+export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://127.0.0.1:4318/v1/metrics
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/trace
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:4318/v1/logs
+
+# re-enter script dir
+cd $(dirname "${BASH_SOURCE[0]}")
+./run_native_testnet.sh $@
\ No newline at end of file

From 7efb4693ac06c84d933569ca93637fc513d22e71 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Tue, 29 Oct 2024 18:03:49 +0000
Subject: [PATCH 20/39] metrics in native testnet script

---
 .github/workflows/ci.yml                      | 14 ++++++++++---
 LICENSE                                       |  6 +++---
 scripts/run_native_testnet_with_metrics.sh    | 21 ++++++++++++++-----
 spartan/metrics/install.sh                    |  2 +-
 spartan/metrics/values.yaml                   |  3 ---
 spartan/scripts/test_spartan.sh               |  2 +-
 yarn-project/Earthfile                        |  3 ++-
 .../src/otelLoggerProvider.ts                 |  2 +-
 8 files changed, 35 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 13c81faeb90..6afc7e74e05 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -643,7 +643,15 @@ jobs:
       max-parallel: 1
       fail-fast: false
       matrix:
-        test: [test-transfer.sh, test-4epochs.sh]
+        scenario:
+          - validators: 3
+            test: test-transfer.sh
+          - validators: 3
+            test: test-4epochs.sh}
+          - validators: 16
+            test: test-4epochs.sh}
+          - validators: 48
+            test: test-4epochs.sh}
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -653,8 +661,8 @@ jobs:
       - name: "Setup and Local Network Tests"
         timeout-minutes: 60
         # Only allow transfer test to run on every (non-network-all) PR
-        if: matrix.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
-        run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }}
+        if: matrix.scenario.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
+        run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.scenario.test }} --validators=${{ matrix.scenario.validators }}
 
   # note: proving disabled
   kind-network-test:
diff --git a/LICENSE b/LICENSE
index f49a4e16e68..8dada3edaf5 100644
--- a/LICENSE
+++ b/LICENSE
@@ -178,7 +178,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
+      boilerplate notice, with the fields enclosed by brackets "{}"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright {yyyy} {name of copyright owner}
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
\ No newline at end of file
+   limitations under the License.
diff --git a/scripts/run_native_testnet_with_metrics.sh b/scripts/run_native_testnet_with_metrics.sh
index ff3a3dd00ee..f6b7cb05c0c 100755
--- a/scripts/run_native_testnet_with_metrics.sh
+++ b/scripts/run_native_testnet_with_metrics.sh
@@ -1,12 +1,23 @@
 #!/bin/bash
 set -eu
 
-echo "Trying to port forward. NOTE: Must be using a k8s context with metrics chart installed"
-kubectl port-forward svc/metrics-opentelemetry-collector 4318:4318 -n metrics &
+NAMESPACE=${1:-staging}
 
-export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://127.0.0.1:4318/v1/metrics
-export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://127.0.0.1:4318/v1/trace
-export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:4318/v1/logs
+echo "Trying to port forward. NOTE: Must be using a production k8s context with metrics chart."
+
+# Helper function to get load balancer URL based on namespace and service name
+function get_load_balancer_url() {
+  local namespace=$1
+  local service_name=$2
+  kubectl get svc -n $namespace -o jsonpath="{.items[?(@.metadata.name=='$service_name')].status.loadBalancer.ingress[0].hostname}"
+}
+
+# Fetch the service URLs based on the namespace for injection in the test-transfer.sh
+OTEL_URL=http://$(get_load_balancer_url metrics metrics-opentelemetry-collector):4318
+
+export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_URL/v1/metrics
+export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_URL/v1/trace
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_URL/v1/logs
 
 # re-enter script dir
 cd $(dirname "${BASH_SOURCE[0]}")
diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh
index 1f163218699..aacd541d3d5 100755
--- a/spartan/metrics/install.sh
+++ b/spartan/metrics/install.sh
@@ -7,4 +7,4 @@ helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm dependency update
-helm template metrics . -n metrics  #--install --create-namespace --atomic
+helm upgrade metrics . -n metrics  --install --create-namespace --atomic
diff --git a/spartan/metrics/values.yaml b/spartan/metrics/values.yaml
index 9b1d9139273..9e33411896c 100644
--- a/spartan/metrics/values.yaml
+++ b/spartan/metrics/values.yaml
@@ -28,9 +28,6 @@ opentelemetry-collector:
       protocol: TCP
 
   presets:
-    logsCollection:
-      enabled: true
-      includeCollectorLogs: true
     kubernetesAttributes:
       enabled: true
   config:
diff --git a/spartan/scripts/test_spartan.sh b/spartan/scripts/test_spartan.sh
index f362a6dbcbf..7154aeb01de 100755
--- a/spartan/scripts/test_spartan.sh
+++ b/spartan/scripts/test_spartan.sh
@@ -34,4 +34,4 @@ echo "ETHEREUM_HOST: $ETHEREUM_HOST"
 
 # hack to ensure L2 contracts are considered deployed
 touch $SCRIPT_DIR/../../yarn-project/end-to-end/scripts/native-network/state/l2-contracts.env
-bash -x $SCRIPT_DIR/../../yarn-project/end-to-end/scripts/native-network/test-transfer.sh
+bash -x $SCRIPT_DIR/../../yarn-project/end-to-end/scripts/native-network/test-4epochs.sh
diff --git a/yarn-project/Earthfile b/yarn-project/Earthfile
index 61415b0c0db..bb756674d48 100644
--- a/yarn-project/Earthfile
+++ b/yarn-project/Earthfile
@@ -283,6 +283,7 @@ prover-client-test:
 # Running this inside the main builder as the point is not to run this through dockerization.
 network-test:
     ARG test=./test-transfer.sh
+    ARG validators=3
     FROM +build
     WORKDIR /usr/src/
     # Bare minimum git setup to run 'git rev-parse --show-toplevel'
@@ -299,7 +300,7 @@ network-test:
         ./ethereum.sh \
         "./prover-node.sh 8078 false" \
         ./pxe.sh \
-        "./validators.sh 3"
+        "./validators.sh $validators"
 
 publish-npm:
     FROM +build
diff --git a/yarn-project/telemetry-client/src/otelLoggerProvider.ts b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
index 8a50754a137..56195e64a02 100644
--- a/yarn-project/telemetry-client/src/otelLoggerProvider.ts
+++ b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
@@ -69,7 +69,7 @@ export function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: UR
     return loggerProvider;
   }
   const logExporter = new OTLPLogExporter({
-    url: otelLogsUrl.toString(),
+    url: otelLogsUrl.href,
   });
   // Add a processor to the logger provider
   loggerProvider.addLogRecordProcessor(new SimpleLogRecordProcessor(logExporter));

From 5fb09732af0854efbaa270036181899e6def7896 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Tue, 29 Oct 2024 23:48:57 +0000
Subject: [PATCH 21/39] telemetry for winston

---
 scripts/run_native_testnet_with_metrics.sh    |  2 +-
 yarn-project/aztec/package.json               |  1 +
 yarn-project/aztec/src/logging.ts             |  2 +
 yarn-project/telemetry-client/src/otel.ts     |  2 +-
 .../src/otelLoggerProvider.ts                 | 72 ++-----------------
 yarn-project/yarn.lock                        |  1 +
 6 files changed, 10 insertions(+), 70 deletions(-)

diff --git a/scripts/run_native_testnet_with_metrics.sh b/scripts/run_native_testnet_with_metrics.sh
index f6b7cb05c0c..51629df2a33 100755
--- a/scripts/run_native_testnet_with_metrics.sh
+++ b/scripts/run_native_testnet_with_metrics.sh
@@ -17,7 +17,7 @@ OTEL_URL=http://$(get_load_balancer_url metrics metrics-opentelemetry-collector)
 
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_URL/v1/metrics
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_URL/v1/trace
-export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_URL/v1/logs
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:5000
 
 # re-enter script dir
 cd $(dirname "${BASH_SOURCE[0]}")
diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json
index b30a23ae584..ee1ceb90931 100644
--- a/yarn-project/aztec/package.json
+++ b/yarn-project/aztec/package.json
@@ -55,6 +55,7 @@
     "@aztec/telemetry-client": "workspace:^",
     "@aztec/txe": "workspace:^",
     "@aztec/types": "workspace:^",
+    "@opentelemetry/winston-transport": "^0.7.0",
     "@types/chalk": "^2.2.0",
     "abitype": "^0.8.11",
     "chalk": "^5.3.0",
diff --git a/yarn-project/aztec/src/logging.ts b/yarn-project/aztec/src/logging.ts
index 9c71fa54205..da1df4900a4 100644
--- a/yarn-project/aztec/src/logging.ts
+++ b/yarn-project/aztec/src/logging.ts
@@ -4,6 +4,7 @@ import * as path from 'path';
 import * as process from 'process';
 import * as winston from 'winston';
 import DailyRotateFile from 'winston-daily-rotate-file';
+import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
 
 const { format } = winston;
 const CURRENT_LOG_FILE_NAME = 'aztec.debug.log';
@@ -38,6 +39,7 @@ function createWinstonJsonStdoutLogger() {
       new winston.transports.Console({
         format: format.combine(format.timestamp(), format.json()),
       }),
+      new OpenTelemetryTransportV3()
     ],
   });
 }
diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts
index 61f2444d3e9..422fee32efe 100644
--- a/yarn-project/telemetry-client/src/otel.ts
+++ b/yarn-project/telemetry-client/src/otel.ts
@@ -120,7 +120,7 @@ export class OpenTelemetryClient implements TelemetryClient {
         }),
       ],
     });
-    const loggerProvider = registerOtelLoggerProvider(resource, logsCollector);
+    const loggerProvider = registerOtelLoggerProvider(logsCollector);
 
     const service = new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
     service.start();
diff --git a/yarn-project/telemetry-client/src/otelLoggerProvider.ts b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
index 56195e64a02..d9ed41bb97c 100644
--- a/yarn-project/telemetry-client/src/otelLoggerProvider.ts
+++ b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
@@ -1,68 +1,8 @@
-// Based on https://github.com/open-telemetry/opentelemetry-js-contrib/blob/9a20e15547669450987b2bb7cab193f17e04ebb7/packages/winston-transport/src/utils.ts
-// Not wanting to pull in the entire winston transport
-import { type LogData, type LogLevel, onLog } from '@aztec/foundation/log';
-
-import {
-  type LogAttributes as OtelLogAttributes,
-  type LogRecord as OtelLogRecord,
-  type Logger as OtelLogger,
-  SeverityNumber as OtelSeverity,
-} from '@opentelemetry/api-logs';
 import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
-import { type IResource } from '@opentelemetry/resources';
 import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
-import { SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
-
-const npmLevels: Record<string, number> = {
-  error: OtelSeverity.ERROR,
-  warn: OtelSeverity.WARN,
-  info: OtelSeverity.INFO,
-  http: OtelSeverity.DEBUG3,
-  verbose: OtelSeverity.DEBUG2,
-  debug: OtelSeverity.DEBUG,
-  silly: OtelSeverity.TRACE,
-};
-
-const sysLoglevels: Record<string, number> = {
-  emerg: OtelSeverity.FATAL3,
-  alert: OtelSeverity.FATAL2,
-  crit: OtelSeverity.FATAL,
-  error: OtelSeverity.ERROR,
-  warning: OtelSeverity.WARN,
-  notice: OtelSeverity.INFO2,
-  info: OtelSeverity.INFO,
-  debug: OtelSeverity.DEBUG,
-};
-
-const cliLevels: Record<string, number> = {
-  error: OtelSeverity.ERROR,
-  warn: OtelSeverity.WARN,
-  help: OtelSeverity.INFO3,
-  data: OtelSeverity.INFO2,
-  info: OtelSeverity.INFO,
-  debug: OtelSeverity.DEBUG,
-  prompt: OtelSeverity.TRACE4,
-  verbose: OtelSeverity.TRACE3,
-  input: OtelSeverity.TRACE2,
-  silly: OtelSeverity.TRACE,
-};
+import { logs as otelLogs } from '@opentelemetry/api-logs';
 
-function getOtelSeverity(level: string): OtelSeverity | undefined {
-  return npmLevels[level] ?? sysLoglevels[level] ?? cliLevels[level];
-}
-
-export function emitLogRecord(logger: OtelLogger, level: LogLevel, message: string, data?: LogData): void {
-  const logRecord: OtelLogRecord = {
-    severityNumber: getOtelSeverity(level),
-    severityText: level,
-    body: message,
-    attributes: (data || {}) as OtelLogAttributes,
-  };
-  (console as any).log("LOG", logRecord);
-  logger.emit(logRecord);
-}
-
-export function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: URL) {
+export function registerOtelLoggerProvider(otelLogsUrl?: URL) {
   const loggerProvider = new LoggerProvider();
   if (!otelLogsUrl) {
     // If no URL provided, return it disconnected.
@@ -73,11 +13,7 @@ export function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: UR
   });
   // Add a processor to the logger provider
   loggerProvider.addLogRecordProcessor(new SimpleLogRecordProcessor(logExporter));
+  otelLogs.setGlobalLoggerProvider(loggerProvider);
 
-  const loggers: { [module: string]: OtelLogger } = {};
-  onLog((level, module, message, data) => {
-    loggers[module] = loggers[module] || loggerProvider.getLogger(module, resource.attributes[SEMRESATTRS_SERVICE_VERSION] as string);
-    emitLogRecord(loggers[module], level, message, data);
-  });
   return loggerProvider;
-}
+}
\ No newline at end of file
diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock
index b2f5cf6cf8e..fa6917c056e 100644
--- a/yarn-project/yarn.lock
+++ b/yarn-project/yarn.lock
@@ -260,6 +260,7 @@ __metadata:
     "@aztec/txe": "workspace:^"
     "@aztec/types": "workspace:^"
     "@jest/globals": ^29.5.0
+    "@opentelemetry/winston-transport": ^0.7.0
     "@types/chalk": ^2.2.0
     "@types/jest": ^29.5.0
     "@types/koa": ^2.13.6

From 6cb4b463d3f5474d1c831d42a03a3272d548a383 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 00:39:51 +0000
Subject: [PATCH 22/39] fix metrics

---
 scripts/run_native_testnet_with_metrics.sh    |  3 +-
 spartan/scripts/post_deploy_spartan.sh        | 41 +++++++++++++++++++
 .../scripts/native-network/transaction-bot.sh |  1 -
 yarn-project/telemetry-client/src/start.ts    | 12 +++---
 4 files changed, 50 insertions(+), 7 deletions(-)
 create mode 100755 spartan/scripts/post_deploy_spartan.sh

diff --git a/scripts/run_native_testnet_with_metrics.sh b/scripts/run_native_testnet_with_metrics.sh
index 51629df2a33..24cb581cace 100755
--- a/scripts/run_native_testnet_with_metrics.sh
+++ b/scripts/run_native_testnet_with_metrics.sh
@@ -17,7 +17,8 @@ OTEL_URL=http://$(get_load_balancer_url metrics metrics-opentelemetry-collector)
 
 export OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=$OTEL_URL/v1/metrics
 export OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=$OTEL_URL/v1/trace
-export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=http://127.0.0.1:5000
+export OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=$OTEL_URL/v1/logs
+export LOG_JSON=1
 
 # re-enter script dir
 cd $(dirname "${BASH_SOURCE[0]}")
diff --git a/spartan/scripts/post_deploy_spartan.sh b/spartan/scripts/post_deploy_spartan.sh
new file mode 100755
index 00000000000..4a0ff06c2c9
--- /dev/null
+++ b/spartan/scripts/post_deploy_spartan.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Targets a running cluster and tests 5 slots worth of transfers against it.
+set -eu
+set -o pipefail
+
+# Test a deployed cluster
+
+NAMESPACE=${1:-spartan}
+TAG=${2:-latest}
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if [ -z "$NAMESPACE" ]; then
+  echo "Usage: $0 (optional: <namespace>)"
+  echo "Example: $0 devnet"
+  exit 1
+fi
+
+# Helper function to get load balancer URL based on namespace and service name
+function get_load_balancer_url() {
+  local namespace=$1
+  local service_name=$2
+  kubectl get svc -n $namespace -o jsonpath="{.items[?(@.metadata.name=='$service_name')].status.loadBalancer.ingress[0].hostname}"
+}
+
+# Fetch the service URLs based on the namespace for injection in the test-transfer.sh
+export BOOTNODE_URL=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-boot-node-lb-tcp"):8080
+export PXE_URL=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-pxe-lb"):8080
+export ETHEREUM_HOST=http://$(get_load_balancer_url $NAMESPACE "$NAMESPACE-aztec-network-ethereum-lb"):8545
+
+echo "BOOTNODE_URL: $BOOTNODE_URL"
+echo "PXE_URL: $PXE_URL"
+echo "ETHEREUM_HOST: $ETHEREUM_HOST"
+
+echo "Bootstrapping contracts for test network. NOTE: This took one hour last run."
+# hack to ensure L2 contracts are considered deployed
+docker run aztecprotocol/aztec:$TAG bootstrap-network \
+  --rpc-url $BOOTNODE_URL \
+  --l1-rpc-url $ETHEREUM_HOST \
+  --l1-chain-id 31337 \
+  --l1-private-key 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \
+  --json | tee ./basic_contracts.json
diff --git a/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh b/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh
index f95306a2182..722bfdcf0ce 100755
--- a/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh
+++ b/yarn-project/end-to-end/scripts/native-network/transaction-bot.sh
@@ -29,7 +29,6 @@ echo "Done waiting."
 # Set environment variables
 export ETHEREUM_HOST="http://127.0.0.1:8545"
 export AZTEC_NODE_URL="http://127.0.0.1:8080"
-export LOG_JSON="1"
 export LOG_LEVEL=${LOG_LEVEL:-"debug"}
 export DEBUG="aztec:*,-aztec:avm_simulator*,-aztec:libp2p_service*,-aztec:circuits:artifact_hash,-json-rpc*,-aztec:l2_block_stream,-aztec:world-state:*"
 export BOT_PRIVATE_KEY="0xcafe"
diff --git a/yarn-project/telemetry-client/src/start.ts b/yarn-project/telemetry-client/src/start.ts
index 39ee91b1bdf..c58692bb568 100644
--- a/yarn-project/telemetry-client/src/start.ts
+++ b/yarn-project/telemetry-client/src/start.ts
@@ -1,22 +1,24 @@
 import { createDebugLogger } from '@aztec/foundation/log';
 
-
-
 import { type TelemetryClientConfig } from './config.js';
 import { NoopTelemetryClient } from './noop.js';
 import { OpenTelemetryClient } from './otel.js';
 import { type TelemetryClient } from './telemetry.js';
 
-
 export * from './config.js';
 
 export async function createAndStartTelemetryClient(config: TelemetryClientConfig): Promise<TelemetryClient> {
   const log = createDebugLogger('aztec:telemetry-client');
   if (config.metricsCollectorUrl) {
     log.info('Using OpenTelemetry client');
-    return await OpenTelemetryClient.createAndStart(config.metricsCollectorUrl, config.tracesCollectorUrl, config.logsCollectorUrl, log);
+    return await OpenTelemetryClient.createAndStart(
+      config.metricsCollectorUrl,
+      config.tracesCollectorUrl,
+      config.logsCollectorUrl,
+      log,
+    );
   } else {
     log.info('Using NoopTelemetryClient');
     return new NoopTelemetryClient();
   }
-}
\ No newline at end of file
+}

From fc299f35ff6bf5685f0ef4d49dc74e57d918eab8 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Tue, 29 Oct 2024 20:41:56 -0400
Subject: [PATCH 23/39] Update install.sh

---
 spartan/metrics/install.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/spartan/metrics/install.sh b/spartan/metrics/install.sh
index aacd541d3d5..2712e900756 100755
--- a/spartan/metrics/install.sh
+++ b/spartan/metrics/install.sh
@@ -7,4 +7,4 @@ helm repo add open-telemetry https://open-telemetry.github.io/opentelemetry-helm
 helm repo add grafana https://grafana.github.io/helm-charts
 helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
 helm dependency update
-helm upgrade metrics . -n metrics  --install --create-namespace --atomic
+helm upgrade metrics . -n metrics --install --create-namespace --atomic

From 587a58efbe606ebeb1e1472f89208613562c13b9 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 00:42:58 +0000
Subject: [PATCH 24/39] updates

---
 metrics/otel/otel.yml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/metrics/otel/otel.yml b/metrics/otel/otel.yml
index 675e6579132..b9e70117034 100644
--- a/metrics/otel/otel.yml
+++ b/metrics/otel/otel.yml
@@ -27,6 +27,3 @@ service:
       receivers: [otlp]
       processors: [batch]
       exporters: [prometheus, debug]
-    logs:
-      receivers: [otlp]
-      exporters: [logging]

From ef41a4d22e11374ee5447035198e7d299b017986 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 00:44:19 +0000
Subject: [PATCH 25/39] rev

---
 LICENSE | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/LICENSE b/LICENSE
index 8dada3edaf5..f49a4e16e68 100644
--- a/LICENSE
+++ b/LICENSE
@@ -178,7 +178,7 @@
    APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
+      boilerplate notice, with the fields enclosed by brackets "[]"
       replaced with your own identifying information. (Don't include
       the brackets!)  The text should be enclosed in the appropriate
       comment syntax for the file format. We also recommend that a
@@ -186,7 +186,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright {yyyy} {name of copyright owner}
+   Copyright [yyyy] [name of copyright owner]
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
@@ -198,4 +198,4 @@
    distributed under the License is distributed on an "AS IS" BASIS,
    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    See the License for the specific language governing permissions and
-   limitations under the License.
+   limitations under the License.
\ No newline at end of file

From 0b9baed9b4aa86f615cf8a92d00e6304387ce347 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 00:44:43 +0000
Subject: [PATCH 26/39] -

---
 .github/workflows/ci.yml | 14 +++-----------
 1 file changed, 3 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6afc7e74e05..13c81faeb90 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -643,15 +643,7 @@ jobs:
       max-parallel: 1
       fail-fast: false
       matrix:
-        scenario:
-          - validators: 3
-            test: test-transfer.sh
-          - validators: 3
-            test: test-4epochs.sh}
-          - validators: 16
-            test: test-4epochs.sh}
-          - validators: 48
-            test: test-4epochs.sh}
+        test: [test-transfer.sh, test-4epochs.sh]
     steps:
       - uses: actions/checkout@v4
         with: { ref: "${{ env.GIT_COMMIT }}" }
@@ -661,8 +653,8 @@ jobs:
       - name: "Setup and Local Network Tests"
         timeout-minutes: 60
         # Only allow transfer test to run on every (non-network-all) PR
-        if: matrix.scenario.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
-        run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.scenario.test }} --validators=${{ matrix.scenario.validators }}
+        if: matrix.test == 'test-transfer.sh' || github.ref_name == 'master' || contains(github.event.pull_request.labels.*.name, 'network-all')
+        run: sudo shutdown -P 60 ; earthly-ci --no-output ./yarn-project/+network-test --test=./${{ matrix.test }}
 
   # note: proving disabled
   kind-network-test:

From 734c2053cf501d9dad2700151dc63254c1cf6164 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 00:54:09 +0000
Subject: [PATCH 27/39] formatting

---
 yarn-project/aztec/src/logging.ts                       | 4 ++--
 yarn-project/telemetry-client/src/config.ts             | 3 +--
 yarn-project/telemetry-client/src/otelLoggerProvider.ts | 4 ++--
 yarn-project/validator-client/src/validator.ts          | 4 +++-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/yarn-project/aztec/src/logging.ts b/yarn-project/aztec/src/logging.ts
index da1df4900a4..a7deed55ae5 100644
--- a/yarn-project/aztec/src/logging.ts
+++ b/yarn-project/aztec/src/logging.ts
@@ -1,10 +1,10 @@
 import { currentLevel, onLog, setLevel } from '@aztec/foundation/log';
 
+import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
 import * as path from 'path';
 import * as process from 'process';
 import * as winston from 'winston';
 import DailyRotateFile from 'winston-daily-rotate-file';
-import { OpenTelemetryTransportV3 } from '@opentelemetry/winston-transport';
 
 const { format } = winston;
 const CURRENT_LOG_FILE_NAME = 'aztec.debug.log';
@@ -39,7 +39,7 @@ function createWinstonJsonStdoutLogger() {
       new winston.transports.Console({
         format: format.combine(format.timestamp(), format.json()),
       }),
-      new OpenTelemetryTransportV3()
+      new OpenTelemetryTransportV3(),
     ],
   });
 }
diff --git a/yarn-project/telemetry-client/src/config.ts b/yarn-project/telemetry-client/src/config.ts
index 1745b5a4e53..c48a9be6bc4 100644
--- a/yarn-project/telemetry-client/src/config.ts
+++ b/yarn-project/telemetry-client/src/config.ts
@@ -1,6 +1,5 @@
 import { type ConfigMappingsType, getConfigFromMappings } from '@aztec/foundation/config';
 
-
 export interface TelemetryClientConfig {
   metricsCollectorUrl?: URL;
   tracesCollectorUrl?: URL;
@@ -39,4 +38,4 @@ export const telemetryClientConfigMappings: ConfigMappingsType<TelemetryClientCo
 
 export function getConfigEnvVars(): TelemetryClientConfig {
   return getConfigFromMappings<TelemetryClientConfig>(telemetryClientConfigMappings);
-}
\ No newline at end of file
+}
diff --git a/yarn-project/telemetry-client/src/otelLoggerProvider.ts b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
index d9ed41bb97c..0a0744c3501 100644
--- a/yarn-project/telemetry-client/src/otelLoggerProvider.ts
+++ b/yarn-project/telemetry-client/src/otelLoggerProvider.ts
@@ -1,6 +1,6 @@
+import { logs as otelLogs } from '@opentelemetry/api-logs';
 import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
 import { LoggerProvider, SimpleLogRecordProcessor } from '@opentelemetry/sdk-logs';
-import { logs as otelLogs } from '@opentelemetry/api-logs';
 
 export function registerOtelLoggerProvider(otelLogsUrl?: URL) {
   const loggerProvider = new LoggerProvider();
@@ -16,4 +16,4 @@ export function registerOtelLoggerProvider(otelLogsUrl?: URL) {
   otelLogs.setGlobalLoggerProvider(loggerProvider);
 
   return loggerProvider;
-}
\ No newline at end of file
+}
diff --git a/yarn-project/validator-client/src/validator.ts b/yarn-project/validator-client/src/validator.ts
index 7c11d474f6d..c7c70d950f0 100644
--- a/yarn-project/validator-client/src/validator.ts
+++ b/yarn-project/validator-client/src/validator.ts
@@ -40,7 +40,9 @@ export class ValidatorClient extends WithTracer implements Validator {
     private attestationPoolingIntervalMs: number,
     private attestationWaitTimeoutMs: number,
     telemetry: TelemetryClient,
-    private log = attachedFixedDataToLogger(createDebugLogger('aztec:validator'), { validatorAddress: keyStore.getAddress().toString() }),
+    private log = attachedFixedDataToLogger(createDebugLogger('aztec:validator'), {
+      validatorAddress: keyStore.getAddress().toString(),
+    }),
   ) {
     // Instantiate tracer
     super(telemetry, 'Validator');

From 90bf69d069c2ba9a34f3f1c5d9f458d23bc1d2a4 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Tue, 29 Oct 2024 20:57:25 -0400
Subject: [PATCH 28/39] cleaner

---
 yarn-project/foundation/src/log/logger.ts | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/yarn-project/foundation/src/log/logger.ts b/yarn-project/foundation/src/log/logger.ts
index ed4305687c8..b78ccdef540 100644
--- a/yarn-project/foundation/src/log/logger.ts
+++ b/yarn-project/foundation/src/log/logger.ts
@@ -101,20 +101,10 @@ export function attachedFixedDataToLogger(debugLogger: DebugLogger, fixedLogData
   const logger = {
     // Silent log level does nothing.
     silent: () => {},
-
-    // Error log level includes a message, error details, and additional data.
     error: (msg: string, err?: unknown, data?: LogData) => debugLogger.error(fmtErr(msg, err), attach(data)),
-
-    // Warning log level includes a message and additional data.
     warn: (msg: string, data?: LogData) => debugLogger.warn(msg, attach(data)),
-
-    // Info log level includes a message and additional data.
     info: (msg: string, data?: LogData) => debugLogger.info(msg, attach(data)),
-
-    // Verbose log level includes a message and additional data.
     verbose: (msg: string, data?: LogData) => debugLogger.verbose(msg, attach(data)),
-
-    // Debug log level includes a message and additional data.
     debug: (msg: string, data?: LogData) => debugLogger.debug(msg, attach(data)),
   };
 

From c9004764a5b182e3649e6b35154c506090d51d78 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Tue, 29 Oct 2024 20:57:51 -0400
Subject: [PATCH 29/39] Update logger.ts

---
 yarn-project/foundation/src/log/logger.ts | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/yarn-project/foundation/src/log/logger.ts b/yarn-project/foundation/src/log/logger.ts
index b78ccdef540..1730d83fb58 100644
--- a/yarn-project/foundation/src/log/logger.ts
+++ b/yarn-project/foundation/src/log/logger.ts
@@ -96,7 +96,6 @@ export function createDebugLogger(name: string): DebugLogger {
 export function attachedFixedDataToLogger(debugLogger: DebugLogger, fixedLogData: LogData): DebugLogger {
   // Helper function to merge fixed data with additional data passed to log entries.
   const attach = (data?: LogData) => ({ ...fixedLogData, ...data });
-
   // Define the logger with all the necessary log level methods.
   const logger = {
     // Silent log level does nothing.
@@ -107,9 +106,6 @@ export function attachedFixedDataToLogger(debugLogger: DebugLogger, fixedLogData
     verbose: (msg: string, data?: LogData) => debugLogger.verbose(msg, attach(data)),
     debug: (msg: string, data?: LogData) => debugLogger.debug(msg, attach(data)),
   };
-
-  // Return the logger as a function with `debug` as the default log level.
-  // The function will act as a shortcut to the `debug` log level.
   return Object.assign((msg: string, data?: LogData) => debugLogger.debug(msg, attach(data)), logger);
 }
 

From cfad19f498cfdca5a96e9d835fa871f0ffb50280 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 20:17:39 +0000
Subject: [PATCH 30/39] fix

---
 scripts/ci/get_e2e_jobs.sh                    |  40 ++--
 .../aztec-network/templates/boot-node.yaml    |  17 +-
 .../aztec-network/templates/validator.yaml    |  21 ++
 spartan/aztec-network/values.yaml             |  16 ++
 yarn-project/aztec/src/cli/cmds/start_node.ts |   3 +-
 .../end-to-end/scripts/e2e_test_config.yml    | 222 +++++++++---------
 6 files changed, 183 insertions(+), 136 deletions(-)

diff --git a/scripts/ci/get_e2e_jobs.sh b/scripts/ci/get_e2e_jobs.sh
index cc8468935ba..c85a7244a22 100755
--- a/scripts/ci/get_e2e_jobs.sh
+++ b/scripts/ci/get_e2e_jobs.sh
@@ -16,26 +16,26 @@ get_test_names() {
 full_list=$(get_test_names)
 
 # Define the jobs that will run on every PR
-allow_list=(
-  "e2e_2_pxes"
-  "e2e_authwit"
-  "e2e_avm_simulator"
-  "e2e_block_building"
-  "e2e_cross_chain_messaging"
-  "e2e_deploy_contract"
-  "e2e_fees"
-  "e2e_fees_gas_estimation"
-  "e2e_fees_private_payments"
-  "e2e_max_block_number"
-  "e2e_nested_contract"
-  "e2e_ordering"
-  "e2e_static_calls"
-  "integration_l1_publisher"
-  "e2e_cheat_codes"
-  "e2e_prover_fake_proofs"
-  "e2e_prover_coordination"
-  "e2e_lending_contract"
-  "kind_network_smoke"
+allow-list=(
+  "e2e-2-pxes"
+  "e2e-authwit"
+  "e2e-avm-simulator"
+  "e2e-block-building"
+  "e2e-cross-chain-messaging"
+  "e2e-deploy-contract"
+  "e2e-fees"
+  "e2e-fees-gas-estimation"
+  "e2e-fees-private-payments"
+  "e2e-max-block-number"
+  "e2e-nested-contract"
+  "e2e-ordering"
+  "e2e-static-calls"
+  "integration-l1-publisher"
+  "e2e-cheat-codes"
+  "e2e-prover-fake-proofs"
+  "e2e-prover-coordination"
+  "e2e-lending-contract"
+  "kind-network-smoke"
 )
 
 # Add labels from input to the allow_list
diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 59f99ba4a7f..1bd2fb4c434 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -69,12 +69,23 @@ spec:
               "-c",
               "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
+          startupProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
+            initialDelaySeconds: {{ .Values.bootNode.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.bootNode.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.bootNode.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.bootNode.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.bootNode.readinessProbe.failureThreshold }}
           livenessProbe:
             exec:
               command:
-              - /bin/sh
-              - -c
-              - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
+                - /bin/sh
+                - -c
+                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
             initialDelaySeconds: 30
             periodSeconds: 5
             timeoutSeconds: 30
diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index 963e778d745..af2aed067cd 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -78,6 +78,27 @@ spec:
             - "/bin/bash"
             - "-c"
             - "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer"
+          startupProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
+            initialDelaySeconds: {{ .Values.validator.readinessProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.validator.readinessProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.validator.readinessProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.validator.readinessProbe.successThreshold }}
+            failureThreshold: {{ .Values.validator.readinessProbe.failureThreshold }}
+          livenessProbe:
+            exec:
+              command:
+                - /bin/sh
+                - -c
+                - curl -fSs http://127.0.0.1:{{ .Values.validator.service.nodePort }}/status
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 30
+            failureThreshold: 3
           volumeMounts:
             - name: shared-volume
               mountPath: /shared
diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml
index 8905fc155eb..8362fb3adac 100644
--- a/spartan/aztec-network/values.yaml
+++ b/spartan/aztec-network/values.yaml
@@ -44,6 +44,14 @@ bootNode:
       memory: "2Gi"
       cpu: "200m"
   deployContracts: true # Set to false to use manual contract addresses
+  startupProbe:
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    successThreshold: 1
+    # Only if we fail for 20 minutes straight do we call it botched
+    # This gives enough time to sync
+    failureThreshold: 120
   contracts:
     rollupAddress: ""
     registryAddress: ""
@@ -75,6 +83,14 @@ validator:
     disabled: false
   p2p:
     enabled: "true"
+  startupProbe:
+    initialDelaySeconds: 5
+    periodSeconds: 10
+    timeoutSeconds: 5
+    successThreshold: 1
+    # Only if we fail for 20 minutes straight do we call it botched
+    # This gives enough time to sync
+    failureThreshold: 120
   resources:
     requests:
       memory: "2Gi"
diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts
index 2a0bf875fc7..880a6e67cba 100644
--- a/yarn-project/aztec/src/cli/cmds/start_node.ts
+++ b/yarn-project/aztec/src/cli/cmds/start_node.ts
@@ -18,8 +18,7 @@ export const startNode = async (
   options: any,
   signalHandlers: (() => Promise<void>)[],
   userLog: LogFn,
-  // ): Promise<ServerList> => {
-) => {
+  ): Promise<ServerList> => {
   // Services that will be started in a single multi-rpc server
   const services: ServerList = [];
 
diff --git a/yarn-project/end-to-end/scripts/e2e_test_config.yml b/yarn-project/end-to-end/scripts/e2e_test_config.yml
index 2f7c718c14c..e07540c7c81 100644
--- a/yarn-project/end-to-end/scripts/e2e_test_config.yml
+++ b/yarn-project/end-to-end/scripts/e2e_test_config.yml
@@ -1,126 +1,126 @@
 tests:
   base: {}
-  bench_prover:
+  bench-prover:
     env:
-      HARDWARE_CONCURRENCY: "32"
-      COMPOSE_FILE: "scripts/docker-compose-no-sandbox.yml"
-      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees"
-    command: "./scripts/e2e_compose_test.sh bench_prover"
-  bench_publish_rollup:
+      HARDWARE-CONCURRENCY: "32"
+      COMPOSE-FILE: "scripts/docker-compose-no-sandbox.yml"
+      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world-state,aztec:merkle-trees"
+    command: "./scripts/e2e-compose-test.sh bench-prover"
+  bench-publish-rollup:
     env:
-      HARDWARE_CONCURRENCY: "32"
-      COMPOSE_FILE: "scripts/docker-compose-no-sandbox.yml"
-      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees"
-    command: "./scripts/e2e_compose_test.sh bench_publish_rollup"
-  bench_tx_size:
+      HARDWARE-CONCURRENCY: "32"
+      COMPOSE-FILE: "scripts/docker-compose-no-sandbox.yml"
+      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world-state,aztec:merkle-trees"
+    command: "./scripts/e2e-compose-test.sh bench-publish-rollup"
+  bench-tx-size:
     env:
-      HARDWARE_CONCURRENCY: "32"
-      COMPOSE_FILE: "scripts/docker-compose-no-sandbox.yml"
-      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world_state,aztec:merkle_trees"
-    command: "./scripts/e2e_compose_test.sh bench_tx_size"
-  e2e_2_pxes: {}
-  e2e_account_contracts: {}
-  e2e_authwit: {}
-  e2e_avm_simulator: {}
-  e2e_blacklist_token_contract: {}
-  e2e_block_building: {}
-  e2e_bot: {}
-  e2e_aztec_js_browser:
-    use_compose: true
-  e2e_card_game: {}
-  e2e_cheat_codes: {}
-  e2e_cross_chain_messaging: {}
-  e2e_crowdfunding_and_claim: {}
-  e2e_deploy_contract: {}
-  e2e_devnet_smoke: {}
-  docs_examples:
-    use_compose: true
-  e2e_escrow_contract: {}
-  e2e_fees_account_init:
-    test_path: "e2e_fees/account_init.test.ts"
+      HARDWARE-CONCURRENCY: "32"
+      COMPOSE-FILE: "scripts/docker-compose-no-sandbox.yml"
+      DEBUG: "aztec:benchmarks:*,aztec:sequencer,aztec:sequencer:*,aztec:world-state,aztec:merkle-trees"
+    command: "./scripts/e2e-compose-test.sh bench-tx-size"
+  e2e-2-pxes: {}
+  e2e-account-contracts: {}
+  e2e-authwit: {}
+  e2e-avm-simulator: {}
+  e2e-blacklist-token-contract: {}
+  e2e-block-building: {}
+  e2e-bot: {}
+  e2e-aztec-js-browser:
+    use-compose: true
+  e2e-card-game: {}
+  e2e-cheat-codes: {}
+  e2e-cross-chain-messaging: {}
+  e2e-crowdfunding-and-claim: {}
+  e2e-deploy-contract: {}
+  e2e-devnet-smoke: {}
+  docs-examples:
+    use-compose: true
+  e2e-escrow-contract: {}
+  e2e-fees-account-init:
+    test-path: "e2e-fees/account-init.test.ts"
   # TODO(https://github.com/AztecProtocol/aztec-packages/issues/9488): reenable
-  # e2e_fees_dapp_subscription:
-  #   test_path: "e2e_fees/dapp_subscription.test.ts"
-  e2e_fees_failures:
-    test_path: "e2e_fees/failures.test.ts"
-  e2e_fees_fee_juice_payments:
-    test_path: "e2e_fees/fee_juice_payments.test.ts"
-  e2e_fees_gas_estimation:
-    test_path: "e2e_fees/gas_estimation.test.ts"
-  e2e_fees_private_payments:
-    test_path: "e2e_fees/private_payments.test.ts"
-  e2e_fees_private_refunds:
-    test_path: "e2e_fees/private_refunds.test.ts"
-  e2e_keys: {}
-  e2e_l1_with_wall_time: {}
-  e2e_lending_contract: {}
-  e2e_event_logs: {}
-  e2e_max_block_number: {}
-  e2e_multiple_accounts_1_enc_key: {}
-  e2e_nested_contract: {}
-  e2e_nft: {}
-  e2e_non_contract_account: {}
-  e2e_note_getter: {}
-  e2e_ordering: {}
-  e2e_outbox: {}
-  e2e_pending_note_hashes_contract: {}
-  e2e_private_voting_contract: {}
-  e2e_prover_coordination: {}
-  e2e_prover_fake_proofs:
-    test_path: "e2e_prover/full.test.ts"
+  # e2e-fees-dapp-subscription:
+  #   test-path: "e2e-fees/dapp-subscription.test.ts"
+  e2e-fees-failures:
+    test-path: "e2e-fees/failures.test.ts"
+  e2e-fees-fee-juice-payments:
+    test-path: "e2e-fees/fee-juice-payments.test.ts"
+  e2e-fees-gas-estimation:
+    test-path: "e2e-fees/gas-estimation.test.ts"
+  e2e-fees-private-payments:
+    test-path: "e2e-fees/private-payments.test.ts"
+  e2e-fees-private-refunds:
+    test-path: "e2e-fees/private-refunds.test.ts"
+  e2e-keys: {}
+  e2e-l1-with-wall-time: {}
+  e2e-lending-contract: {}
+  e2e-event-logs: {}
+  e2e-max-block-number: {}
+  e2e-multiple-accounts-1-enc-key: {}
+  e2e-nested-contract: {}
+  e2e-nft: {}
+  e2e-non-contract-account: {}
+  e2e-note-getter: {}
+  e2e-ordering: {}
+  e2e-outbox: {}
+  e2e-pending-note-hashes-contract: {}
+  e2e-private-voting-contract: {}
+  e2e-prover-coordination: {}
+  e2e-prover-fake-proofs:
+    test-path: "e2e-prover/full.test.ts"
     env:
-      FAKE_PROOFS: "1"
-  e2e_prover_full:
-    test_path: "e2e_prover/full.test.ts"
+      FAKE-PROOFS: "1"
+  e2e-prover-full:
+    test-path: "e2e-prover/full.test.ts"
     env:
-      HARDWARE_CONCURRENCY: "32"
-  e2e_public_testnet: {}
-  e2e_sandbox_example:
-    use_compose: true
-  e2e_state_vars: {}
-  e2e_static_calls: {}
-  e2e_synching: {}
-  e2e_token_contract: {}
-  flakey_e2e_tests:
-    test_path: "./src/flakey"
-    ignore_failures: true
-  guides_dapp_testing:
-    use_compose: true
-    test_path: "guides/dapp_testing.test.ts"
-  guides_sample_dapp:
-    use_compose: true
-    test_path: "sample-dapp/index.test.mjs"
-  guides_sample_dapp_ci:
-    use_compose: true
-    test_path: "sample-dapp/ci/index.test.mjs"
-  guides_up_quick_start:
-    use_compose: true
-    test_path: "guides/up_quick_start.test.ts"
-  guides_writing_an_account_contract:
-    use_compose: true
-    test_path: "guides/writing_an_account_contract.test.ts"
-  integration_l1_publisher:
-    use_compose: true
-  kind_network_4epochs:
+      HARDWARE-CONCURRENCY: "32"
+  e2e-public-testnet: {}
+  e2e-sandbox-example:
+    use-compose: true
+  e2e-state-vars: {}
+  e2e-static-calls: {}
+  e2e-synching: {}
+  e2e-token-contract: {}
+  flakey-e2e-tests:
+    test-path: "./src/flakey"
+    ignore-failures: true
+  guides-dapp-testing:
+    use-compose: true
+    test-path: "guides/dapp-testing.test.ts"
+  guides-sample-dapp:
+    use-compose: true
+    test-path: "sample-dapp/index.test.mjs"
+  guides-sample-dapp-ci:
+    use-compose: true
+    test-path: "sample-dapp/ci/index.test.mjs"
+  guides-up-quick-start:
+    use-compose: true
+    test-path: "guides/up-quick-start.test.ts"
+  guides-writing-an-account-contract:
+    use-compose: true
+    test-path: "guides/writing-an-account-contract.test.ts"
+  integration--l1-publisher:
+    use-compose: true
+  kind-network-4epochs:
     env:
       NAMESPACE: "smoke"
-      FRESH_INSTALL: "true"
-      VALUES_FILE: "$-default.yaml"
-    command: "./scripts/network_test.sh ./src/spartan/4epochs.test.ts"
-    ignore_failures: true
-  kind_network_smoke:
+      FRESH-INSTALL: "true"
+      VALUES-FILE: "$-default.yaml"
+    command: "./scripts/network-test.sh ./src/spartan/4epochs.test.ts"
+    ignore-failures: true
+  kind-network-smoke:
     env:
       NAMESPACE: "smoke"
-      FRESH_INSTALL: "true"
-      VALUES_FILE: "$-default.yaml"
-    command: "./scripts/network_test.sh ./src/spartan/smoke.test.ts"
-  kind_network_transfer:
+      FRESH-INSTALL: "true"
+      VALUES-FILE: "$-default.yaml"
+    command: "./scripts/network-test.sh ./src/spartan/smoke.test.ts"
+  kind-network-transfer:
     env:
       NAMESPACE: "transfer"
-      FRESH_INSTALL: "true"
-      VALUES_FILE: "$-default.yaml"
-    command: "./scripts/network_test.sh ./src/spartan/smoke.test.ts"
+      FRESH-INSTALL: "true"
+      VALUES-FILE: "$-default.yaml"
+    command: "./scripts/network-test.sh ./src/spartan/smoke.test.ts"
   pxe:
-    use_compose: true
-  uniswap_trade_on_l1_from_l2:
-    use_compose: true
+    use-compose: true
+  uniswap-trade-on-l1-from-l2:
+    use-compose: true

From 8fb3a0e7775dcfc3afbd9154a842d08bf4437563 Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 21:03:37 +0000
Subject: [PATCH 31/39] try startup probe

---
 spartan/aztec-network/templates/boot-node.yaml | 10 +++++-----
 spartan/aztec-network/templates/validator.yaml | 10 +++++-----
 2 files changed, 10 insertions(+), 10 deletions(-)

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 1bd2fb4c434..ac1d64959db 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -75,11 +75,11 @@ spec:
                 - /bin/sh
                 - -c
                 - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
-            initialDelaySeconds: {{ .Values.bootNode.readinessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.bootNode.readinessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.bootNode.readinessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.bootNode.readinessProbe.successThreshold }}
-            failureThreshold: {{ .Values.bootNode.readinessProbe.failureThreshold }}
+            initialDelaySeconds: {{ .Values.bootNode.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.bootNode.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.bootNode.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
           livenessProbe:
             exec:
               command:
diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index af2aed067cd..f9311cffe1f 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -84,11 +84,11 @@ spec:
                 - /bin/sh
                 - -c
                 - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
-            initialDelaySeconds: {{ .Values.validator.readinessProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.validator.readinessProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.validator.readinessProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.validator.readinessProbe.successThreshold }}
-            failureThreshold: {{ .Values.validator.readinessProbe.failureThreshold }}
+            initialDelaySeconds: {{ .Values.validator.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.validator.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.validator.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.validator.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.validator.startupProbe.failureThreshold }}
           livenessProbe:
             exec:
               command:

From 56612c184116fd7b90a2f170c3b64f695c2c7c6b Mon Sep 17 00:00:00 2001
From: ludamad <domuradical@gmail.com>
Date: Wed, 30 Oct 2024 22:30:23 +0000
Subject: [PATCH 32/39] revert

---
 scripts/ci/get_e2e_jobs.sh | 40 +++++++++++++++++++-------------------
 1 file changed, 20 insertions(+), 20 deletions(-)

diff --git a/scripts/ci/get_e2e_jobs.sh b/scripts/ci/get_e2e_jobs.sh
index dd32e768e7d..2d33dd12f37 100755
--- a/scripts/ci/get_e2e_jobs.sh
+++ b/scripts/ci/get_e2e_jobs.sh
@@ -17,26 +17,26 @@ get_test_names() {
 full_list=$(get_test_names)
 
 # Define the jobs that will run on every PR
-allow-list=(
-  "e2e-2-pxes"
-  "e2e-authwit"
-  "e2e-avm-simulator"
-  "e2e-block-building"
-  "e2e-cross-chain-messaging"
-  "e2e-deploy-contract"
-  "e2e-fees"
-  "e2e-fees-gas-estimation"
-  "e2e-fees-private-payments"
-  "e2e-max-block-number"
-  "e2e-nested-contract"
-  "e2e-ordering"
-  "e2e-static-calls"
-  "integration-l1-publisher"
-  "e2e-cheat-codes"
-  "e2e-prover-fake-proofs"
-  "e2e-prover-coordination"
-  "e2e-lending-contract"
-  "kind-network-smoke"
+allow_list=(
+  "e2e_2_pxes"
+  "e2e_authwit"
+  "e2e_avm_simulator"
+  "e2e_block_building"
+  "e2e_cross_chain_messaging"
+  "e2e_deploy_contract"
+  "e2e_fees"
+  "e2e_fees_gas_estimation"
+  "e2e_fees_private_payments"
+  "e2e_max_block_number"
+  "e2e_nested_contract"
+  "e2e_ordering"
+  "e2e_static_calls"
+  "integration_l1_publisher"
+  "e2e_cheat_codes"
+  "e2e_prover_fake_proofs"
+  "e2e_prover_coordination"
+  "e2e_lending_contract"
+  "kind_network_smoke"
 )
 
 # Add labels from input to the allow_list

From a77fca2939045eebf3c6d26fa907a8879bdc1300 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Wed, 30 Oct 2024 20:17:17 -0400
Subject: [PATCH 33/39] Update start_node.ts

---
 yarn-project/aztec/src/cli/cmds/start_node.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts
index 880a6e67cba..06bb311b3cb 100644
--- a/yarn-project/aztec/src/cli/cmds/start_node.ts
+++ b/yarn-project/aztec/src/cli/cmds/start_node.ts
@@ -18,7 +18,7 @@ export const startNode = async (
   options: any,
   signalHandlers: (() => Promise<void>)[],
   userLog: LogFn,
-  ): Promise<ServerList> => {
+): Promise<ServerList> => {
   // Services that will be started in a single multi-rpc server
   const services: ServerList = [];
 

From a2606a91309681f1b39b5ab7e9fc97b50140fa5f Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 11:52:44 -0400
Subject: [PATCH 34/39] Update boot-node.yaml

---
 spartan/aztec-network/templates/boot-node.yaml | 11 +++--------
 1 file changed, 3 insertions(+), 8 deletions(-)

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index ac1d64959db..688ed840455 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -70,15 +70,10 @@ spec:
               "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
           startupProbe:
-            exec:
-              command:
-                - /bin/sh
-                - -c
-                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
-            initialDelaySeconds: {{ .Values.bootNode.startupProbe.initialDelaySeconds }}
+            httpGet:
+              path: /status
+              port: {{ .Values.bootNode.service.nodePort }}
             periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.bootNode.startupProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.bootNode.startupProbe.successThreshold }}
             failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
           livenessProbe:
             exec:

From 6b0a2f850d149fc004cfc71b577ffee5bf66a787 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 11:54:04 -0400
Subject: [PATCH 35/39] Update validator.yaml

---
 spartan/aztec-network/templates/validator.yaml | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index f9311cffe1f..f3416d56647 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -79,16 +79,11 @@ spec:
             - "-c"
             - "sleep 10 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer"
           startupProbe:
-            exec:
-              command:
-                - /bin/sh
-                - -c
-                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
-            initialDelaySeconds: {{ .Values.validator.startupProbe.initialDelaySeconds }}
-            periodSeconds: {{ .Values.validator.startupProbe.periodSeconds }}
-            timeoutSeconds: {{ .Values.validator.startupProbe.timeoutSeconds }}
-            successThreshold: {{ .Values.validator.startupProbe.successThreshold }}
+            httpGet:
+              path: /status
+              port: {{ .Values.bootNode.service.nodePort }}
             failureThreshold: {{ .Values.validator.startupProbe.failureThreshold }}
+            periodSeconds: {{ .Values.validator.startupProbe.periodSeconds }}
           livenessProbe:
             exec:
               command:

From 76e879eba11405e628aec6a313d4b71083351e93 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 11:56:11 -0400
Subject: [PATCH 36/39] Update values.yaml

---
 spartan/aztec-network/values.yaml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml
index 8362fb3adac..3dc885cb56c 100644
--- a/spartan/aztec-network/values.yaml
+++ b/spartan/aztec-network/values.yaml
@@ -45,10 +45,7 @@ bootNode:
       cpu: "200m"
   deployContracts: true # Set to false to use manual contract addresses
   startupProbe:
-    initialDelaySeconds: 5
     periodSeconds: 10
-    timeoutSeconds: 5
-    successThreshold: 1
     # Only if we fail for 20 minutes straight do we call it botched
     # This gives enough time to sync
     failureThreshold: 120
@@ -84,10 +81,7 @@ validator:
   p2p:
     enabled: "true"
   startupProbe:
-    initialDelaySeconds: 5
     periodSeconds: 10
-    timeoutSeconds: 5
-    successThreshold: 1
     # Only if we fail for 20 minutes straight do we call it botched
     # This gives enough time to sync
     failureThreshold: 120

From 9b2cd6d76b338b4c2a49555bf03153fec750afce Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 12:23:31 -0400
Subject: [PATCH 37/39] Update boot-node.yaml

---
 spartan/aztec-network/templates/boot-node.yaml | 18 ++++++++----------
 1 file changed, 8 insertions(+), 10 deletions(-)

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 688ed840455..97aaba0b159 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -75,16 +75,14 @@ spec:
               port: {{ .Values.bootNode.service.nodePort }}
             periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
             failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
-          livenessProbe:
-            exec:
-              command:
-                - /bin/sh
-                - -c
-                - curl -fSs http://127.0.0.1:{{ .Values.bootNode.service.nodePort }}/status
-            initialDelaySeconds: 30
-            periodSeconds: 5
-            timeoutSeconds: 30
-            failureThreshold: 3
+          # livenessProbe:
+          #   httpGet:
+          #     path: /status
+          #     port: {{ .Values.bootNode.service.nodePort }}
+          #   initialDelaySeconds: 30
+          #   periodSeconds: 5
+          #   timeoutSeconds: 30
+          #   failureThreshold: 3
           volumeMounts:
           {{- if .Values.bootNode.deployContracts }}
             - name: scripts-output

From 7a31663b8703d215b81c09c77e8a74d6fd3b1685 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 13:08:22 -0400
Subject: [PATCH 38/39] Update boot-node.yaml

---
 spartan/aztec-network/templates/boot-node.yaml | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 97aaba0b159..898eef3663c 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -69,12 +69,12 @@ spec:
               "-c",
               "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
-          startupProbe:
-            httpGet:
-              path: /status
-              port: {{ .Values.bootNode.service.nodePort }}
-            periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
-            failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
+          # startupProbe:
+          #   httpGet:
+          #     path: /status
+          #     port: {{ .Values.bootNode.service.nodePort }}
+          #   periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
+          #   failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
           # livenessProbe:
           #   httpGet:
           #     path: /status

From 531896c45425f690ab251153a468e88eb01a16c4 Mon Sep 17 00:00:00 2001
From: ludamad <adam.domurad@gmail.com>
Date: Thu, 31 Oct 2024 16:01:57 -0400
Subject: [PATCH 39/39] try and fix startup probe issue with pod dns

---
 .../aztec-network/templates/boot-node.yaml    | 42 ++++++++-----------
 .../aztec-network/templates/prover-node.yaml  | 14 ++-----
 .../aztec-network/templates/validator.yaml    | 16 +++----
 3 files changed, 27 insertions(+), 45 deletions(-)

diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml
index 898eef3663c..abe934d44dd 100644
--- a/spartan/aztec-network/templates/boot-node.yaml
+++ b/spartan/aztec-network/templates/boot-node.yaml
@@ -69,20 +69,20 @@ spec:
               "-c",
               "sleep 30 && source /shared/contracts.env && env && node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver --sequencer --pxe",
             ]
-          # startupProbe:
-          #   httpGet:
-          #     path: /status
-          #     port: {{ .Values.bootNode.service.nodePort }}
-          #   periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
-          #   failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
-          # livenessProbe:
-          #   httpGet:
-          #     path: /status
-          #     port: {{ .Values.bootNode.service.nodePort }}
-          #   initialDelaySeconds: 30
-          #   periodSeconds: 5
-          #   timeoutSeconds: 30
-          #   failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.bootNode.service.nodePort }}
+            periodSeconds: {{ .Values.bootNode.startupProbe.periodSeconds }}
+            failureThreshold: {{ .Values.bootNode.startupProbe.failureThreshold }}
+          livenessProbe:
+            httpGet:
+              path: /status
+              port: {{ .Values.bootNode.service.nodePort }}
+            initialDelaySeconds: 30
+            periodSeconds: 5
+            timeoutSeconds: 30
+            failureThreshold: 3
           volumeMounts:
           {{- if .Values.bootNode.deployContracts }}
             - name: scripts-output
@@ -93,16 +93,10 @@ spec:
               subPath: contracts.env
           {{- end }}
           env:
-            - name: POD_NAME
+            - name: POD_IP
               valueFrom:
                 fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: POD_DNS_NAME
-              value: "$(POD_NAME).{{ include "aztec-network.fullname" . }}-boot-node.$(POD_NAMESPACE).svc.cluster.local"
+                  fieldPath: status.podIP
             - name: PORT
               value: "{{ .Values.bootNode.service.nodePort }}"
             - name: LOG_LEVEL
@@ -127,13 +121,13 @@ spec:
               {{- if .Values.bootNode.externalTcpHost }}
               value: "{{ .Values.bootNode.externalTcpHost }}:{{ .Values.bootNode.service.p2pTcpPort }}"
               {{- else }}
-              value: "$(POD_DNS_NAME):{{ .Values.bootNode.service.p2pTcpPort }}"
+              value: "$(POD_IP):{{ .Values.bootNode.service.p2pTcpPort }}"
               {{- end }}
             - name: P2P_UDP_ANNOUNCE_ADDR
               {{- if .Values.bootNode.externalUdpHost }}
               value: "{{ .Values.bootNode.externalUdpHost }}:{{ .Values.bootNode.service.p2pUdpPort }}"
               {{- else }}
-              value: "$(POD_DNS_NAME):{{ .Values.bootNode.service.p2pUdpPort }}"
+              value: "$(POD_IP):{{ .Values.bootNode.service.p2pUdpPort }}"
               {{- end }}
             - name: P2P_TCP_LISTEN_ADDR
               value: "0.0.0.0:{{ .Values.bootNode.service.p2pTcpPort }}"
diff --git a/spartan/aztec-network/templates/prover-node.yaml b/spartan/aztec-network/templates/prover-node.yaml
index 6ec388150d3..42d00ffa83e 100644
--- a/spartan/aztec-network/templates/prover-node.yaml
+++ b/spartan/aztec-network/templates/prover-node.yaml
@@ -72,16 +72,10 @@ spec:
             - name: shared-volume
               mountPath: /shared
           env:
-            - name: POD_NAME
+            - name: POD_IP
               valueFrom:
                 fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.namespace
-            - name: POD_DNS_NAME
-              value: "$(POD_NAME).{{ include "aztec-network.fullname" . }}-prover-node.$(POD_NAMESPACE).svc.cluster.local"
+                  fieldPath: status.podIP
             - name: PORT
               value: "{{ .Values.proverNode.service.nodePort }}"
             - name: LOG_LEVEL
@@ -102,7 +96,7 @@ spec:
             - name: PROVER_COORDINATION_NODE_URL
               value: {{ include "aztec-network.bootNodeUrl" . | quote }}
             - name: PROVER_JOB_SOURCE_URL
-              value: "http://$(POD_DNS_NAME):{{ .Values.proverNode.service.nodePort }}"
+              value: "http://$(POD_IP):{{ .Values.proverNode.service.nodePort }}"
           ports:
             - containerPort: {{ .Values.proverNode.service.nodePort }}
           resources:
@@ -163,4 +157,4 @@ spec:
     - port: {{ .Values.proverNode.service.nodePort }}
       name: node
 {{ end }}
-{{ end }}
\ No newline at end of file
+{{ end }}
diff --git a/spartan/aztec-network/templates/validator.yaml b/spartan/aztec-network/templates/validator.yaml
index f3416d56647..310bf2bcacf 100644
--- a/spartan/aztec-network/templates/validator.yaml
+++ b/spartan/aztec-network/templates/validator.yaml
@@ -81,7 +81,7 @@ spec:
           startupProbe:
             httpGet:
               path: /status
-              port: {{ .Values.bootNode.service.nodePort }}
+              port: {{ .Values.validator.service.nodePort }}
             failureThreshold: {{ .Values.validator.startupProbe.failureThreshold }}
             periodSeconds: {{ .Values.validator.startupProbe.periodSeconds }}
           livenessProbe:
@@ -98,16 +98,10 @@ spec:
             - name: shared-volume
               mountPath: /shared
           env:
-            - name: POD_NAME
-              valueFrom:
-                fieldRef:
-                  fieldPath: metadata.name
-            - name: POD_NAMESPACE
+            - name: POD_IP
               valueFrom:
                 fieldRef:
-                  fieldPath: metadata.namespace
-            - name: POD_DNS_NAME
-              value: "$(POD_NAME).{{ include "aztec-network.fullname" . }}-validator.$(POD_NAMESPACE).svc.cluster.local"
+                  fieldPath: status.podIP
             - name: PORT
               value: "{{ .Values.validator.service.nodePort }}"
             - name: LOG_LEVEL
@@ -130,13 +124,13 @@ spec:
               {{- if .Values.validator.externalTcpHost }}
               value: "{{ .Values.validator.externalTcpHost }}:{{ .Values.validator.service.p2pTcpPort }}"
               {{- else }}
-              value: "$(POD_DNS_NAME):{{ .Values.validator.service.p2pTcpPort }}"
+              value: "$(POD_IP):{{ .Values.validator.service.p2pTcpPort }}"
               {{- end }}
             - name: P2P_UDP_ANNOUNCE_ADDR
               {{- if .Values.validator.externalUdpHost }}
               value: "{{ .Values.validator.externalUdpHost }}:{{ .Values.validator.service.p2pUdpPort }}"
               {{- else }}
-              value: "$(POD_DNS_NAME):{{ .Values.validator.service.p2pUdpPort }}"
+              value: "$(POD_IP):{{ .Values.validator.service.p2pUdpPort }}"
               {{- end }}
             - name: P2P_TCP_LISTEN_ADDR
               value: "0.0.0.0:{{ .Values.validator.service.p2pTcpPort }}"