From 6bf2b7269fddb5bd7fe4c567710146b4969d2845 Mon Sep 17 00:00:00 2001 From: Alex Gherghisan Date: Mon, 24 Jun 2024 15:29:12 +0100 Subject: [PATCH] feat: add OpenTelemetry to node (#7102) This PR adds OpenTelemetry to the monorepo and starts tracking the following metrics in the node: - resource usage (CPU, system memory) - block height - average block size (how many txs per block) - mempool status - time taken to generate the witness and prove the protocol circuits - time to simulate the circuits (only relevant if using mocked proofs) The witgen/proving time are using gauges rather than histograms because of a quirk with how often Prometheus scrapes the metrics vs how many proofs we generate of the same type. In PromQL recreating the histogram quantile requires looking at the rate of change of the individual buckets but if proving takes tens-of-seconds it doesn't change often enough so it ends up dividing by 0. Using a guage gives us instantaneous values, but we potentially lose data (e.g. if two proofs finish in the same scrape interval) and in the dashboard the numbers won't "decay" (meaning if for some reason the node stop producing blocks the proof duration will stay at the previous value). Three new components are added to the architecture: - an instance of the OpenTelemetry collector (aggregates metrics pushed by the node) - an instance of Prometheus to scrape to collector for data - an instance of Grafana to chart the data The top-level docker-compose has been updated to include these components under the `metrics` profile. To run the metrics stack: ``` $ docker compose --profile metrics up -d ``` Then e2e tests can be run with metrics being exported to Grafana: ``` OTEL_COLLECTOR_HOST=127.0.0.1:4318 yarn test e2e_block_building ``` Two Grafana dashboards are included with this PR. Node stats: ![image](https://github.com/AztecProtocol/aztec-packages/assets/3816165/0536ffde-934f-46de-b864-c8464925de4c) Protocol circuit stats: ![image](https://github.com/AztecProtocol/aztec-packages/assets/3816165/8e1e15ce-daaf-4d65-b5e8-a681335cac80) --- cspell.json | 11 +- docker-compose.yml | 116 ++- .../aztec/aztec-node-dashboard.json | 576 ++++++++++++++ .../aztec/protocol-circuits-dashboard.json | 747 ++++++++++++++++++ grafana_dashboards/default.yml | 11 + yarn-project/archiver/package.json | 1 + .../archiver/src/archiver/archiver.test.ts | 3 + .../archiver/src/archiver/archiver.ts | 13 +- .../archiver/src/archiver/instrumentation.ts | 30 + .../archiver/kv_archiver_store/block_store.ts | 1 - yarn-project/archiver/src/index.ts | 3 + yarn-project/archiver/tsconfig.json | 3 + yarn-project/aztec-node/package.json | 1 + .../aztec-node/src/aztec-node/server.test.ts | 5 +- .../aztec-node/src/aztec-node/server.ts | 13 +- yarn-project/aztec-node/src/bin/index.ts | 3 +- yarn-project/aztec-node/tsconfig.json | 3 + yarn-project/aztec/package.json | 1 + .../aztec/src/cli/cmds/start_archiver.ts | 7 +- yarn-project/aztec/src/cli/cmds/start_node.ts | 7 +- .../aztec/src/cli/cmds/start_prover.ts | 22 +- yarn-project/aztec/src/sandbox.ts | 8 +- yarn-project/aztec/tsconfig.json | 3 + yarn-project/bb-prover/package.json | 1 + yarn-project/bb-prover/src/bb/execute.ts | 20 +- yarn-project/bb-prover/src/instrumentation.ts | 144 ++++ .../src/prover/bb_native_proof_creator.ts | 6 +- .../bb-prover/src/prover/bb_prover.ts | 60 +- .../bb-prover/src/test/test_circuit_prover.ts | 19 +- yarn-project/bb-prover/tsconfig.json | 3 + yarn-project/end-to-end/package.json | 1 + .../composed/integration_l1_publisher.test.ts | 3 +- .../end-to-end/src/e2e_p2p_network.test.ts | 7 +- yarn-project/end-to-end/src/fixtures/utils.ts | 10 +- yarn-project/end-to-end/tsconfig.json | 3 + yarn-project/p2p/package.json | 1 + .../p2p/src/tx_pool/aztec_kv_tx_pool.test.ts | 3 +- .../p2p/src/tx_pool/aztec_kv_tx_pool.ts | 15 +- .../p2p/src/tx_pool/instrumentation.ts | 58 ++ .../p2p/src/tx_pool/memory_tx_pool.test.ts | 4 +- .../p2p/src/tx_pool/memory_tx_pool.ts | 9 +- yarn-project/p2p/tsconfig.json | 3 + yarn-project/package.json | 3 +- yarn-project/prover-client/package.json | 1 + .../prover-client/src/mocks/test_context.ts | 5 +- .../orchestrator_failures.test.ts | 3 +- .../orchestrator_lifecycle.test.ts | 3 +- .../src/test/bb_prover_base_rollup.test.ts | 3 +- .../src/test/bb_prover_full_rollup.test.ts | 3 +- .../src/test/bb_prover_parity.test.ts | 3 +- .../prover-client/src/tx-prover/tx-prover.ts | 18 +- yarn-project/prover-client/tsconfig.json | 3 + yarn-project/telemetry-client/.eslintrc.cjs | 1 + yarn-project/telemetry-client/package.json | 67 ++ .../telemetry-client/src/attributes.ts | 36 + yarn-project/telemetry-client/src/index.ts | 1 + yarn-project/telemetry-client/src/metrics.ts | 30 + yarn-project/telemetry-client/src/noop.ts | 13 + yarn-project/telemetry-client/src/otel.ts | 53 ++ yarn-project/telemetry-client/src/start.ts | 27 + .../telemetry-client/src/telemetry.ts | 69 ++ yarn-project/telemetry-client/tsconfig.json | 14 + yarn-project/yarn.lock | 286 +++++++ 63 files changed, 2517 insertions(+), 83 deletions(-) create mode 100644 grafana_dashboards/aztec/aztec-node-dashboard.json create mode 100644 grafana_dashboards/aztec/protocol-circuits-dashboard.json create mode 100644 grafana_dashboards/default.yml create mode 100644 yarn-project/archiver/src/archiver/instrumentation.ts create mode 100644 yarn-project/bb-prover/src/instrumentation.ts create mode 100644 yarn-project/p2p/src/tx_pool/instrumentation.ts create mode 100644 yarn-project/telemetry-client/.eslintrc.cjs create mode 100644 yarn-project/telemetry-client/package.json create mode 100644 yarn-project/telemetry-client/src/attributes.ts create mode 100644 yarn-project/telemetry-client/src/index.ts create mode 100644 yarn-project/telemetry-client/src/metrics.ts create mode 100644 yarn-project/telemetry-client/src/noop.ts create mode 100644 yarn-project/telemetry-client/src/otel.ts create mode 100644 yarn-project/telemetry-client/src/start.ts create mode 100644 yarn-project/telemetry-client/src/telemetry.ts create mode 100644 yarn-project/telemetry-client/tsconfig.json diff --git a/cspell.json b/cspell.json index 78f4bd23c72..e7e5c5a5447 100644 --- a/cspell.json +++ b/cspell.json @@ -169,6 +169,9 @@ "nullifer", "offchain", "onchain", + "opentelemetry", + "otel", + "OTLP", "otterscan", "outdir", "overlayfs", @@ -253,6 +256,7 @@ "typegen", "typeparam", "undeployed", + "undici", "unexclude", "unexcluded", "unprefixed", @@ -270,6 +274,7 @@ "viem", "wasms", "webassembly", + "WITGEN", "workdir", "yamux", "yarnrc", @@ -301,5 +306,7 @@ "lib", "*.cmake" ], - "flagWords": ["anonymous"] -} \ No newline at end of file + "flagWords": [ + "anonymous" + ] +} diff --git a/docker-compose.yml b/docker-compose.yml index cc5bec7a7ec..f0001adcebd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -28,6 +28,8 @@ services: - aztec:/var/lib/aztec ports: - 8080:8080/tcp + profiles: + - pxe node: image: aztecprotocol/aztec${AZTEC_DOCKER_TAG:-@sha256:03feac60e91f1aabf678cecbcd13271dda229120ec6007f2c1bac718ff550c70} @@ -59,18 +61,34 @@ services: P2P_ENABLED: true PEER_ID_PRIVATE_KEY: AZTEC_PORT: 8999 + OTEL_COLLECTOR_BASE_URL: ${OTEL_COLLECTOR_BASE_URL:-http://otel-collector:4318} secrets: - ethereum-host - p2p-boot-node - entrypoint: [ - "/bin/sh", - "-c", - "export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host);\ - export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node);\ - test -z \"$$PEER_ID_PRIVATE_KEY\" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d' ' -f 3 | tee /var/lib/aztec/p2p-private-key || echo 'Re-using existing P2P private key';\ - test -z \"$$PEER_ID_PRIVATE_KEY\" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key);\ - node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver", - ] + entrypoint: | + /bin/sh -c ' + export ETHEREUM_HOST=$$(cat /var/run/secrets/ethereum-host) + export BOOTSTRAP_NODES=$$(cat /var/run/secrets/p2p-boot-node) + + test -z "$$PEER_ID_PRIVATE_KEY" -a ! -f /var/lib/aztec/p2p-private-key && node /usr/src/yarn-project/cli/dest/bin/index.js generate-p2p-private-key | head -1 | cut -d" " -f 3 | tee /var/lib/aztec/p2p-private-key || echo "Re-using existing P2P private key" + test -z "$$PEER_ID_PRIVATE_KEY" && export PEER_ID_PRIVATE_KEY=$$(cat /var/lib/aztec/p2p-private-key) + + # if the stack is started with --profile metrics --profile node, give the collector a chance to start before the node + i=0 + max=3 + while ! curl --head --silent $$OTEL_COLLECTOR_BASE_URL > /dev/null; do + echo "OpenTelemetry collector not up. Retrying after 1s"; + sleep 1; + i=$$((i+1)); + if [ $$i -eq $$max ]; then + echo "OpenTelemetry collector at $$OTEL_COLLECTOR_BASE_URL not up after $${max}s. Running without metrics"; + unset OTEL_COLLECTOR_BASE_URL; + break + fi; + done; + + node /usr/src/yarn-project/aztec/dest/bin/index.js start --node --archiver + ' volumes: - aztec:/var/lib/aztec profiles: @@ -94,8 +112,88 @@ services: profiles: - cli + otel-collector: + image: otel/opentelemetry-collector-contrib + configs: + - source: otel-collector-config + target: /etc/otelcol-contrib/config.yaml + profiles: + - metrics + ports: + - 4318:4318 + + prometheus: + image: prom/prometheus + profiles: + - metrics + configs: + - source: prometheus-config + target: /etc/prometheus/prometheus.yml + + grafana: + image: grafana/grafana + ports: + - 3000:3000 + profiles: + - metrics + volumes: + - ./grafana_dashboards:/etc/grafana/provisioning/dashboards + - grafana:/var/lib/grafana + configs: + - source: grafana-sources + target: /etc/grafana/provisioning/datasources/default.yml + volumes: aztec: + grafana: + +configs: + grafana-sources: + content: | + apiVersion: 1 + datasources: + - name: Prometheus + uid: aztec-node-metrics + type: prometheus + url: http://prometheus:9090 + editable: false + isDefault: true + jsonData: + timeInterval: 10s + + prometheus-config: + content: | + global: + evaluation_interval: 30s + scrape_interval: 10s + scrape_configs: + - job_name: otel-collector + static_configs: + - targets: ['otel-collector:8888'] + - job_name: aztec + static_configs: + - targets: ['otel-collector:8889'] + otel-collector-config: + content: | + receivers: + otlp: + protocols: + http: + + processors: + batch: + + exporters: + prometheus: + endpoint: 0.0.0.0:8889 + metric_expiration: 5m + + service: + pipelines: + metrics: + receivers: [otlp] + processors: [batch] + exporters: [prometheus] secrets: aztec-node-url: diff --git a/grafana_dashboards/aztec/aztec-node-dashboard.json b/grafana_dashboards/aztec/aztec-node-dashboard.json new file mode 100644 index 00000000000..863e0079d49 --- /dev/null +++ b/grafana_dashboards/aztec/aztec-node-dashboard.json @@ -0,0 +1,576 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Stats from the Aztec Node", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 6, + "panels": [], + "title": "Node status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "fieldMinMax": false, + "mappings": [], + "max": 1, + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 0, + "y": 1 + }, + "id": 7, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "valueSize": 64 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "sum(process_cpu_utilization)", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "CPU utilization", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 15, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 5, + "y": 1 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "system_memory_usage{system_memory_state=\"used\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Memory use", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 12, + "y": 1 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aztec_archiver_block_height", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Current block height", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "displayName": "txs/block", + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 17, + "y": 1 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": { + "titleSize": 12 + }, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(aztec_archiver_block_size_sum[$__rate_interval]) / rate(aztec_archiver_block_size_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average block size", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 7 + }, + "id": 3, + "panels": [], + "title": "Mempool", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "rate(aztec_mempool_tx_size_bytes_sum[$__rate_interval]) / rate(aztec_mempool_tx_size_bytes_count[$__rate_interval])", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Tx size", + "range": true, + "refId": "Avg tx size", + "useBackend": false + } + ], + "title": "Average transaction size ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["last"], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.0.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "aztec_mempool_tx_count", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "tx", + "useBackend": false + } + ], + "title": "Transactions in mempool", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Aztec Node", + "uid": "edp4qxqgjoav4e", + "version": 1, + "weekStart": "" +} diff --git a/grafana_dashboards/aztec/protocol-circuits-dashboard.json b/grafana_dashboards/aztec/protocol-circuits-dashboard.json new file mode 100644 index 00000000000..1849485d30c --- /dev/null +++ b/grafana_dashboards/aztec/protocol-circuits-dashboard.json @@ -0,0 +1,747 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Metrics relating to protocol circuits", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 3, + "panels": [], + "title": "Circuit proving", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"base-parity\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Base parity", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"root-parity\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Root parity", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"base-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Base rollup", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"merge-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Merge rollup", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"root-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Root rollup", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-setup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Setup", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-app-logic\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - App logic", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-teardown\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Teardown", + "range": true, + "refId": "H", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_proving_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-tail\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Tail", + "range": true, + "refId": "I", + "useBackend": false + } + ], + "title": "Circuit proving", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"base-parity\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Base parity", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"root-parity\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Root parity", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"base-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Base rollup", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"merge-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Merge rollup", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"root-rollup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Root rollup", + "range": true, + "refId": "E", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-setup\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Setup", + "range": true, + "refId": "F", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-app-logic\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - App logic", + "range": true, + "refId": "G", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-teardown\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Teardown", + "range": true, + "refId": "H", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "disableTextWrap": false, + "editorMode": "code", + "expr": "aztec_circuit_witness_generation_duration_seconds{aztec_circuit_protocol_circuit_name=\"public-kernel-tail\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Public Kernel - Tail", + "range": true, + "refId": "I", + "useBackend": false + } + ], + "title": "Circuit witness generation", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 2, + "panels": [], + "title": "Circuit simulation", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 10 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "exemplar": false, + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"base-parity\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"base-parity\"}[$__rate_interval])", + "instant": false, + "legendFormat": "Base paritiy", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"root-parity\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"root-parity\"}[$__rate_interval])", + "hide": true, + "instant": false, + "legendFormat": "Root paritiy", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"base-rollup\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"base-rollup\"}[$__rate_interval])", + "hide": true, + "instant": false, + "legendFormat": "Base rollup", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"merge-rollup\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"merge-rollup\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Merge rollup", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"root-rollup\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"root-rollup\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Root rollup", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"public-kernel-setup\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"public-kernel-setup\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Public kernel - Setup", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"public-kernel-app-logic\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"public-kernel-app-logic\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Public kernel - App logic", + "range": true, + "refId": "H" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"public-kernel-teardown\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"public-kernel-teardown\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Public kernel - Teardown", + "range": true, + "refId": "I" + }, + { + "datasource": { + "type": "prometheus", + "uid": "aztec-node-metrics" + }, + "editorMode": "code", + "expr": "rate(aztec_circuit_simulation_duration_seconds_sum{aztec_circuit_protocol_circuit_name=\"public-kernel-tail\"}[$__rate_interval]) / rate(aztec_circuit_simulation_duration_seconds_count{aztec_circuit_protocol_circuit_name=\"public-kernel-tail\"}[$__rate_interval])", + "hide": false, + "instant": false, + "legendFormat": "Public kernel - Tail", + "range": true, + "refId": "G" + } + ], + "title": "Circuit simulation (only when faking proofs)", + "type": "timeseries" + } + ], + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timeRangeUpdatedDuringEditOrView": false, + "timepicker": {}, + "timezone": "browser", + "title": "Protocol circuits", + "uid": "ddp5sfpkscb9cf", + "version": 3, + "weekStart": "" +} diff --git a/grafana_dashboards/default.yml b/grafana_dashboards/default.yml new file mode 100644 index 00000000000..d83924c0ffb --- /dev/null +++ b/grafana_dashboards/default.yml @@ -0,0 +1,11 @@ +apiVersion: 1 + +providers: + - name: "Aztec" + orgId: 1 + folder: "Aztec" + type: file + disableDeletion: false + editable: true + options: + path: /etc/grafana/provisioning/dashboards/aztec diff --git a/yarn-project/archiver/package.json b/yarn-project/archiver/package.json index 40f0a179937..645d038c2f6 100644 --- a/yarn-project/archiver/package.json +++ b/yarn-project/archiver/package.json @@ -57,6 +57,7 @@ "@aztec/kv-store": "workspace:^", "@aztec/l1-artifacts": "workspace:^", "@aztec/protocol-contracts": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@aztec/types": "workspace:^", "debug": "^4.3.4", "lodash.groupby": "^4.6.0", diff --git a/yarn-project/archiver/src/archiver/archiver.test.ts b/yarn-project/archiver/src/archiver/archiver.test.ts index 9c83e57981d..1040f308fae 100644 --- a/yarn-project/archiver/src/archiver/archiver.test.ts +++ b/yarn-project/archiver/src/archiver/archiver.test.ts @@ -10,6 +10,7 @@ import { EthAddress } from '@aztec/foundation/eth-address'; import { Fr } from '@aztec/foundation/fields'; import { sleep } from '@aztec/foundation/sleep'; import { AvailabilityOracleAbi, type InboxAbi, RollupAbi } from '@aztec/l1-artifacts'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type MockProxy, mock } from 'jest-mock-extended'; import { @@ -49,6 +50,7 @@ describe('Archiver', () => { registryAddress, archiverStore, 1000, + new NoopTelemetryClient(), ); let latestBlockNum = await archiver.getBlockNumber(); @@ -152,6 +154,7 @@ describe('Archiver', () => { registryAddress, archiverStore, 1000, + new NoopTelemetryClient(), ); let latestBlockNum = await archiver.getBlockNumber(); diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 9face3a26ae..b03ce4d2115 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -29,6 +29,7 @@ import { Fr } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { RunningPromise } from '@aztec/foundation/running-promise'; import { ClassRegistererAddress } from '@aztec/protocol-contracts/class-registerer'; +import { type TelemetryClient } from '@aztec/telemetry-client'; import { type ContractClassPublic, type ContractDataSource, @@ -49,6 +50,7 @@ import { retrieveBlockMetadataFromRollup, retrieveL1ToL2Messages, } from './data_retrieval.js'; +import { ArchiverInstrumentation } from './instrumentation.js'; /** * Helper interface to combine all sources this archiver implementation provides. @@ -66,6 +68,9 @@ export class Archiver implements ArchiveSource { */ private runningPromise?: RunningPromise; + /** Capture runtime metrics */ + private instrumentation: ArchiverInstrumentation; + /** * Creates a new instance of the Archiver. * @param publicClient - A client for interacting with the Ethereum node. @@ -84,8 +89,11 @@ export class Archiver implements ArchiveSource { private readonly registryAddress: EthAddress, private readonly store: ArchiverDataStore, private readonly pollingIntervalMs = 10_000, + telemetry: TelemetryClient, private readonly log: DebugLogger = createDebugLogger('aztec:archiver'), - ) {} + ) { + this.instrumentation = new ArchiverInstrumentation(telemetry); + } /** * Creates a new instance of the Archiver and blocks until it syncs from chain. @@ -97,6 +105,7 @@ export class Archiver implements ArchiveSource { public static async createAndSync( config: ArchiverConfig, archiverStore: ArchiverDataStore, + telemetry: TelemetryClient, blockUntilSynced = true, ): Promise { const chain = createEthereumChain(config.rpcUrl, config.apiKey); @@ -114,6 +123,7 @@ export class Archiver implements ArchiveSource { config.l1Contracts.registryAddress, archiverStore, config.archiverPollingIntervalMS, + telemetry, ); await archiver.start(blockUntilSynced); return archiver; @@ -286,6 +296,7 @@ export class Archiver implements ArchiveSource { ); await this.store.addBlocks(retrievedBlocks); + this.instrumentation.processNewBlocks(retrievedBlocks.retrievedData); } /** diff --git a/yarn-project/archiver/src/archiver/instrumentation.ts b/yarn-project/archiver/src/archiver/instrumentation.ts new file mode 100644 index 00000000000..837b00af7f2 --- /dev/null +++ b/yarn-project/archiver/src/archiver/instrumentation.ts @@ -0,0 +1,30 @@ +import { type L2Block } from '@aztec/circuit-types'; +import { type Gauge, type Histogram, Metrics, type TelemetryClient, ValueType } from '@aztec/telemetry-client'; + +export class ArchiverInstrumentation { + private blockHeight: Gauge; + private blockSize: Histogram; + + constructor(telemetry: TelemetryClient) { + const meter = telemetry.getMeter('Archiver'); + this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT, { + description: 'The height of the latest block processed by the archiver', + valueType: ValueType.INT, + }); + + this.blockSize = meter.createHistogram(Metrics.ARCHIVER_BLOCK_SIZE, { + description: 'The number of transactions processed per block', + valueType: ValueType.INT, + advice: { + explicitBucketBoundaries: [2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 2048, 4096, 8192], + }, + }); + } + + public processNewBlocks(blocks: L2Block[]) { + this.blockHeight.record(Math.max(...blocks.map(b => b.number))); + for (const block of blocks) { + this.blockSize.record(block.body.txEffects.length); + } + } +} diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index 693b1e9c60c..d22537ae824 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -187,7 +187,6 @@ export class BlockStore { } if (start < INITIAL_L2_BLOCK_NUM) { - this.#log.verbose(`Clamping start block ${start} to ${INITIAL_L2_BLOCK_NUM}`); start = INITIAL_L2_BLOCK_NUM; } diff --git a/yarn-project/archiver/src/index.ts b/yarn-project/archiver/src/index.ts index fb3f8da310a..cf4549e81a0 100644 --- a/yarn-project/archiver/src/index.ts +++ b/yarn-project/archiver/src/index.ts @@ -1,5 +1,6 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { fileURLToPath } from '@aztec/foundation/url'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { createPublicClient, http } from 'viem'; import { localhost } from 'viem/chains'; @@ -34,6 +35,8 @@ async function main() { l1Contracts.inboxAddress, l1Contracts.registryAddress, archiverStore, + 1000, + new NoopTelemetryClient(), ); const shutdown = async () => { diff --git a/yarn-project/archiver/tsconfig.json b/yarn-project/archiver/tsconfig.json index ea0bb3a5469..dbe9915c010 100644 --- a/yarn-project/archiver/tsconfig.json +++ b/yarn-project/archiver/tsconfig.json @@ -27,6 +27,9 @@ { "path": "../protocol-contracts" }, + { + "path": "../telemetry-client" + }, { "path": "../types" }, diff --git a/yarn-project/aztec-node/package.json b/yarn-project/aztec-node/package.json index 6689da13001..2ef2eb39a5f 100644 --- a/yarn-project/aztec-node/package.json +++ b/yarn-project/aztec-node/package.json @@ -62,6 +62,7 @@ "@aztec/prover-client": "workspace:^", "@aztec/sequencer-client": "workspace:^", "@aztec/simulator": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@aztec/types": "workspace:^", "@aztec/world-state": "workspace:^", "koa": "^2.14.2", diff --git a/yarn-project/aztec-node/src/aztec-node/server.test.ts b/yarn-project/aztec-node/src/aztec-node/server.test.ts index 309a9ef3bcc..a1d559bf498 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.test.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.test.ts @@ -1,4 +1,5 @@ import { createEthereumChain } from '@aztec/ethereum'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type AztecNodeConfig, AztecNodeService } from '../index.js'; @@ -10,7 +11,9 @@ describe('aztec node service', () => { chainId: 12345, // not the testnet chain id }; const ethereumChain = createEthereumChain(config.rpcUrl!, config.apiKey); - await expect(() => AztecNodeService.createAndSync(config as AztecNodeConfig)).rejects.toThrow( + await expect(() => + AztecNodeService.createAndSync(config as AztecNodeConfig, new NoopTelemetryClient()), + ).rejects.toThrow( `RPC URL configured for chain id ${ethereumChain.chainInfo.id} but expected id ${config.chainId}`, ); }); diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index da82aa420e7..9afb6ed9ddb 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -63,6 +63,8 @@ import { getCanonicalMultiCallEntrypointAddress } from '@aztec/protocol-contract import { TxProver } from '@aztec/prover-client'; import { type GlobalVariableBuilder, SequencerClient, getGlobalVariableBuilder } from '@aztec/sequencer-client'; import { PublicProcessorFactory, WASMSimulator } from '@aztec/simulator'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type ContractClassPublic, type ContractDataSource, @@ -104,6 +106,7 @@ export class AztecNodeService implements AztecNode { protected readonly merkleTreesDb: AztecKVStore, private readonly prover: ProverClient | undefined, private txValidator: TxValidator, + private telemetry: TelemetryClient, private log = createDebugLogger('aztec:node'), ) { this.packageVersion = getPackageInfo().version; @@ -124,9 +127,11 @@ export class AztecNodeService implements AztecNode { */ public static async createAndSync( config: AztecNodeConfig, + telemetry?: TelemetryClient, log = createDebugLogger('aztec:node'), storeLog = createDebugLogger('aztec:node:lmdb'), - ) { + ): Promise { + telemetry ??= new NoopTelemetryClient(); const ethereumChain = createEthereumChain(config.rpcUrl, config.apiKey); //validate that the actual chain id matches that specified in configuration if (config.chainId !== ethereumChain.chainInfo.id) { @@ -145,7 +150,7 @@ export class AztecNodeService implements AztecNode { if (!config.archiverUrl) { // first create and sync the archiver const archiverStore = new KVArchiverDataStore(store, config.maxLogs); - archiver = await Archiver.createAndSync(config, archiverStore, true); + archiver = await Archiver.createAndSync(config, archiverStore, telemetry, true); } else { archiver = createArchiverClient(config.archiverUrl); } @@ -155,7 +160,7 @@ export class AztecNodeService implements AztecNode { config.transactionProtocol = `/aztec/tx/${config.l1Contracts.rollupAddress.toString()}`; // create the tx pool and the p2p client, which will need the l2 block source - const p2pClient = await createP2PClient(store, config, new AztecKVTxPool(store), archiver); + const p2pClient = await createP2PClient(store, config, new AztecKVTxPool(store, telemetry), archiver); // now create the merkle trees and the world state synchronizer const merkleTrees = await MerkleTrees.new(store); @@ -179,6 +184,7 @@ export class AztecNodeService implements AztecNode { config, await proofVerifier.getVerificationKeys(), worldStateSynchronizer, + telemetry, await archiver .getBlock(-1) .then(b => b?.header ?? worldStateSynchronizer.getCommitted().buildInitialHeader()), @@ -218,6 +224,7 @@ export class AztecNodeService implements AztecNode { store, prover, txValidator, + telemetry, log, ); } diff --git a/yarn-project/aztec-node/src/bin/index.ts b/yarn-project/aztec-node/src/bin/index.ts index e1688b79198..41aba729aeb 100644 --- a/yarn-project/aztec-node/src/bin/index.ts +++ b/yarn-project/aztec-node/src/bin/index.ts @@ -1,5 +1,6 @@ #!/usr/bin/env -S node --no-warnings import { createDebugLogger } from '@aztec/foundation/log'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import http from 'http'; @@ -15,7 +16,7 @@ const logger = createDebugLogger('aztec:node'); async function createAndDeployAztecNode() { const aztecNodeConfig: AztecNodeConfig = { ...getConfigEnvVars() }; - return await AztecNodeService.createAndSync(aztecNodeConfig); + return await AztecNodeService.createAndSync(aztecNodeConfig, new NoopTelemetryClient()); } /** diff --git a/yarn-project/aztec-node/tsconfig.json b/yarn-project/aztec-node/tsconfig.json index f023c003bff..5a6637a7bac 100644 --- a/yarn-project/aztec-node/tsconfig.json +++ b/yarn-project/aztec-node/tsconfig.json @@ -48,6 +48,9 @@ { "path": "../simulator" }, + { + "path": "../telemetry-client" + }, { "path": "../types" }, diff --git a/yarn-project/aztec/package.json b/yarn-project/aztec/package.json index 6caffb8c0ce..b0e86433144 100644 --- a/yarn-project/aztec/package.json +++ b/yarn-project/aztec/package.json @@ -46,6 +46,7 @@ "@aztec/protocol-contracts": "workspace:^", "@aztec/prover-client": "workspace:^", "@aztec/pxe": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "abitype": "^0.8.11", "commander": "^11.1.0", "koa": "^2.14.2", diff --git a/yarn-project/aztec/src/cli/cmds/start_archiver.ts b/yarn-project/aztec/src/cli/cmds/start_archiver.ts index 567f2b160b0..bf0df3808c2 100644 --- a/yarn-project/aztec/src/cli/cmds/start_archiver.ts +++ b/yarn-project/aztec/src/cli/cmds/start_archiver.ts @@ -9,6 +9,10 @@ import { createDebugLogger } from '@aztec/aztec.js'; import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { AztecLmdbStore } from '@aztec/kv-store/lmdb'; import { initStoreForRollup } from '@aztec/kv-store/utils'; +import { + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryClientConfig, +} from '@aztec/telemetry-client/start'; import { mergeEnvVarsAndCliOptions, parseModuleOptions } from '../util.js'; @@ -30,7 +34,8 @@ export const startArchiver = async (options: any, signalHandlers: (() => Promise ); const archiverStore = new KVArchiverDataStore(store, archiverConfig.maxLogs); - const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, true); + const telemetry = createAndStartTelemetryClient(getTelemetryClientConfig(), 'aztec-archiver'); + const archiver = await Archiver.createAndSync(archiverConfig, archiverStore, telemetry, true); const archiverServer = createArchiverRpcServer(archiver); services.push({ archiver: archiverServer }); signalHandlers.push(archiver.stop); diff --git a/yarn-project/aztec/src/cli/cmds/start_node.ts b/yarn-project/aztec/src/cli/cmds/start_node.ts index 9245bd32708..c6ff3024794 100644 --- a/yarn-project/aztec/src/cli/cmds/start_node.ts +++ b/yarn-project/aztec/src/cli/cmds/start_node.ts @@ -8,6 +8,10 @@ import { type ServerList } from '@aztec/foundation/json-rpc/server'; import { type LogFn } from '@aztec/foundation/log'; import { createProvingJobSourceServer } from '@aztec/prover-client/prover-agent'; import { type PXEServiceConfig, createPXERpcServer, getPXEServiceConfig } from '@aztec/pxe'; +import { + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryClientConfig, +} from '@aztec/telemetry-client/start'; import { mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; @@ -81,7 +85,8 @@ export const startNode = async ( } // Create and start Aztec Node. - const node = await createAztecNode(nodeConfig); + const telemetryClient = createAndStartTelemetryClient(getTelemetryClientConfig(), 'aztec-node'); + const node = await createAztecNode(telemetryClient, nodeConfig); const nodeServer = createAztecNodeRpcServer(node); // Add node to services list diff --git a/yarn-project/aztec/src/cli/cmds/start_prover.ts b/yarn-project/aztec/src/cli/cmds/start_prover.ts index 4b299ab5661..f5e32e7e741 100644 --- a/yarn-project/aztec/src/cli/cmds/start_prover.ts +++ b/yarn-project/aztec/src/cli/cmds/start_prover.ts @@ -2,6 +2,10 @@ import { BBNativeRollupProver, TestCircuitProver } from '@aztec/bb-prover'; import { type ServerCircuitProver } from '@aztec/circuit-types'; import { getProverEnvVars } from '@aztec/prover-client'; import { ProverAgent, createProvingJobSourceClient } from '@aztec/prover-client/prover-agent'; +import { + createAndStartTelemetryClient, + getConfigEnvVars as getTelemetryClientConfig, +} from '@aztec/telemetry-client/start'; import { type ServiceStarter, parseModuleOptions } from '../util.js'; @@ -30,20 +34,24 @@ export const startProver: ServiceStarter = async (options, signalHandlers, logge ? parseInt(proverOptions.proverAgentPollInterval, 10) : proverOptions.proverAgentPollInterval; + const telemetry = createAndStartTelemetryClient(getTelemetryClientConfig(), 'aztec-prover'); let circuitProver: ServerCircuitProver; if (proverOptions.realProofs) { if (!proverOptions.acvmBinaryPath || !proverOptions.bbBinaryPath) { throw new Error('Cannot start prover without simulation or native prover options'); } - circuitProver = await BBNativeRollupProver.new({ - acvmBinaryPath: proverOptions.acvmBinaryPath, - bbBinaryPath: proverOptions.bbBinaryPath, - acvmWorkingDirectory: proverOptions.acvmWorkingDirectory, - bbWorkingDirectory: proverOptions.bbWorkingDirectory, - }); + circuitProver = await BBNativeRollupProver.new( + { + acvmBinaryPath: proverOptions.acvmBinaryPath, + bbBinaryPath: proverOptions.bbBinaryPath, + acvmWorkingDirectory: proverOptions.acvmWorkingDirectory, + bbWorkingDirectory: proverOptions.bbWorkingDirectory, + }, + telemetry, + ); } else { - circuitProver = new TestCircuitProver(); + circuitProver = new TestCircuitProver(telemetry); } const agent = new ProverAgent(circuitProver, agentConcurrency, pollInterval); diff --git a/yarn-project/aztec/src/sandbox.ts b/yarn-project/aztec/src/sandbox.ts index d9c7f0d113d..79d6e6e3218 100644 --- a/yarn-project/aztec/src/sandbox.ts +++ b/yarn-project/aztec/src/sandbox.ts @@ -36,6 +36,8 @@ import { getCanonicalAuthRegistry } from '@aztec/protocol-contracts/auth-registr import { GasTokenAddress, getCanonicalGasToken } from '@aztec/protocol-contracts/gas-token'; import { getCanonicalKeyRegistry } from '@aztec/protocol-contracts/key-registry'; import { type PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type HDAccount, type PrivateKeyAccount, createPublicClient, http as httpViemTransport } from 'viem'; import { mnemonicToAccount } from 'viem/accounts'; @@ -252,7 +254,7 @@ export async function createSandbox(config: Partial = {}) { await deployContractsToL1(aztecNodeConfig, hdAccount); } - const node = await createAztecNode(aztecNodeConfig); + const node = await createAztecNode(new NoopTelemetryClient(), aztecNodeConfig); const pxe = await createAztecPXE(node); await deployCanonicalKeyRegistry( @@ -281,9 +283,9 @@ export async function createSandbox(config: Partial = {}) { * Create and start a new Aztec RPC HTTP Server * @param config - Optional Aztec node settings. */ -export async function createAztecNode(config: Partial = {}) { +export async function createAztecNode(telemetryClient: TelemetryClient, config: Partial = {}) { const aztecNodeConfig: AztecNodeConfig = { ...getConfigEnvVars(), ...config }; - const node = await AztecNodeService.createAndSync(aztecNodeConfig); + const node = await AztecNodeService.createAndSync(aztecNodeConfig, telemetryClient); return node; } diff --git a/yarn-project/aztec/tsconfig.json b/yarn-project/aztec/tsconfig.json index ef88fd56147..557c0080199 100644 --- a/yarn-project/aztec/tsconfig.json +++ b/yarn-project/aztec/tsconfig.json @@ -62,6 +62,9 @@ }, { "path": "../pxe" + }, + { + "path": "../telemetry-client" } ], "include": ["src"] diff --git a/yarn-project/bb-prover/package.json b/yarn-project/bb-prover/package.json index 0d46748757b..07441ca4c14 100644 --- a/yarn-project/bb-prover/package.json +++ b/yarn-project/bb-prover/package.json @@ -56,6 +56,7 @@ "@aztec/foundation": "workspace:^", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/simulator": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@noir-lang/noirc_abi": "portal:../../noir/packages/noirc_abi", "@noir-lang/types": "portal:../../noir/packages/types", "commander": "^9.0.0", diff --git a/yarn-project/bb-prover/src/bb/execute.ts b/yarn-project/bb-prover/src/bb/execute.ts index 60113d1142b..c3b28317377 100644 --- a/yarn-project/bb-prover/src/bb/execute.ts +++ b/yarn-project/bb-prover/src/bb/execute.ts @@ -21,7 +21,7 @@ export enum BB_RESULT { export type BBSuccess = { status: BB_RESULT.SUCCESS | BB_RESULT.ALREADY_PRESENT; - duration: number; + durationMs: number; /** Full path of the public key. */ pkPath?: string; /** Base directory for the VKs (raw, fields). */ @@ -155,7 +155,7 @@ export async function generateKeyForNoirCircuit( if (result.status == BB_RESULT.SUCCESS) { return { status: BB_RESULT.SUCCESS, - duration, + durationMs: duration, pkPath: key === 'pk' ? outputPath : undefined, vkPath: key === 'vk' ? outputPath : undefined, proofPath: undefined, @@ -174,7 +174,7 @@ export async function generateKeyForNoirCircuit( if (!res) { return { status: BB_RESULT.ALREADY_PRESENT, - duration: 0, + durationMs: 0, pkPath: key === 'pk' ? outputPath : undefined, vkPath: key === 'vk' ? outputPath : undefined, }; @@ -237,7 +237,7 @@ export async function generateProof( if (result.status == BB_RESULT.SUCCESS) { return { status: BB_RESULT.SUCCESS, - duration, + durationMs: duration, proofPath: `${outputPath}`, pkPath: undefined, vkPath: `${outputPath}`, @@ -346,7 +346,7 @@ export async function generateAvmProof( if (result.status == BB_RESULT.SUCCESS) { return { status: BB_RESULT.SUCCESS, - duration, + durationMs: duration, proofPath: join(outputPath, PROOF_FILENAME), pkPath: undefined, vkPath: outputPath, @@ -426,7 +426,7 @@ async function verifyProofInternal( const result = await executeBB(pathToBB, command, args, log); const duration = timer.ms(); if (result.status == BB_RESULT.SUCCESS) { - return { status: BB_RESULT.SUCCESS, duration }; + return { status: BB_RESULT.SUCCESS, durationMs: duration }; } // Not a great error message here but it is difficult to decipher what comes from bb return { @@ -466,7 +466,7 @@ export async function writeVkAsFields( const result = await executeBB(pathToBB, 'vk_as_fields', args, log); const duration = timer.ms(); if (result.status == BB_RESULT.SUCCESS) { - return { status: BB_RESULT.SUCCESS, duration, vkPath: verificationKeyPath }; + return { status: BB_RESULT.SUCCESS, durationMs: duration, vkPath: verificationKeyPath }; } // Not a great error message here but it is difficult to decipher what comes from bb return { @@ -508,7 +508,7 @@ export async function writeProofAsFields( const result = await executeBB(pathToBB, 'proof_as_fields', args, log); const duration = timer.ms(); if (result.status == BB_RESULT.SUCCESS) { - return { status: BB_RESULT.SUCCESS, duration, proofPath: proofPath }; + return { status: BB_RESULT.SUCCESS, durationMs: duration, proofPath: proofPath }; } // Not a great error message here but it is difficult to decipher what comes from bb return { @@ -549,7 +549,7 @@ export async function generateContractForVerificationKey( const result = await executeBB(pathToBB, 'contract', args, log); const duration = timer.ms(); if (result.status == BB_RESULT.SUCCESS) { - return { status: BB_RESULT.SUCCESS, duration, contractPath }; + return { status: BB_RESULT.SUCCESS, durationMs: duration, contractPath }; } // Not a great error message here but it is difficult to decipher what comes from bb return { @@ -564,7 +564,7 @@ export async function generateContractForVerificationKey( if (!res) { return { status: BB_RESULT.ALREADY_PRESENT, - duration: 0, + durationMs: 0, contractPath, }; } diff --git a/yarn-project/bb-prover/src/instrumentation.ts b/yarn-project/bb-prover/src/instrumentation.ts new file mode 100644 index 00000000000..c3adeebc345 --- /dev/null +++ b/yarn-project/bb-prover/src/instrumentation.ts @@ -0,0 +1,144 @@ +import { type CircuitName } from '@aztec/circuit-types/stats'; +import { type Timer } from '@aztec/foundation/timer'; +import { + Attributes, + type Gauge, + type Histogram, + Metrics, + type TelemetryClient, + ValueType, +} from '@aztec/telemetry-client'; + +/** + * Instrumentation class for Prover implementations. + */ +export class ProverInstrumentation { + private simulationDuration: Histogram; + private witGenDuration: Gauge; + private provingDuration: Gauge; + + private witGenInputSize: Gauge; + private witGenOutputSize: Gauge; + + private proofSize: Gauge; + private circuitSize: Gauge; + private circuitPublicInputCount: Gauge; + + constructor(telemetry: TelemetryClient, name: string) { + const meter = telemetry.getMeter(name); + this.simulationDuration = meter.createHistogram(Metrics.CIRCUIT_SIMULATION_DURATION, { + description: 'Records how long it takes to simulate a circuit', + unit: 's', + valueType: ValueType.DOUBLE, + advice: { + explicitBucketBoundaries: [0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60], + }, + }); + + this.witGenDuration = meter.createGauge(Metrics.CIRCUIT_WITNESS_GEN_DURATION, { + description: 'Records how long it takes to generate the partial witness for a circuit', + unit: 's', + valueType: ValueType.DOUBLE, + }); + + // ideally this would be a histogram, but proving takes a long time on the server + // and they don't happen that often so Prometheus & Grafana have a hard time handling it + this.provingDuration = meter.createGauge(Metrics.CIRCUIT_PROVING_DURATION, { + unit: 's', + description: 'Records how long it takes to prove a circuit', + valueType: ValueType.DOUBLE, + }); + + this.witGenInputSize = meter.createGauge(Metrics.CIRCUIT_WITNESS_GEN_INPUT_SIZE, { + unit: 'By', + description: 'Records the size of the input to the witness generation', + valueType: ValueType.INT, + }); + + this.witGenOutputSize = meter.createGauge(Metrics.CIRCUIT_WITNESS_GEN_OUTPUT_SIZE, { + unit: 'By', + description: 'Records the size of the output of the witness generation', + valueType: ValueType.INT, + }); + + this.proofSize = meter.createGauge(Metrics.CIRCUIT_PROVING_PROOF_SIZE, { + unit: 'By', + description: 'Records the size of the proof generated for a circuit', + valueType: ValueType.INT, + }); + + this.circuitPublicInputCount = meter.createGauge(Metrics.CIRCUIT_PUBLIC_INPUTS_COUNT, { + description: 'Records the number of public inputs in a circuit', + valueType: ValueType.INT, + }); + + this.circuitSize = meter.createGauge(Metrics.CIRCUIT_SIZE, { + description: 'Records the size of the circuit in gates', + valueType: ValueType.INT, + }); + } + + /** + * Records the duration of a circuit operation. + * @param metric - The metric to record + * @param circuitName - The name of the circuit + * @param timerOrS - The duration + */ + recordDuration( + metric: 'simulationDuration' | 'witGenDuration' | 'provingDuration', + circuitName: CircuitName, + timerOrS: Timer | number, + ) { + const s = typeof timerOrS === 'number' ? timerOrS : timerOrS.s(); + this[metric].record(s, { + [Attributes.PROTOCOL_CIRCUIT_NAME]: circuitName, + [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server', + }); + } + + /** + * Records the duration of an AVM circuit operation. + * @param metric - The metric to record + * @param appCircuitName - The name of the function circuit (should be a `contract:function` string) + * @param timerOrS - The duration + */ + recordAvmDuration(metric: 'witGenDuration' | 'provingDuration', appCircuitName: string, timerOrS: Timer | number) { + const s = typeof timerOrS === 'number' ? timerOrS : timerOrS.s(); + this[metric].record(s, { + [Attributes.APP_CIRCUIT_NAME]: appCircuitName, + }); + } + + /** + * Records the size of a circuit operation. + * @param metric - Records the size of a circuit operation. + * @param circuitName - The name of the circuit + * @param size - The size + */ + recordSize( + metric: 'witGenInputSize' | 'witGenOutputSize' | 'proofSize' | 'circuitSize' | 'circuitPublicInputCount', + circuitName: CircuitName, + size: number, + ) { + this[metric].record(Math.ceil(size), { + [Attributes.PROTOCOL_CIRCUIT_NAME]: circuitName, + [Attributes.PROTOCOL_CIRCUIT_TYPE]: 'server', + }); + } + + /** + * Records the size of an AVM circuit operation. + * @param metric - The metric to record + * @param appCircuitName - The name of the function circuit (should be a `contract:function` string) + * @param size - The size + */ + recordAvmSize( + metric: 'witGenInputSize' | 'witGenOutputSize' | 'proofSize' | 'circuitSize' | 'circuitPublicInputCount', + appCircuitName: string, + size: number, + ) { + this[metric].record(Math.ceil(size), { + [Attributes.APP_CIRCUIT_NAME]: appCircuitName, + }); + } +} diff --git a/yarn-project/bb-prover/src/prover/bb_native_proof_creator.ts b/yarn-project/bb-prover/src/prover/bb_native_proof_creator.ts index 3cc08097278..ea93f78fe77 100644 --- a/yarn-project/bb-prover/src/prover/bb_native_proof_creator.ts +++ b/yarn-project/bb-prover/src/prover/bb_native_proof_creator.ts @@ -176,7 +176,7 @@ export class BBNativeProofCreator implements ProofCreator { throw new Error(errorMessage); } - this.log.info(`Successfully verified ${circuitType} proof in ${Math.ceil(result.duration)} ms`); + this.log.info(`Successfully verified ${circuitType} proof in ${Math.ceil(result.durationMs)} ms`); } private async verifyProofFromKey( @@ -339,7 +339,7 @@ export class BBNativeProofCreator implements ProofCreator { this.log.debug(`Generated proof`, { eventName: 'circuit-proving', circuitName: 'app-circuit', - duration: provingResult.duration, + duration: provingResult.durationMs, inputSize: compressedBincodedWitness.length, proofSize: proof.binaryProof.buffer.length, appCircuitName, @@ -358,7 +358,7 @@ export class BBNativeProofCreator implements ProofCreator { this.log.debug(`Generated proof`, { circuitName: mapProtocolArtifactNameToCircuitName(circuitType), - duration: provingResult.duration, + duration: provingResult.durationMs, eventName: 'circuit-proving', inputSize: compressedBincodedWitness.length, proofSize: proof.binaryProof.buffer.length, diff --git a/yarn-project/bb-prover/src/prover/bb_prover.ts b/yarn-project/bb-prover/src/prover/bb_prover.ts index 33210ed6444..30b4f6f3c6b 100644 --- a/yarn-project/bb-prover/src/prover/bb_prover.ts +++ b/yarn-project/bb-prover/src/prover/bb_prover.ts @@ -57,6 +57,7 @@ import { convertRootRollupOutputsFromWitnessMap, } from '@aztec/noir-protocol-circuits-types'; import { NativeACVMSimulator } from '@aztec/simulator'; +import { type TelemetryClient } from '@aztec/telemetry-client'; import { abiEncode } from '@noir-lang/noirc_abi'; import { type Abi, type WitnessMap } from '@noir-lang/types'; @@ -78,6 +79,7 @@ import { writeProofAsFields, } from '../bb/execute.js'; import type { ACVMConfig, BBConfig } from '../config.js'; +import { ProverInstrumentation } from '../instrumentation.js'; import { PublicKernelArtifactMapping } from '../mappings/mappings.js'; import { mapProtocolArtifactNameToCircuitName } from '../stats.js'; import { extractVkData } from '../verification_key/verification_key_data.js'; @@ -102,9 +104,14 @@ export class BBNativeRollupProver implements ServerCircuitProver { ServerProtocolArtifact, Promise >(); - constructor(private config: BBProverConfig) {} - static async new(config: BBProverConfig) { + private instrumentation: ProverInstrumentation; + + constructor(private config: BBProverConfig, telemetry: TelemetryClient) { + this.instrumentation = new ProverInstrumentation(telemetry, 'BBNativeRollupProver'); + } + + static async new(config: BBProverConfig, telemetry: TelemetryClient) { await fs.access(config.acvmBinaryPath, fs.constants.R_OK); await fs.mkdir(config.acvmWorkingDirectory, { recursive: true }); await fs.access(config.bbBinaryPath, fs.constants.R_OK); @@ -112,7 +119,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { logger.info(`Using native BB at ${config.bbBinaryPath} and working directory ${config.bbWorkingDirectory}`); logger.info(`Using native ACVM at ${config.acvmBinaryPath} and working directory ${config.acvmWorkingDirectory}`); - return new BBNativeRollupProver(config); + return new BBNativeRollupProver(config, telemetry); } /** @@ -385,11 +392,16 @@ export class BBNativeRollupProver implements ServerCircuitProver { const inputWitness = convertInput(input); const timer = new Timer(); const outputWitness = await simulator.simulateCircuit(inputWitness, artifact); - const witnessGenerationDuration = timer.ms(); const output = convertOutput(outputWitness); + + const circuitName = mapProtocolArtifactNameToCircuitName(circuitType); + this.instrumentation.recordDuration('witGenDuration', circuitName, timer); + this.instrumentation.recordSize('witGenInputSize', circuitName, input.toBuffer().length); + this.instrumentation.recordSize('witGenOutputSize', circuitName, output.toBuffer().length); + logger.debug(`Generated witness`, { - circuitName: mapProtocolArtifactNameToCircuitName(circuitType), - duration: witnessGenerationDuration, + circuitName, + duration: timer.ms(), inputSize: input.toBuffer().length, outputSize: output.toBuffer().length, eventName: 'circuit-witness-generation', @@ -439,10 +451,17 @@ export class BBNativeRollupProver implements ServerCircuitProver { const rawProof = await fs.readFile(`${provingResult.proofPath!}/${PROOF_FILENAME}`); const proof = new Proof(rawProof, vkData.numPublicInputs); - logger.info(`Generated proof for ${circuitType} in ${Math.ceil(provingResult.duration)} ms`, { - circuitName: mapProtocolArtifactNameToCircuitName(circuitType), + const circuitName = mapProtocolArtifactNameToCircuitName(circuitType); + + this.instrumentation.recordDuration('provingDuration', circuitName, provingResult.durationMs / 1000); + this.instrumentation.recordSize('proofSize', circuitName, proof.buffer.length); + this.instrumentation.recordSize('circuitPublicInputCount', circuitName, vkData.numPublicInputs); + this.instrumentation.recordSize('circuitSize', circuitName, vkData.circuitSize); + + logger.info(`Generated proof for ${circuitType} in ${Math.ceil(provingResult.durationMs)} ms`, { + circuitName, // does not include reading the proof from disk - duration: provingResult.duration, + duration: provingResult.durationMs, proofSize: proof.buffer.length, eventName: 'circuit-proving', // circuitOutput is the partial witness that became the input to the proof @@ -484,13 +503,19 @@ export class BBNativeRollupProver implements ServerCircuitProver { const proof = new Proof(rawProof, verificationKey.numPublicInputs); const circuitType = 'avm-circuit' as const; + const appCircuitName = 'unknown' as const; + this.instrumentation.recordAvmDuration('provingDuration', appCircuitName, provingResult.durationMs); + this.instrumentation.recordAvmSize('proofSize', appCircuitName, proof.buffer.length); + this.instrumentation.recordAvmSize('circuitPublicInputCount', appCircuitName, verificationKey.numPublicInputs); + this.instrumentation.recordAvmSize('circuitSize', appCircuitName, verificationKey.circuitSize); + logger.info( - `Generated proof for ${circuitType}(${input.functionName}) in ${Math.ceil(provingResult.duration)} ms`, + `Generated proof for ${circuitType}(${input.functionName}) in ${Math.ceil(provingResult.durationMs)} ms`, { circuitName: circuitType, appCircuitName: input.functionName, // does not include reading the proof from disk - duration: provingResult.duration, + duration: provingResult.durationMs, proofSize: proof.buffer.length, eventName: 'circuit-proving', inputSize: input.toBuffer().length, @@ -534,14 +559,19 @@ export class BBNativeRollupProver implements ServerCircuitProver { // Read the proof as fields const proof = await this.readProofAsFields(provingResult.proofPath!, circuitType, proofLength); + const circuitName = mapProtocolArtifactNameToCircuitName(circuitType); + this.instrumentation.recordDuration('provingDuration', circuitName, provingResult.durationMs / 1000); + this.instrumentation.recordSize('proofSize', circuitName, proof.binaryProof.buffer.length); + this.instrumentation.recordSize('circuitPublicInputCount', circuitName, vkData.numPublicInputs); + this.instrumentation.recordSize('circuitSize', circuitName, vkData.circuitSize); logger.info( - `Generated proof for ${circuitType} in ${Math.ceil(provingResult.duration)} ms, size: ${ + `Generated proof for ${circuitType} in ${Math.ceil(provingResult.durationMs)} ms, size: ${ proof.proof.length } fields`, { - circuitName: mapProtocolArtifactNameToCircuitName(circuitType), + circuitName, circuitSize: vkData.circuitSize, - duration: provingResult.duration, + duration: provingResult.durationMs, inputSize: output.toBuffer().length, proofSize: proof.binaryProof.buffer.length, eventName: 'circuit-proving', @@ -603,7 +633,7 @@ export class BBNativeRollupProver implements ServerCircuitProver { throw new Error(errorMessage); } - logger.debug(`Successfully verified proof from key in ${result.duration} ms`); + logger.debug(`Successfully verified proof from key in ${result.durationMs} ms`); }; await runInDirectory(this.config.bbWorkingDirectory, operation); diff --git a/yarn-project/bb-prover/src/test/test_circuit_prover.ts b/yarn-project/bb-prover/src/test/test_circuit_prover.ts index c4c24794e8f..5fd4a9efe7e 100644 --- a/yarn-project/bb-prover/src/test/test_circuit_prover.ts +++ b/yarn-project/bb-prover/src/test/test_circuit_prover.ts @@ -57,7 +57,9 @@ import { convertSimulatedPublicTailOutputFromWitnessMap, } from '@aztec/noir-protocol-circuits-types'; import { type SimulationProvider, WASMSimulator, emitCircuitSimulationStats } from '@aztec/simulator'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { ProverInstrumentation } from '../instrumentation.js'; import { SimulatedPublicKernelArtifactMapping } from '../mappings/mappings.js'; import { mapPublicKernelToCircuitName } from '../stats.js'; @@ -81,11 +83,15 @@ const VERIFICATION_KEYS: Record */ export class TestCircuitProver implements ServerCircuitProver { private wasmSimulator = new WASMSimulator(); + private instrumentation: ProverInstrumentation; constructor( + telemetry: TelemetryClient, private simulationProvider?: SimulationProvider, private logger = createDebugLogger('aztec:test-prover'), - ) {} + ) { + this.instrumentation = new ProverInstrumentation(telemetry, 'TestCircuitProver'); + } public async getEmptyPrivateKernelProof( inputs: PrivateKernelEmptyInputData, @@ -125,6 +131,8 @@ export class TestCircuitProver implements ServerCircuitProver { result, ); + this.instrumentation.recordDuration('simulationDuration', 'base-parity', timer); + emitCircuitSimulationStats( 'base-parity', timer.ms(), @@ -158,6 +166,7 @@ export class TestCircuitProver implements ServerCircuitProver { result, ); + this.instrumentation.recordDuration('simulationDuration', 'root-parity', timer); emitCircuitSimulationStats( 'root-parity', timer.ms(), @@ -185,6 +194,7 @@ export class TestCircuitProver implements ServerCircuitProver { const result = convertSimulatedBaseRollupOutputsFromWitnessMap(witness); + this.instrumentation.recordDuration('simulationDuration', 'base-rollup', timer); emitCircuitSimulationStats( 'base-rollup', timer.ms(), @@ -214,6 +224,7 @@ export class TestCircuitProver implements ServerCircuitProver { const result = convertMergeRollupOutputsFromWitnessMap(witness); + this.instrumentation.recordDuration('simulationDuration', 'merge-rollup', timer); emitCircuitSimulationStats( 'merge-rollup', timer.ms(), @@ -244,6 +255,7 @@ export class TestCircuitProver implements ServerCircuitProver { const result = convertRootRollupOutputsFromWitnessMap(witness); + this.instrumentation.recordDuration('simulationDuration', 'root-rollup', timer); emitCircuitSimulationStats( 'root-rollup', timer.ms(), @@ -274,8 +286,10 @@ export class TestCircuitProver implements ServerCircuitProver { ); const result = kernelOps.convertOutputs(witness); + const circuitName = mapPublicKernelToCircuitName(kernelRequest.type); + this.instrumentation.recordDuration('simulationDuration', circuitName, timer); emitCircuitSimulationStats( - mapPublicKernelToCircuitName(kernelRequest.type), + circuitName, timer.ms(), kernelRequest.inputs.toBuffer().length, result.toBuffer().length, @@ -301,6 +315,7 @@ export class TestCircuitProver implements ServerCircuitProver { ); const result = convertSimulatedPublicTailOutputFromWitnessMap(witness); + this.instrumentation.recordDuration('simulationDuration', 'public-kernel-tail', timer); emitCircuitSimulationStats( 'public-kernel-tail', timer.ms(), diff --git a/yarn-project/bb-prover/tsconfig.json b/yarn-project/bb-prover/tsconfig.json index d2906818893..e0e59ed584c 100644 --- a/yarn-project/bb-prover/tsconfig.json +++ b/yarn-project/bb-prover/tsconfig.json @@ -20,6 +20,9 @@ }, { "path": "../simulator" + }, + { + "path": "../telemetry-client" } ], "include": ["src"] diff --git a/yarn-project/end-to-end/package.json b/yarn-project/end-to-end/package.json index 1167abad54a..027d9403c2d 100644 --- a/yarn-project/end-to-end/package.json +++ b/yarn-project/end-to-end/package.json @@ -40,6 +40,7 @@ "@aztec/pxe": "workspace:^", "@aztec/sequencer-client": "workspace:^", "@aztec/simulator": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@aztec/types": "workspace:^", "@aztec/world-state": "workspace:^", "@jest/globals": "^29.5.0", diff --git a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts index 4445829658c..17340f06675 100644 --- a/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts +++ b/yarn-project/end-to-end/src/composed/integration_l1_publisher.test.ts @@ -40,6 +40,7 @@ import { AvailabilityOracleAbi, InboxAbi, OutboxAbi, RollupAbi } from '@aztec/l1 import { SHA256Trunc, StandardTree } from '@aztec/merkle-tree'; import { TxProver } from '@aztec/prover-client'; import { type L1Publisher, getL1Publisher } from '@aztec/sequencer-client'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { MerkleTrees, ServerWorldStateSynchronizer, type WorldStateConfig } from '@aztec/world-state'; import { beforeEach, describe, expect, it } from '@jest/globals'; @@ -145,7 +146,7 @@ describe('L1Publisher integration', () => { }; const worldStateSynchronizer = new ServerWorldStateSynchronizer(tmpStore, builderDb, blockSource, worldStateConfig); await worldStateSynchronizer.start(); - builder = await TxProver.new(config, getMockVerificationKeys(), worldStateSynchronizer); + builder = await TxProver.new(config, getMockVerificationKeys(), worldStateSynchronizer, new NoopTelemetryClient()); l2Proof = makeEmptyProof(); publisher = getL1Publisher({ diff --git a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts index 42a5f26cb8b..024857ab6e2 100644 --- a/yarn-project/end-to-end/src/e2e_p2p_network.test.ts +++ b/yarn-project/end-to-end/src/e2e_p2p_network.test.ts @@ -13,6 +13,7 @@ import { } from '@aztec/aztec.js'; import { type BootNodeConfig, BootstrapNode, createLibP2PPeerId } from '@aztec/p2p'; import { type PXEService, createPXEService, getPXEServiceConfig as getRpcConfig } from '@aztec/pxe'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import fs from 'fs'; import { mnemonicToAccount } from 'viem/accounts'; @@ -203,7 +204,11 @@ describe('e2e_p2p_network', () => { dataDirectory, bootstrapNodes: bootstrapNode ? [bootstrapNode] : [], }; - return await AztecNodeService.createAndSync(newConfig, createDebugLogger(`aztec:node-${tcpListenPort}`)); + return await AztecNodeService.createAndSync( + newConfig, + new NoopTelemetryClient(), + createDebugLogger(`aztec:node-${tcpListenPort}`), + ); }; // creates an instance of the PXE and submit a given number of transactions to it. diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 07014915b30..03f069dd84e 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -63,6 +63,7 @@ import { getCanonicalKeyRegistry } from '@aztec/protocol-contracts/key-registry' import { type ProverClient } from '@aztec/prover-client'; import { PXEService, type PXEServiceConfig, createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import { type SequencerClient } from '@aztec/sequencer-client'; +import { createAndStartTelemetryClient, getConfigEnvVars as getTelemetryConfig } from '@aztec/telemetry-client/start'; import { type Anvil, createAnvil } from '@viem/anvil'; import getPort from 'get-port'; @@ -89,6 +90,13 @@ export { deployAndInitializeTokenAndBridgeContracts } from '../shared/cross_chai const { PXE_URL = '' } = process.env; +const telemetry = createAndStartTelemetryClient(getTelemetryConfig(), 'aztec-test'); +if (typeof afterAll === 'function') { + afterAll(async () => { + await telemetry.stop(); + }); +} + const getAztecUrl = () => { return PXE_URL; }; @@ -369,7 +377,7 @@ export async function setup( config.bbWorkingDirectory = bbConfig.bbWorkingDirectory; } config.l1BlockPublishRetryIntervalMS = 100; - const aztecNode = await AztecNodeService.createAndSync(config); + const aztecNode = await AztecNodeService.createAndSync(config, telemetry); const sequencer = aztecNode.getSequencer(); const prover = aztecNode.getProver(); diff --git a/yarn-project/end-to-end/tsconfig.json b/yarn-project/end-to-end/tsconfig.json index 7273cee65f5..28bde215732 100644 --- a/yarn-project/end-to-end/tsconfig.json +++ b/yarn-project/end-to-end/tsconfig.json @@ -66,6 +66,9 @@ { "path": "../simulator" }, + { + "path": "../telemetry-client" + }, { "path": "../types" }, diff --git a/yarn-project/p2p/package.json b/yarn-project/p2p/package.json index 0cc79a46b28..e37434e949d 100644 --- a/yarn-project/p2p/package.json +++ b/yarn-project/p2p/package.json @@ -52,6 +52,7 @@ "@aztec/circuits.js": "workspace:^", "@aztec/foundation": "workspace:^", "@aztec/kv-store": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@chainsafe/discv5": "9.0.0", "@chainsafe/enr": "3.0.0", "@chainsafe/libp2p-gossipsub": "13.0.0", diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts index 9dc6e8ddc11..4bd4f3e63f4 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.test.ts @@ -1,4 +1,5 @@ import { openTmpStore } from '@aztec/kv-store/utils'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { AztecKVTxPool } from './aztec_kv_tx_pool.js'; import { describeTxPool } from './tx_pool_test_suite.js'; @@ -6,7 +7,7 @@ import { describeTxPool } from './tx_pool_test_suite.js'; describe('In-Memory TX pool', () => { let txPool: AztecKVTxPool; beforeEach(() => { - txPool = new AztecKVTxPool(openTmpStore()); + txPool = new AztecKVTxPool(openTmpStore(), new NoopTelemetryClient()); }); describeTxPool(() => txPool); diff --git a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts index 13729720692..f3756f83713 100644 --- a/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts +++ b/yarn-project/p2p/src/tx_pool/aztec_kv_tx_pool.ts @@ -2,7 +2,9 @@ import { Tx, TxHash } from '@aztec/circuit-types'; import { type TxAddedToPoolStats } from '@aztec/circuit-types/stats'; import { type Logger, createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore, type AztecMap } from '@aztec/kv-store'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { TxPoolInstrumentation } from './instrumentation.js'; import { type TxPool } from './tx_pool.js'; /** @@ -18,15 +20,18 @@ export class AztecKVTxPool implements TxPool { #log: Logger; + #metrics: TxPoolInstrumentation; + /** * Class constructor for in-memory TxPool. Initiates our transaction pool as a JS Map. * @param store - A KV store. * @param log - A logger. */ - constructor(store: AztecKVStore, log = createDebugLogger('aztec:tx_pool')) { + constructor(store: AztecKVStore, telemetry: TelemetryClient, log = createDebugLogger('aztec:tx_pool')) { this.#txs = store.openMap('txs'); this.#store = store; this.#log = log; + this.#metrics = new TxPoolInstrumentation(telemetry, 'AztecKVTxPool'); } /** @@ -44,8 +49,8 @@ export class AztecKVTxPool implements TxPool { * @param txs - An array of txs to be added to the pool. * @returns Empty promise. */ - public async addTxs(txs: Tx[]): Promise { - const txHashes = await Promise.all(txs.map(tx => tx.getTxHash())); + public addTxs(txs: Tx[]): Promise { + const txHashes = txs.map(tx => tx.getTxHash()); return this.#store.transaction(() => { for (const [i, tx] of txs.entries()) { const txHash = txHashes[i]; @@ -56,6 +61,8 @@ export class AztecKVTxPool implements TxPool { void this.#txs.set(txHash.toString(), tx.toBuffer()); } + + this.#metrics.recordTxs(txs); }); } @@ -69,6 +76,8 @@ export class AztecKVTxPool implements TxPool { for (const hash of txHashes) { void this.#txs.delete(hash.toString()); } + + this.#metrics.removeTxs(txHashes.length); }); } diff --git a/yarn-project/p2p/src/tx_pool/instrumentation.ts b/yarn-project/p2p/src/tx_pool/instrumentation.ts new file mode 100644 index 00000000000..099afe22522 --- /dev/null +++ b/yarn-project/p2p/src/tx_pool/instrumentation.ts @@ -0,0 +1,58 @@ +import { type Tx } from '@aztec/circuit-types'; +import { type Histogram, Metrics, type TelemetryClient, type UpDownCounter } from '@aztec/telemetry-client'; + +/** + * Instrumentation class for the TxPool. + */ +export class TxPoolInstrumentation { + /** The number of txs in the mempool */ + private txInMempool: UpDownCounter; + /** Tracks tx size */ + private txSize: Histogram; + + constructor(telemetry: TelemetryClient, name: string) { + const meter = telemetry.getMeter(name); + this.txInMempool = meter.createUpDownCounter(Metrics.MEMPOOL_TX_COUNT, { + description: 'The current number of transactions in the mempool', + }); + + this.txSize = meter.createHistogram(Metrics.MEMPOOL_TX_SIZE, { + unit: 'By', + description: 'The size of transactions in the mempool', + advice: { + explicitBucketBoundaries: [ + 5_000, // 5KB + 10_000, + 20_000, + 50_000, + 75_000, + 100_000, // 100KB + 200_000, + ], + }, + }); + } + + /** + * Updates the metrics with the new transactions. + * @param txs - The transactions to record + */ + public recordTxs(txs: Tx[]) { + for (const tx of txs) { + this.txSize.record(tx.getSize()); + } + + this.txInMempool.add(txs.length); + } + + /** + * Updates the metrics by removing transactions from the mempool. + * @param count - The number of transactions to remove from the mempool + */ + public removeTxs(count = 1) { + if (count < 0) { + throw new Error('Count must be positive'); + } + this.txInMempool.add(-1 * count); + } +} diff --git a/yarn-project/p2p/src/tx_pool/memory_tx_pool.test.ts b/yarn-project/p2p/src/tx_pool/memory_tx_pool.test.ts index fb910b4755c..c4435a5613a 100644 --- a/yarn-project/p2p/src/tx_pool/memory_tx_pool.test.ts +++ b/yarn-project/p2p/src/tx_pool/memory_tx_pool.test.ts @@ -1,10 +1,12 @@ +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; + import { InMemoryTxPool } from './index.js'; import { describeTxPool } from './tx_pool_test_suite.js'; describe('In-Memory TX pool', () => { let inMemoryTxPool: InMemoryTxPool; beforeEach(() => { - inMemoryTxPool = new InMemoryTxPool(); + inMemoryTxPool = new InMemoryTxPool(new NoopTelemetryClient()); }); describeTxPool(() => inMemoryTxPool); diff --git a/yarn-project/p2p/src/tx_pool/memory_tx_pool.ts b/yarn-project/p2p/src/tx_pool/memory_tx_pool.ts index 858af51370c..924f907214f 100644 --- a/yarn-project/p2p/src/tx_pool/memory_tx_pool.ts +++ b/yarn-project/p2p/src/tx_pool/memory_tx_pool.ts @@ -1,7 +1,9 @@ import { Tx, TxHash } from '@aztec/circuit-types'; import { type TxAddedToPoolStats } from '@aztec/circuit-types/stats'; import { createDebugLogger } from '@aztec/foundation/log'; +import { type TelemetryClient } from '@aztec/telemetry-client'; +import { TxPoolInstrumentation } from './instrumentation.js'; import { type TxPool } from './tx_pool.js'; /** @@ -13,12 +15,15 @@ export class InMemoryTxPool implements TxPool { */ private txs: Map; + private metrics: TxPoolInstrumentation; + /** * Class constructor for in-memory TxPool. Initiates our transaction pool as a JS Map. * @param log - A logger. */ - constructor(private log = createDebugLogger('aztec:tx_pool')) { + constructor(telemetry: TelemetryClient, private log = createDebugLogger('aztec:tx_pool')) { this.txs = new Map(); + this.metrics = new TxPoolInstrumentation(telemetry, 'InMemoryTxPool'); } /** @@ -37,6 +42,7 @@ export class InMemoryTxPool implements TxPool { * @returns Empty promise. */ public addTxs(txs: Tx[]): Promise { + this.metrics.recordTxs(txs); for (const tx of txs) { const txHash = tx.getTxHash(); this.log.debug(`Adding tx with id ${txHash.toString()}`, { @@ -54,6 +60,7 @@ export class InMemoryTxPool implements TxPool { * @returns The number of transactions that was deleted from the pool. */ public deleteTxs(txHashes: TxHash[]): Promise { + this.metrics.removeTxs(txHashes.length); for (const txHash of txHashes) { this.txs.delete(txHash.toBigInt()); } diff --git a/yarn-project/p2p/tsconfig.json b/yarn-project/p2p/tsconfig.json index 4e0866fd521..fcbafbb11d0 100644 --- a/yarn-project/p2p/tsconfig.json +++ b/yarn-project/p2p/tsconfig.json @@ -17,6 +17,9 @@ }, { "path": "../kv-store" + }, + { + "path": "../telemetry-client" } ], "include": ["src"] diff --git a/yarn-project/package.json b/yarn-project/package.json index 388c8f4d6df..f9a72eb8eea 100644 --- a/yarn-project/package.json +++ b/yarn-project/package.json @@ -52,7 +52,8 @@ "scripts", "types", "txe", - "world-state" + "world-state", + "telemetry-client" ], "prettier": "@aztec/foundation/prettier", "devDependencies": { diff --git a/yarn-project/prover-client/package.json b/yarn-project/prover-client/package.json index 85080060741..87cd8921482 100644 --- a/yarn-project/prover-client/package.json +++ b/yarn-project/prover-client/package.json @@ -57,6 +57,7 @@ "@aztec/kv-store": "workspace:^", "@aztec/noir-protocol-circuits-types": "workspace:^", "@aztec/simulator": "workspace:^", + "@aztec/telemetry-client": "workspace:^", "@aztec/world-state": "workspace:^", "@noir-lang/types": "portal:../../noir/packages/types", "commander": "^9.0.0", diff --git a/yarn-project/prover-client/src/mocks/test_context.ts b/yarn-project/prover-client/src/mocks/test_context.ts index 1af8c556c48..32e0fcae8be 100644 --- a/yarn-project/prover-client/src/mocks/test_context.ts +++ b/yarn-project/prover-client/src/mocks/test_context.ts @@ -31,6 +31,7 @@ import { WASMSimulator, type WorldStatePublicDB, } from '@aztec/simulator'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { type MerkleTreeOperations, MerkleTrees } from '@aztec/world-state'; import * as fs from 'fs/promises'; @@ -85,7 +86,7 @@ export class TestContext { logger: DebugLogger, proverCount = 4, createProver: (bbConfig: BBProverConfig) => Promise = _ => - Promise.resolve(new TestCircuitProver(new WASMSimulator())), + Promise.resolve(new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator())), blockNumber = 3, ) { const globalVariables = makeGlobals(blockNumber); @@ -112,7 +113,7 @@ export class TestContext { acvmBinaryPath: config?.expectedAcvmPath, }); if (!config) { - localProver = new TestCircuitProver(simulationProvider); + localProver = new TestCircuitProver(new NoopTelemetryClient(), simulationProvider); } else { const bbConfig: BBProverConfig = { acvmBinaryPath: config.expectedAcvmPath, diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts index 2c6a6b52118..e61cf418c3c 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_failures.test.ts @@ -2,6 +2,7 @@ import { PROVING_STATUS, type ServerCircuitProver } from '@aztec/circuit-types'; import { getMockVerificationKeys } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { WASMSimulator } from '@aztec/simulator'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; @@ -28,7 +29,7 @@ describe('prover/orchestrator/failures', () => { let mockProver: ServerCircuitProver; beforeEach(() => { - mockProver = new TestCircuitProver(new WASMSimulator()); + mockProver = new TestCircuitProver(new NoopTelemetryClient(), new WASMSimulator()); orchestrator = new ProvingOrchestrator(context.actualDb, mockProver); }); diff --git a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts index 3e68baee196..715211200cc 100644 --- a/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts +++ b/yarn-project/prover-client/src/orchestrator/orchestrator_lifecycle.test.ts @@ -10,6 +10,7 @@ import { range } from '@aztec/foundation/array'; import { createDebugLogger } from '@aztec/foundation/log'; import { type PromiseWithResolvers, promiseWithResolvers } from '@aztec/foundation/promise'; import { sleep } from '@aztec/foundation/sleep'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { jest } from '@jest/globals'; @@ -141,7 +142,7 @@ describe('prover/orchestrator/lifecycle', () => { }, 60000); it('cancels proving requests', async () => { - const prover: ServerCircuitProver = new TestCircuitProver(); + const prover: ServerCircuitProver = new TestCircuitProver(new NoopTelemetryClient()); const orchestrator = new ProvingOrchestrator(context.actualDb, prover); const spy = jest.spyOn(prover, 'getBaseParityProof'); diff --git a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts index 0f41135091f..2bc202a947b 100644 --- a/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_base_rollup.test.ts @@ -1,6 +1,7 @@ import { BBNativeRollupProver, type BBProverConfig } from '@aztec/bb-prover'; import { makePaddingProcessedTx } from '@aztec/circuit-types'; import { createDebugLogger } from '@aztec/foundation/log'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { TestContext } from '../mocks/test_context.js'; import { buildBaseRollupInput } from '../orchestrator/block-building-helpers.js'; @@ -13,7 +14,7 @@ describe('prover/bb_prover/base-rollup', () => { beforeAll(async () => { const buildProver = async (bbConfig: BBProverConfig) => { - prover = await BBNativeRollupProver.new(bbConfig); + prover = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return prover; }; context = await TestContext.new(logger, 1, buildProver); diff --git a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts index f7e6ad99910..5b6791a1e58 100644 --- a/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_full_rollup.test.ts @@ -4,6 +4,7 @@ import { Fr, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP, getMockVerificationKeys } from import { makeTuple } from '@aztec/foundation/array'; import { times } from '@aztec/foundation/collection'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { TestContext } from '../mocks/test_context.js'; @@ -14,7 +15,7 @@ describe('prover/bb_prover/full-rollup', () => { beforeAll(async () => { const buildProver = async (bbConfig: BBProverConfig) => { - prover = await BBNativeRollupProver.new(bbConfig); + prover = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return prover; }; logger = createDebugLogger('aztec:bb-prover-full-rollup'); diff --git a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts index 595723e49db..b43f1c8aafd 100644 --- a/yarn-project/prover-client/src/test/bb_prover_parity.test.ts +++ b/yarn-project/prover-client/src/test/bb_prover_parity.test.ts @@ -15,6 +15,7 @@ import { makeTuple } from '@aztec/foundation/array'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { type Tuple } from '@aztec/foundation/serialize'; +import { NoopTelemetryClient } from '@aztec/telemetry-client/noop'; import { TestContext } from '../mocks/test_context.js'; @@ -27,7 +28,7 @@ describe('prover/bb_prover/parity', () => { beforeAll(async () => { const buildProver = async (bbConfig: BBProverConfig) => { bbConfig.circuitFilter = ['BaseParityArtifact', 'RootParityArtifact']; - bbProver = await BBNativeRollupProver.new(bbConfig); + bbProver = await BBNativeRollupProver.new(bbConfig, new NoopTelemetryClient()); return bbProver; }; context = await TestContext.new(logger, 1, buildProver); diff --git a/yarn-project/prover-client/src/tx-prover/tx-prover.ts b/yarn-project/prover-client/src/tx-prover/tx-prover.ts index 6008cfe9db5..94353ae6c87 100644 --- a/yarn-project/prover-client/src/tx-prover/tx-prover.ts +++ b/yarn-project/prover-client/src/tx-prover/tx-prover.ts @@ -9,6 +9,7 @@ import { } from '@aztec/circuit-types/interfaces'; import { type Fr, type GlobalVariables, type Header, type VerificationKeys } from '@aztec/circuits.js'; import { NativeACVMSimulator } from '@aztec/simulator'; +import { type TelemetryClient } from '@aztec/telemetry-client'; import { type WorldStateSynchronizer } from '@aztec/world-state'; import { type ProverClientConfig } from '../config.js'; @@ -28,6 +29,7 @@ export class TxProver implements ProverClient { private config: ProverClientConfig, private worldStateSynchronizer: WorldStateSynchronizer, private vks: VerificationKeys, + private telemetry: TelemetryClient, private agent?: ProverAgent, initialHeader?: Header, ) { @@ -43,7 +45,7 @@ export class TxProver implements ProverClient { } if (newConfig.realProofs !== this.config.realProofs && this.agent) { - const circuitProver = await TxProver.buildCircuitProver(newConfig); + const circuitProver = await TxProver.buildCircuitProver(newConfig, this.telemetry); this.agent.setCircuitProver(circuitProver); } @@ -95,31 +97,35 @@ export class TxProver implements ProverClient { config: ProverClientConfig, vks: VerificationKeys, worldStateSynchronizer: WorldStateSynchronizer, + telemetry: TelemetryClient, initialHeader?: Header, ) { const agent = config.proverAgentEnabled ? new ProverAgent( - await TxProver.buildCircuitProver(config), + await TxProver.buildCircuitProver(config, telemetry), config.proverAgentConcurrency, config.proverAgentPollInterval, ) : undefined; - const prover = new TxProver(config, worldStateSynchronizer, vks, agent, initialHeader); + const prover = new TxProver(config, worldStateSynchronizer, vks, telemetry, agent, initialHeader); await prover.start(); return prover; } - private static async buildCircuitProver(config: ProverClientConfig): Promise { + private static async buildCircuitProver( + config: ProverClientConfig, + telemetry: TelemetryClient, + ): Promise { if (config.realProofs) { - return await BBNativeRollupProver.new(config); + return await BBNativeRollupProver.new(config, telemetry); } const simulationProvider = config.acvmBinaryPath ? new NativeACVMSimulator(config.acvmWorkingDirectory, config.acvmBinaryPath) : undefined; - return new TestCircuitProver(simulationProvider); + return new TestCircuitProver(telemetry, simulationProvider); } /** diff --git a/yarn-project/prover-client/tsconfig.json b/yarn-project/prover-client/tsconfig.json index 5f4666ebf03..9a0e67ac6c2 100644 --- a/yarn-project/prover-client/tsconfig.json +++ b/yarn-project/prover-client/tsconfig.json @@ -27,6 +27,9 @@ { "path": "../simulator" }, + { + "path": "../telemetry-client" + }, { "path": "../world-state" } diff --git a/yarn-project/telemetry-client/.eslintrc.cjs b/yarn-project/telemetry-client/.eslintrc.cjs new file mode 100644 index 00000000000..e659927475c --- /dev/null +++ b/yarn-project/telemetry-client/.eslintrc.cjs @@ -0,0 +1 @@ +module.exports = require('@aztec/foundation/eslint'); diff --git a/yarn-project/telemetry-client/package.json b/yarn-project/telemetry-client/package.json new file mode 100644 index 00000000000..4c07997cc50 --- /dev/null +++ b/yarn-project/telemetry-client/package.json @@ -0,0 +1,67 @@ +{ + "name": "@aztec/telemetry-client", + "inherits": [ + "../package.common.json" + ], + "type": "module", + "exports": { + ".": "./dest/index.js", + "./start": "./dest/start.js", + "./noop": "./dest/noop.js" + }, + "scripts": { + "build": "yarn clean && tsc -b", + "build:dev": "tsc -b --watch", + "clean": "rm -rf ./dest .tsbuildinfo", + "formatting": "run -T prettier --check ./src && run -T eslint ./src", + "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src", + "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests" + }, + "engines": { + "node": ">=18" + }, + "files": [ + "dest", + "src", + "!*.test.*" + ], + "dependencies": { + "@aztec/foundation": "workspace:^", + "@opentelemetry/api": "^1.9.0", + "@opentelemetry/exporter-metrics-otlp-http": "^0.52.0", + "@opentelemetry/host-metrics": "^0.35.2", + "@opentelemetry/resources": "^1.25.0", + "@opentelemetry/sdk-metrics": "^1.25.0", + "@opentelemetry/semantic-conventions": "^1.25.0" + }, + "devDependencies": { + "@jest/globals": "^29.5.0", + "@types/jest": "^29.5.0", + "jest": "^29.5.0", + "ts-node": "^10.9.1", + "typescript": "^5.0.4" + }, + "jest": { + "extensionsToTreatAsEsm": [ + ".ts" + ], + "transform": { + "^.+\\.tsx?$": [ + "@swc/jest" + ] + }, + "moduleNameMapper": { + "^(\\.{1,2}/.*)\\.[cm]?js$": "$1" + }, + "reporters": [ + [ + "default", + { + "summaryThreshold": 9999 + } + ] + ], + "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$", + "rootDir": "./src" + } +} diff --git a/yarn-project/telemetry-client/src/attributes.ts b/yarn-project/telemetry-client/src/attributes.ts new file mode 100644 index 00000000000..bf39983a967 --- /dev/null +++ b/yarn-project/telemetry-client/src/attributes.ts @@ -0,0 +1,36 @@ +/** + * @overview This file contains the custom attributes used in telemetry events. + * Attribute names exist in a global namespace, alongside metric names. Use this file to ensure that attribute names are unique. + * + * To define a new attribute follow these steps: + * 1. Make sure it's not a semantic attribute that's already been defined by {@link @opentelemetry/semantic-conventions | OpenTelemetry} (e.g. `service.name`) + * 2. Come up with a unique name for it so that it doesn't clash with other attributes or metrics. + * 3. Prefix the attribute name with `aztec` to make it clear that it's a custom attribute. + * 4. Add a description of what the attribute represents and examples of what it might contain. + * 5. Start using it. + * + * @note Attributes and metric names exist in a hierarchy of namespaces. If a name has been used as a namespace, then it can not be used as a name for an attribute or metric. + * @example If `aztec.circuit.name` has been defined as an attribute then `aztec.circuit` alone can not be re-used for a metric or attribute because it is already a namespace. + * @see {@link https://opentelemetry.io/docs/specs/semconv/general/attribute-naming/} + */ + +/** + * The name of the protocol circuit being run (e.g. public-kernel-setup or base-rollup) + * @see {@link @aztec/circuit-types/stats:CircuitName} + */ +export const PROTOCOL_CIRCUIT_NAME = 'aztec.circuit.protocol_circuit_name'; + +/** + * The type of protocol circuit being run: server or client + */ +export const PROTOCOL_CIRCUIT_TYPE = 'aztec.circuit.protocol_circuit_type'; + +/** + * For an app circuit, the contract:function being run (e.g. Token:transfer) + */ +export const APP_CIRCUIT_NAME = 'aztec.circuit.app_circuit_name'; + +/** + * The type of app circuit being run: server or client + */ +export const APP_CIRCUIT_TYPE = 'aztec.circuit.app_circuit_type'; diff --git a/yarn-project/telemetry-client/src/index.ts b/yarn-project/telemetry-client/src/index.ts new file mode 100644 index 00000000000..f84f46bf75c --- /dev/null +++ b/yarn-project/telemetry-client/src/index.ts @@ -0,0 +1 @@ +export * from './telemetry.js'; diff --git a/yarn-project/telemetry-client/src/metrics.ts b/yarn-project/telemetry-client/src/metrics.ts new file mode 100644 index 00000000000..e5487ef41b3 --- /dev/null +++ b/yarn-project/telemetry-client/src/metrics.ts @@ -0,0 +1,30 @@ +/** + * @file Metric names used in Aztec. + * Metric names must be unique and not clash with {@link attributes.ts | Attribute names}. + * Prefix metric names with `aztec` and use dots `.` to separate namespaces. + * + * @see {@link https://opentelemetry.io/docs/specs/semconv/general/metrics/ | OpenTelemetry Metrics} for naming conventions. + */ + +/** How long it takes to simulate a circuit */ +export const CIRCUIT_SIMULATION_DURATION = 'aztec.circuit.simulation.duration'; +export const CIRCUIT_SIMULATION_INPUT_SIZE = 'aztec.circuit.simulation.input_size'; +export const CIRCUIT_SIMULATION_OUTPUT_SIZE = 'aztec.circuit.simulation.output_size'; + +export const CIRCUIT_WITNESS_GEN_DURATION = 'aztec.circuit.witness_generation.duration'; +export const CIRCUIT_WITNESS_GEN_INPUT_SIZE = 'aztec.circuit.witness_generation.input_size'; +export const CIRCUIT_WITNESS_GEN_OUTPUT_SIZE = 'aztec.circuit.witness_generation.output_size'; + +export const CIRCUIT_PROVING_DURATION = 'aztec.circuit.proving.duration'; +export const CIRCUIT_PROVING_INPUT_SIZE = 'aztec.circuit.proving.input_size'; +export const CIRCUIT_PROVING_PROOF_SIZE = 'aztec.circuit.proving.proof_size'; + +export const CIRCUIT_PUBLIC_INPUTS_COUNT = 'aztec.circuit.public_inputs_count'; +export const CIRCUIT_GATE_COUNT = 'aztec.circuit.gate_count'; +export const CIRCUIT_SIZE = 'aztec.circuit.size'; + +export const MEMPOOL_TX_COUNT = 'aztec.mempool.tx_count'; +export const MEMPOOL_TX_SIZE = 'aztec.mempool.tx_size'; + +export const ARCHIVER_BLOCK_HEIGHT = 'aztec.archiver.block_height'; +export const ARCHIVER_BLOCK_SIZE = 'aztec.archiver.block_size'; diff --git a/yarn-project/telemetry-client/src/noop.ts b/yarn-project/telemetry-client/src/noop.ts new file mode 100644 index 00000000000..2ef3c8344e1 --- /dev/null +++ b/yarn-project/telemetry-client/src/noop.ts @@ -0,0 +1,13 @@ +import { type Meter, createNoopMeter } from '@opentelemetry/api'; + +import { type TelemetryClient } from './telemetry.js'; + +export class NoopTelemetryClient implements TelemetryClient { + getMeter(): Meter { + return createNoopMeter(); + } + + stop(): Promise { + return Promise.resolve(); + } +} diff --git a/yarn-project/telemetry-client/src/otel.ts b/yarn-project/telemetry-client/src/otel.ts new file mode 100644 index 00000000000..002b70ca9c9 --- /dev/null +++ b/yarn-project/telemetry-client/src/otel.ts @@ -0,0 +1,53 @@ +import { type Meter } from '@opentelemetry/api'; +import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http'; +import { HostMetrics } from '@opentelemetry/host-metrics'; +import { Resource } from '@opentelemetry/resources'; +import { MeterProvider, PeriodicExportingMetricReader } from '@opentelemetry/sdk-metrics'; +import { SEMRESATTRS_SERVICE_NAME, SEMRESATTRS_SERVICE_VERSION } from '@opentelemetry/semantic-conventions'; + +import { type TelemetryClient } from './telemetry.js'; + +export class OpenTelemetryClient implements TelemetryClient { + hostMetrics: HostMetrics | undefined; + protected constructor(private resource: Resource, private meterProvider: MeterProvider) {} + + getMeter(name: string): Meter { + return this.meterProvider.getMeter(name, this.resource.attributes[SEMRESATTRS_SERVICE_VERSION] as string); + } + + public start() { + this.hostMetrics = new HostMetrics({ + name: this.resource.attributes[SEMRESATTRS_SERVICE_NAME] as string, + meterProvider: this.meterProvider, + }); + + this.hostMetrics.start(); + } + + public async stop() { + await Promise.all([this.meterProvider.shutdown()]); + } + + public static createAndStart(name: string, version: string, collectorBaseUrl: URL): OpenTelemetryClient { + const resource = new Resource({ + [SEMRESATTRS_SERVICE_NAME]: name, + [SEMRESATTRS_SERVICE_VERSION]: version, + }); + + const meterProvider = new MeterProvider({ + resource, + readers: [ + new PeriodicExportingMetricReader({ + exporter: new OTLPMetricExporter({ + url: new URL('/v1/metrics', collectorBaseUrl).href, + }), + }), + ], + }); + + const service = new OpenTelemetryClient(resource, meterProvider); + service.start(); + + return service; + } +} diff --git a/yarn-project/telemetry-client/src/start.ts b/yarn-project/telemetry-client/src/start.ts new file mode 100644 index 00000000000..f811f9c6ec6 --- /dev/null +++ b/yarn-project/telemetry-client/src/start.ts @@ -0,0 +1,27 @@ +import { NoopTelemetryClient } from './noop.js'; +import { OpenTelemetryClient } from './otel.js'; +import { type TelemetryClient } from './telemetry.js'; + +export interface TelemetryClientConfig { + collectorBaseUrl?: URL; +} + +export function createAndStartTelemetryClient( + config: TelemetryClientConfig, + serviceName: string, + serviceVersion?: string, +): TelemetryClient { + if (config.collectorBaseUrl) { + return OpenTelemetryClient.createAndStart(serviceName, serviceVersion ?? '0.0.0', config.collectorBaseUrl); + } else { + return new NoopTelemetryClient(); + } +} + +export function getConfigEnvVars(): TelemetryClientConfig { + const { OTEL_COLLECTOR_BASE_URL } = process.env; + + return { + collectorBaseUrl: OTEL_COLLECTOR_BASE_URL ? new URL(OTEL_COLLECTOR_BASE_URL) : undefined, + }; +} diff --git a/yarn-project/telemetry-client/src/telemetry.ts b/yarn-project/telemetry-client/src/telemetry.ts new file mode 100644 index 00000000000..0aa7dedcbe1 --- /dev/null +++ b/yarn-project/telemetry-client/src/telemetry.ts @@ -0,0 +1,69 @@ +import { + type AttributeValue, + type MetricOptions, + type Gauge as OtelGauge, + type Histogram as OtelHistogram, + type UpDownCounter as OtelUpDownCounter, +} from '@opentelemetry/api'; + +import * as Attributes from './attributes.js'; +import * as Metrics from './metrics.js'; + +export { ValueType } from '@opentelemetry/api'; + +type ValuesOf = T extends Record ? U : never; + +/** Global registry of attributes */ +type Attributes = Partial, AttributeValue>>; +export { Attributes }; + +/** Global registry of metrics */ +type Metrics = (typeof Metrics)[keyof typeof Metrics]; +export { Metrics }; + +export type Gauge = OtelGauge; +export type Histogram = OtelHistogram; +export type UpDownCounter = OtelUpDownCounter; + +// INTERNAL NOTE: this interface is the same as opentelemetry's Meter, but with proper types +/** + * A meter that provides instruments for recording metrics. + */ +export interface Meter { + /** + * Creates a new gauge instrument. A gauge is a metric that represents a single numerical value that can arbitrarily go up and down. + * @param name - The name of the gauge + * @param options - The options for the gauge + */ + createGauge(name: Metrics, options?: MetricOptions): Gauge; + + /** + * Creates a new histogram instrument. A histogram is a metric that samples observations (usually things like request durations or response sizes) and counts them in configurable buckets. + * @param name - The name of the histogram + * @param options - The options for the histogram + */ + createHistogram(name: Metrics, options?: MetricOptions): Histogram; + + /** + * Creates a new counter instrument. A counter can go up or down with a delta from the previous value. + * @param name - The name of the counter + * @param options - The options for the counter + */ + createUpDownCounter(name: Metrics, options?: MetricOptions): UpDownCounter; +} + +/** + * A telemetry client that provides meters for recording metrics. + */ +export interface TelemetryClient { + /** + * Creates a new meter + * @param name - The name of the meter. + */ + getMeter(name: string): Meter; + + /** + * Stops the telemetry client. + */ + stop(): Promise; +} diff --git a/yarn-project/telemetry-client/tsconfig.json b/yarn-project/telemetry-client/tsconfig.json new file mode 100644 index 00000000000..63f8ab3e9f7 --- /dev/null +++ b/yarn-project/telemetry-client/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "..", + "compilerOptions": { + "outDir": "dest", + "rootDir": "src", + "tsBuildInfoFile": ".tsbuildinfo" + }, + "references": [ + { + "path": "../foundation" + } + ], + "include": ["src"] +} diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 3b6e8061c0d..2e13e78b916 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -57,6 +57,7 @@ __metadata: "@aztec/l1-artifacts": "workspace:^" "@aztec/noir-contracts.js": "workspace:^" "@aztec/protocol-contracts": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@aztec/types": "workspace:^" "@jest/globals": ^29.5.0 "@types/debug": ^4.1.7 @@ -119,6 +120,7 @@ __metadata: "@aztec/prover-client": "workspace:^" "@aztec/sequencer-client": "workspace:^" "@aztec/simulator": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@aztec/types": "workspace:^" "@aztec/world-state": "workspace:^" "@jest/globals": ^29.5.0 @@ -206,6 +208,7 @@ __metadata: "@aztec/protocol-contracts": "workspace:^" "@aztec/prover-client": "workspace:^" "@aztec/pxe": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0 "@types/koa": ^2.13.6 @@ -233,6 +236,7 @@ __metadata: "@aztec/foundation": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/simulator": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@jest/globals": ^29.5.0 "@noir-lang/noirc_abi": "portal:../../noir/packages/noirc_abi" "@noir-lang/types": "portal:../../noir/packages/types" @@ -422,6 +426,7 @@ __metadata: "@aztec/pxe": "workspace:^" "@aztec/sequencer-client": "workspace:^" "@aztec/simulator": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@aztec/types": "workspace:^" "@aztec/world-state": "workspace:^" "@jest/globals": ^29.5.0 @@ -706,6 +711,7 @@ __metadata: "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@aztec/kv-store": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@chainsafe/discv5": 9.0.0 "@chainsafe/enr": 3.0.0 "@chainsafe/libp2p-gossipsub": 13.0.0 @@ -774,6 +780,7 @@ __metadata: "@aztec/kv-store": "workspace:^" "@aztec/noir-protocol-circuits-types": "workspace:^" "@aztec/simulator": "workspace:^" + "@aztec/telemetry-client": "workspace:^" "@aztec/world-state": "workspace:^" "@jest/globals": ^29.5.0 "@noir-lang/types": "portal:../../noir/packages/types" @@ -931,6 +938,25 @@ __metadata: languageName: unknown linkType: soft +"@aztec/telemetry-client@workspace:^, @aztec/telemetry-client@workspace:telemetry-client": + version: 0.0.0-use.local + resolution: "@aztec/telemetry-client@workspace:telemetry-client" + dependencies: + "@aztec/foundation": "workspace:^" + "@jest/globals": ^29.5.0 + "@opentelemetry/api": ^1.9.0 + "@opentelemetry/exporter-metrics-otlp-http": ^0.52.0 + "@opentelemetry/host-metrics": ^0.35.2 + "@opentelemetry/resources": ^1.25.0 + "@opentelemetry/sdk-metrics": ^1.25.0 + "@opentelemetry/semantic-conventions": ^1.25.0 + "@types/jest": ^29.5.0 + jest: ^29.5.0 + ts-node: ^10.9.1 + typescript: ^5.0.4 + languageName: unknown + linkType: soft + "@aztec/txe@workspace:txe": version: 0.0.0-use.local resolution: "@aztec/txe@workspace:txe" @@ -3025,6 +3051,147 @@ __metadata: languageName: node linkType: hard +"@opentelemetry/api-logs@npm:0.52.0": + version: 0.52.0 + resolution: "@opentelemetry/api-logs@npm:0.52.0" + dependencies: + "@opentelemetry/api": ^1.0.0 + checksum: 502f60fd3a4b08fb7e54eaf22d0415e34dcbc9995696945eff8a4a12910e933149900cc470fb476b9411b4bbb98f8b598e3f4d4a37137698fcf0a7ea6ab240d6 + languageName: node + linkType: hard + +"@opentelemetry/api@npm:^1.0.0, @opentelemetry/api@npm:^1.9.0": + version: 1.9.0 + resolution: "@opentelemetry/api@npm:1.9.0" + checksum: 9e88e59d53ced668f3daaecfd721071c5b85a67dd386f1c6f051d1be54375d850016c881f656ffbe9a03bedae85f7e89c2f2b635313f9c9b195ad033cdc31020 + languageName: node + linkType: hard + +"@opentelemetry/core@npm:1.25.0": + version: 1.25.0 + resolution: "@opentelemetry/core@npm:1.25.0" + dependencies: + "@opentelemetry/semantic-conventions": 1.25.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.10.0" + checksum: 46a851081e95ff1b9e3f8b518d064fd25c342522f11f0a082a9692bbfbcd947ed6602372f370fab48f8cbc8ebd7358dfa094e6d31bd26f4696b9bde418296045 + languageName: node + linkType: hard + +"@opentelemetry/exporter-metrics-otlp-http@npm:^0.52.0": + version: 0.52.0 + resolution: "@opentelemetry/exporter-metrics-otlp-http@npm:0.52.0" + dependencies: + "@opentelemetry/core": 1.25.0 + "@opentelemetry/otlp-exporter-base": 0.52.0 + "@opentelemetry/otlp-transformer": 0.52.0 + "@opentelemetry/resources": 1.25.0 + "@opentelemetry/sdk-metrics": 1.25.0 + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 8438733189879e3162ab4a374d7f22a4f9655257cbcde156f1041954cbc86bfab7299e696df49187684f1c219a76b263e6489c411b7008b81a05d5b0e7dcd92d + languageName: node + linkType: hard + +"@opentelemetry/host-metrics@npm:^0.35.2": + version: 0.35.2 + resolution: "@opentelemetry/host-metrics@npm:0.35.2" + dependencies: + "@opentelemetry/sdk-metrics": ^1.8.0 + systeminformation: 5.22.9 + peerDependencies: + "@opentelemetry/api": ^1.3.0 + checksum: 541df2585f9cbf8b6606f6782a2d351383f7a5b0a92b92ad4011ac46adac513474463d0c2474d6902d9d6d3b633be67c60ea0716ea2de277cebc1cb2538fa7a4 + languageName: node + linkType: hard + +"@opentelemetry/otlp-exporter-base@npm:0.52.0": + version: 0.52.0 + resolution: "@opentelemetry/otlp-exporter-base@npm:0.52.0" + dependencies: + "@opentelemetry/core": 1.25.0 + "@opentelemetry/otlp-transformer": 0.52.0 + peerDependencies: + "@opentelemetry/api": ^1.0.0 + checksum: 5230ba86d274f4d05fa2820a21e8278d796a299299e2af96150085c871427fe5ef4c6fa4954cdc1b8cdd0a87d5d6677ca0e547cc51253968572a6ede51f63ea2 + languageName: node + linkType: hard + +"@opentelemetry/otlp-transformer@npm:0.52.0": + version: 0.52.0 + resolution: "@opentelemetry/otlp-transformer@npm:0.52.0" + dependencies: + "@opentelemetry/api-logs": 0.52.0 + "@opentelemetry/core": 1.25.0 + "@opentelemetry/resources": 1.25.0 + "@opentelemetry/sdk-logs": 0.52.0 + "@opentelemetry/sdk-metrics": 1.25.0 + "@opentelemetry/sdk-trace-base": 1.25.0 + protobufjs: ^7.3.0 + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.10.0" + checksum: 5f75f41a710e5e536faecdec7b1687352e450d185d12613bbcbb206570d96ca2833db15e1d7945cb27040a04c017135b07df2f607ccf9ca9a061f86ad87e8c35 + languageName: node + linkType: hard + +"@opentelemetry/resources@npm:1.25.0, @opentelemetry/resources@npm:^1.25.0": + version: 1.25.0 + resolution: "@opentelemetry/resources@npm:1.25.0" + dependencies: + "@opentelemetry/core": 1.25.0 + "@opentelemetry/semantic-conventions": 1.25.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.10.0" + checksum: 6b9e59b7fc70944b418a1ae61396ec82d80869b2918bc664e3bd6d302ddc217e2e8fc5e37bcbd04bac46234f2057a005fa2a657caa1288a5c4ab7b697b0665cb + languageName: node + linkType: hard + +"@opentelemetry/sdk-logs@npm:0.52.0": + version: 0.52.0 + resolution: "@opentelemetry/sdk-logs@npm:0.52.0" + dependencies: + "@opentelemetry/api-logs": 0.52.0 + "@opentelemetry/core": 1.25.0 + "@opentelemetry/resources": 1.25.0 + peerDependencies: + "@opentelemetry/api": ">=1.4.0 <1.10.0" + checksum: 7bf7aed40a168866d76e2260237f6cec9c82acaebcc02a3597985b2be644e4aebf69e0f57739e7fd7cc8e75ecd0bdc98b0429ea985d7de6064148477ffd6432e + languageName: node + linkType: hard + +"@opentelemetry/sdk-metrics@npm:1.25.0, @opentelemetry/sdk-metrics@npm:^1.25.0, @opentelemetry/sdk-metrics@npm:^1.8.0": + version: 1.25.0 + resolution: "@opentelemetry/sdk-metrics@npm:1.25.0" + dependencies: + "@opentelemetry/core": 1.25.0 + "@opentelemetry/resources": 1.25.0 + lodash.merge: ^4.6.2 + peerDependencies: + "@opentelemetry/api": ">=1.3.0 <1.10.0" + checksum: dcb3e80bb41f937db77cb2a91574e2e434875b1740fdcff657d4223ce40002039dac915640a981deada86d53961607150b52fe32497b19c6a17dfd5fb9ed3f05 + languageName: node + linkType: hard + +"@opentelemetry/sdk-trace-base@npm:1.25.0": + version: 1.25.0 + resolution: "@opentelemetry/sdk-trace-base@npm:1.25.0" + dependencies: + "@opentelemetry/core": 1.25.0 + "@opentelemetry/resources": 1.25.0 + "@opentelemetry/semantic-conventions": 1.25.0 + peerDependencies: + "@opentelemetry/api": ">=1.0.0 <1.10.0" + checksum: 4c0ce40dbe9dcf5e5f79c60c44ffadb6806f1a8cf45c13d901ea6a2345f6cf26a83a1dad4358859fcf941e01f8bd8654f907f88137d5051e023211f8d645e959 + languageName: node + linkType: hard + +"@opentelemetry/semantic-conventions@npm:1.25.0, @opentelemetry/semantic-conventions@npm:^1.25.0": + version: 1.25.0 + resolution: "@opentelemetry/semantic-conventions@npm:1.25.0" + checksum: 8c9d36f57f0d3d1d4945effe626894ffea860b4be4d5257666ee28b90843ce22694c5b01f9b25ed47a08043958b7e89a65b7ae8e4128f5ed72dcdfe71ac7a19a + languageName: node + linkType: hard + "@pkgjs/parseargs@npm:^0.11.0": version: 0.11.0 resolution: "@pkgjs/parseargs@npm:0.11.0" @@ -3032,6 +3199,79 @@ __metadata: languageName: node linkType: hard +"@protobufjs/aspromise@npm:^1.1.1, @protobufjs/aspromise@npm:^1.1.2": + version: 1.1.2 + resolution: "@protobufjs/aspromise@npm:1.1.2" + checksum: 011fe7ef0826b0fd1a95935a033a3c0fd08483903e1aa8f8b4e0704e3233406abb9ee25350ec0c20bbecb2aad8da0dcea58b392bbd77d6690736f02c143865d2 + languageName: node + linkType: hard + +"@protobufjs/base64@npm:^1.1.2": + version: 1.1.2 + resolution: "@protobufjs/base64@npm:1.1.2" + checksum: 67173ac34de1e242c55da52c2f5bdc65505d82453893f9b51dc74af9fe4c065cf4a657a4538e91b0d4a1a1e0a0642215e31894c31650ff6e3831471061e1ee9e + languageName: node + linkType: hard + +"@protobufjs/codegen@npm:^2.0.4": + version: 2.0.4 + resolution: "@protobufjs/codegen@npm:2.0.4" + checksum: 59240c850b1d3d0b56d8f8098dd04787dcaec5c5bd8de186fa548de86b86076e1c50e80144b90335e705a044edf5bc8b0998548474c2a10a98c7e004a1547e4b + languageName: node + linkType: hard + +"@protobufjs/eventemitter@npm:^1.1.0": + version: 1.1.0 + resolution: "@protobufjs/eventemitter@npm:1.1.0" + checksum: 0369163a3d226851682f855f81413cbf166cd98f131edb94a0f67f79e75342d86e89df9d7a1df08ac28be2bc77e0a7f0200526bb6c2a407abbfee1f0262d5fd7 + languageName: node + linkType: hard + +"@protobufjs/fetch@npm:^1.1.0": + version: 1.1.0 + resolution: "@protobufjs/fetch@npm:1.1.0" + dependencies: + "@protobufjs/aspromise": ^1.1.1 + "@protobufjs/inquire": ^1.1.0 + checksum: 3fce7e09eb3f1171dd55a192066450f65324fd5f7cc01a431df01bb00d0a895e6bfb5b0c5561ce157ee1d886349c90703d10a4e11a1a256418ff591b969b3477 + languageName: node + linkType: hard + +"@protobufjs/float@npm:^1.0.2": + version: 1.0.2 + resolution: "@protobufjs/float@npm:1.0.2" + checksum: 5781e1241270b8bd1591d324ca9e3a3128d2f768077a446187a049e36505e91bc4156ed5ac3159c3ce3d2ba3743dbc757b051b2d723eea9cd367bfd54ab29b2f + languageName: node + linkType: hard + +"@protobufjs/inquire@npm:^1.1.0": + version: 1.1.0 + resolution: "@protobufjs/inquire@npm:1.1.0" + checksum: ca06f02eaf65ca36fb7498fc3492b7fc087bfcc85c702bac5b86fad34b692bdce4990e0ef444c1e2aea8c034227bd1f0484be02810d5d7e931c55445555646f4 + languageName: node + linkType: hard + +"@protobufjs/path@npm:^1.1.2": + version: 1.1.2 + resolution: "@protobufjs/path@npm:1.1.2" + checksum: 856eeb532b16a7aac071cacde5c5620df800db4c80cee6dbc56380524736205aae21e5ae47739114bf669ab5e8ba0e767a282ad894f3b5e124197cb9224445ee + languageName: node + linkType: hard + +"@protobufjs/pool@npm:^1.1.0": + version: 1.1.0 + resolution: "@protobufjs/pool@npm:1.1.0" + checksum: d6a34fbbd24f729e2a10ee915b74e1d77d52214de626b921b2d77288bd8f2386808da2315080f2905761527cceffe7ec34c7647bd21a5ae41a25e8212ff79451 + languageName: node + linkType: hard + +"@protobufjs/utf8@npm:^1.1.0": + version: 1.1.0 + resolution: "@protobufjs/utf8@npm:1.1.0" + checksum: f9bf3163d13aaa3b6f5e6fbf37a116e094ea021c0e1f2a7ccd0e12a29e2ce08dafba4e8b36e13f8ed7397e1591610ce880ed1289af4d66cf4ace8a36a9557278 + languageName: node + linkType: hard + "@puppeteer/browsers@npm:2.2.3": version: 2.2.3 resolution: "@puppeteer/browsers@npm:2.2.3" @@ -3920,6 +4160,15 @@ __metadata: languageName: node linkType: hard +"@types/node@npm:>=13.7.0": + version: 20.14.2 + resolution: "@types/node@npm:20.14.2" + dependencies: + undici-types: ~5.26.4 + checksum: 265362479b8f3b50fcd1e3f9e9af6121feb01a478dff0335ae67cccc3babfe45d0f12209d3d350595eebd7e67471762697b877c380513f8e5d27a238fa50c805 + languageName: node + linkType: hard + "@types/node@npm:^18.14.6, @types/node@npm:^18.15.11, @types/node@npm:^18.15.3, @types/node@npm:^18.7.23": version: 18.19.33 resolution: "@types/node@npm:18.19.33" @@ -10269,6 +10518,13 @@ __metadata: languageName: node linkType: hard +"long@npm:^5.0.0": + version: 5.2.3 + resolution: "long@npm:5.2.3" + checksum: 885ede7c3de4facccbd2cacc6168bae3a02c3e836159ea4252c87b6e34d40af819824b2d4edce330bfb5c4d6e8ce3ec5864bdcf9473fa1f53a4f8225860e5897 + languageName: node + linkType: hard + "lru-cache@npm:^10.0.1, lru-cache@npm:^10.1.0, lru-cache@npm:^10.2.0": version: 10.2.2 resolution: "lru-cache@npm:10.2.2" @@ -11792,6 +12048,26 @@ __metadata: languageName: node linkType: hard +"protobufjs@npm:^7.3.0": + version: 7.3.2 + resolution: "protobufjs@npm:7.3.2" + dependencies: + "@protobufjs/aspromise": ^1.1.2 + "@protobufjs/base64": ^1.1.2 + "@protobufjs/codegen": ^2.0.4 + "@protobufjs/eventemitter": ^1.1.0 + "@protobufjs/fetch": ^1.1.0 + "@protobufjs/float": ^1.0.2 + "@protobufjs/inquire": ^1.1.0 + "@protobufjs/path": ^1.1.2 + "@protobufjs/pool": ^1.1.0 + "@protobufjs/utf8": ^1.1.0 + "@types/node": ">=13.7.0" + long: ^5.0.0 + checksum: cfb2a744787f26ee7c82f3e7c4b72cfc000e9bb4c07828ed78eb414db0ea97a340c0cc3264d0e88606592f847b12c0351411f10e9af255b7ba864eec44d7705f + languageName: node + linkType: hard + "protons-runtime@npm:5.4.0, protons-runtime@npm:^5.0.0, protons-runtime@npm:^5.4.0": version: 5.4.0 resolution: "protons-runtime@npm:5.4.0" @@ -13198,6 +13474,16 @@ __metadata: languageName: node linkType: hard +"systeminformation@npm:5.22.9": + version: 5.22.9 + resolution: "systeminformation@npm:5.22.9" + bin: + systeminformation: lib/cli.js + checksum: c605e568395041e57483722b38802928bc6122e347f9e1c6a9588b30297e28c19ffb425be0306fcd6e4f14cd443fa0bbbb407e69ef15d891f6776946718b26bb + conditions: (os=darwin | os=linux | os=win32 | os=freebsd | os=openbsd | os=netbsd | os=sunos | os=android) + languageName: node + linkType: hard + "table-layout@npm:^1.0.2": version: 1.0.2 resolution: "table-layout@npm:1.0.2"