feat!: aztec binary (AztecProtocol#3927)

Closes AztecProtocol#3552 
Closes AztecProtocol#3811 

- New `aztec` package, replaces `aztec-sandbox`
- New `aztec start` CLI to run Aztec infra (node, sequencer,
archiver, p2p-bootstrap)

.circleci/config.yml

@@ -411,7 +411,7 @@ jobs:
           name: Test
           command: cond_spot_run_container yarn-project 64 test | add_timestamps
 
-  aztec-sandbox:
+  aztec-package:
     machine:
       image: ubuntu-2204:2023.07.2
     resource_class: large
@@ -420,7 +420,7 @@ jobs:
       - *setup_env
       - run:
           name: "Build and test"
-          command: build aztec-sandbox
+          command: build aztec
 
   cli:
     machine:
@@ -993,7 +993,7 @@ jobs:
           command: |
             should_release || exit 0
             deploy_dockerhub noir
-            deploy_dockerhub aztec-sandbox
+            deploy_dockerhub aztec
             deploy_dockerhub cli
             deploy_dockerhub aztec-faucet
             deploy_dockerhub mainnet-fork
@@ -1045,7 +1045,7 @@ jobs:
             # Export variables for Terraform.
             export TF_VAR_BOOTNODE_1_PRIVATE_KEY=$BOOTNODE_1_PRIVATE_KEY
             export TF_VAR_BOOTNODE_2_PRIVATE_KEY=$BOOTNODE_2_PRIVATE_KEY
-            deploy_terraform_services yarn-project/p2p-bootstrap aztec-sandbox
+            deploy_terraform_services yarn-project/p2p-bootstrap aztec
       - run:
           name: "Deploy Aztec Nodes to AWS"
           command: |
@@ -1059,16 +1059,16 @@ jobs:
             # Check if l1-contracts have changed
             if [ "$CONTRACTS_DEPLOYED" -eq 1 ]; then
               echo "Contracts have changed, taint nodes to force redeploy.."
-              deploy_terraform_services yarn-project/aztec-node aztec-sandbox aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]"
+              deploy_terraform_services yarn-project/aztec-node aztec aztec-node "aws_ecs_task_definition.aztec-node[0],aws_ecs_task_definition.aztec-node[1]"
             else
-              deploy_terraform_services yarn-project/aztec-node aztec-sandbox
+              deploy_terraform_services yarn-project/aztec-node aztec
             fi
       - run:
           name: "Deploy Aztec Faucet to AWS"
           command: |
             should_deploy 0 || exit 0
             export TF_VAR_FAUCET_PRIVATE_KEY=$FAUCET_PRIVATE_KEY
-            deploy_terraform_services yarn-project/aztec-faucet aztec-sandbox
+            deploy_terraform_services yarn-project/aztec-faucet aztec
 
 # Repeatable config for defining the workflow below.
 defaults: &defaults
@@ -1202,14 +1202,14 @@ workflows:
       - build-docs: *defaults_yarn_project
 
       # Artifacts
-      - aztec-sandbox: *defaults_yarn_project_prod
+      - aztec-package: *defaults_yarn_project_prod
       - cli: *defaults_yarn_project_prod
       - aztec-faucet: *defaults_yarn_project_prod
 
       # Boxes.
       - boxes:
           requires:
-            - aztec-sandbox
+            - aztec-package
           <<: *defaults
       - boxes-blank:
           requires:
@@ -1228,7 +1228,7 @@ workflows:
       - e2e-join:
           requires:
             - end-to-end
-            - aztec-sandbox
+            - aztec-package
             - cli
           <<: *defaults
       - e2e-2-pxes: *e2e_test

README.md

@@ -9,7 +9,7 @@ All the packages that make up [Aztec](https://docs.aztec.network).
 ## Popular packages
 
 - [Aztec.nr](./yarn-project/aztec-nr/): A [Noir](https://noir-lang.org) framework for smart contracts on Aztec.
-- [Aztec Sandbox](./yarn-project/aztec-sandbox/): A package for setting up a local dev net, including a local Ethereum network, deployed rollup contracts and Aztec execution environment.
+- [Aztec](./yarn-project/aztec/): A package for starting up local dev net modules, including a local 'sandbox' devnet, an Ethereum network, deployed rollup contracts and Aztec execution environment.
 - [Aztec.js](./yarn-project/aztec.js/): A tool for interacting with the Aztec network. It communicates via the [Private Execution Environment (PXE)](./yarn-project/pxe/).
 - [Example contracts](./yarn-project/noir-contracts/): Example contracts for the Aztec network, written in Noir.
 - [End to end tests](./yarn-project/end-to-end/): Integration tests written in Typescript--a good reference for how to use the packages for specific tasks.

aztec-up/bin/aztec

@@ -1,4 +1,21 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-$(dirname $0)/.aztec-run aztecprotocol/aztec-sandbox $@
+# Call cli image if used with `aztec cli ...args`
+if [ -n "${1-}" ] && [ "$1" != "--help" ]; then
+  if [ "$1" == "cli" ]; then
+    shift
+    $(dirname $0)/.aztec-run aztecprotocol/cli "$@"
+  elif [ "$1" == "sandbox" ]; then
+    $(dirname $0)/aztec-sandbox
+  else
+    $(dirname $0)/.aztec-run aztecprotocol/aztec "$@"
+  fi
+else
+  # TODO - display help message
+  echo
+  echo "Using 'aztec' CLI:"
+  echo "  aztec start <args>  - Start aztec infrastructure components. See 'aztec start --help' for detailed command info."
+  echo "  aztec sandbox       - Run a local sandbox network (same as aztec-sandbox)."
+  echo "  aztec cli <args>    - Run the aztec client CLI. See 'aztec cli --help' for detailed command info."
+fi

aztec-up/bin/aztec-cli

@@ -3,5 +3,6 @@ set -euo pipefail
 
 export ENV_VARS_TO_INJECT="PXE_URL PRIVATE_KEY DEBUG"
 export PXE_URL=${PXE_URL:-"http://host.docker.internal:8080"}
+export ETHEREUM_HOST=${ETHEREUM_HOST:-"http://host.docker.internal:8545"}
 
-$(dirname $0)/.aztec-run aztecprotocol/cli $@
+$(dirname $0)/.aztec-run aztecprotocol/cli $@

aztec-up/bin/aztec-install

@@ -40,7 +40,7 @@ function title() {
     echo -e "${r}"
   fi
   echo -e "This will install the following scripts and update your PATH if necessary:"
-  echo -e "  ${bold}${g}aztec${r}         - launches various infrastructure subsystems (sequencer, prover, pxe, etc)."
+  echo -e "  ${bold}${g}aztec${r}         - launches various infrastructure subsystems (node, sequencer, prover, pxe, etc)."
   echo -e "  ${bold}${g}aztec-cli${r}     - a command line tool for interfacing and experimenting with infrastructure."
   echo -e "  ${bold}${g}aztec-nargo${r}   - aztec's build of nargo, the noir compiler toolchain."
   echo -e "  ${bold}${g}aztec-sandbox${r} - a wrapper around docker-compose that launches services needed for sandbox testing."
@@ -106,12 +106,12 @@ export DOCKER_CLI_HINTS=false
 
 if [ -z "${SKIP_PULL:-}" ]; then
   info "Pulling aztec version $VERSION..."
-  pull_container aztec-sandbox
+  pull_container aztec
   pull_container cli
   pull_container noir
 fi
 
-# Download the Docker Compose file. Used by aztec-sandbox.
+# Download the Docker Compose file. Used by aztec.
 curl -fsSL http://$INSTALL_HOST/docker-compose.yml -o $AZTEC_PATH/docker-compose.yml
 
 function install_bin {

aztec-up/bin/docker-compose.yml

@@ -17,9 +17,8 @@ services:
       ANVIL_PORT: ${ANVIL_PORT:-8545}
 
   aztec:
-    image: "aztecprotocol/aztec-sandbox"
+    image: "aztecprotocol/aztec"
     ports:
-      - "${AZTEC_NODE_PORT:-8079}:${AZTEC_NODE_PORT:-8079}"
       - "${PXE_PORT:-8080}:${PXE_PORT:-8080}"
     environment:
       DEBUG: # Loaded from the user shell if explicitly set
@@ -32,7 +31,6 @@ services:
       WS_BLOCK_CHECK_INTERVAL_MS: 50
       PXE_BLOCK_POLLING_INTERVAL_MS: 50
       ARCHIVER_VIEM_POLLING_INTERVAL_MS: 500
-      AZTEC_NODE_PORT: ${AZTEC_NODE_PORT:-8079}
       PXE_PORT: ${PXE_PORT:-8080}
     volumes:
-      - ./log:/usr/src/yarn-project/aztec-sandbox/log:rw
+      - ./log:/usr/src/yarn-project/aztec/log:rw

boxes/Dockerfile

@@ -1,11 +1,11 @@
 # Builds the boxes (they were copied into yarn-project-base so the cli can unbox).
 # Produces a container that can be run to test a specific box. See docker-compose.yml.
-FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec-sandbox AS aztec-sandbox
+FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/aztec AS aztec
 FROM 278380418400.dkr.ecr.eu-west-2.amazonaws.com/noir as noir
 
 # We need yarn. Start fresh container.
 FROM node:18.19.0
-COPY --from=aztec-sandbox /usr/src /usr/src
+COPY --from=aztec /usr/src /usr/src
 COPY --from=noir /usr/src/noir/target/release/nargo /usr/src/noir/target/release/nargo
 WORKDIR /usr/src/boxes
 ENV AZTEC_NARGO=/usr/src/noir/target/release/nargo

boxes/blank/webpack.config.js

@@ -56,6 +56,7 @@ export default (_, argv) => ({
       crypto: false,
       fs: false,
       path: false,
+      events: require.resolve('events/'),
       stream: require.resolve('stream-browserify'),
       tty: require.resolve('tty-browserify'),
       util: require.resolve('util/'),

boxes/docker-compose.yml

@@ -5,7 +5,7 @@ services:
     command: "'anvil --silent -p 8545 --host 0.0.0.0 --chain-id 31337'"
 
   aztec:
-    image: aztecprotocol/aztec-sandbox
+    image: aztecprotocol/aztec
     environment:
       ETHEREUM_HOST: http://ethereum:8545
       CHAIN_ID: 31337

boxes/yarn.lock

@@ -302,6 +302,7 @@ __metadata:
   resolution: "@aztec/types@portal:../yarn-project/types::locator=%40aztec%2Fboxes%40workspace%3A."
   dependencies:
     "@aztec/ethereum": "workspace:^"
+    "@aztec/foundation": "workspace:^"
   languageName: node
   linkType: soft
 

build_manifest.yml

@@ -152,9 +152,9 @@ yarn-project-prod:
     - yarn-project
   multiarch: buildx
 
-aztec-sandbox:
+aztec:
   buildDir: yarn-project
-  projectDir: yarn-project/aztec-sandbox
+  projectDir: yarn-project/aztec
   dependencies:
     - yarn-project-prod
   multiarch: buildx
@@ -175,18 +175,18 @@ cli:
 boxes:
   buildDir: boxes
   dependencies:
-    - aztec-sandbox
+    - aztec
     - noir
   runDependencies:
-    - aztec-sandbox
+    - aztec
 
 end-to-end:
   buildDir: yarn-project
   projectDir: yarn-project/end-to-end
   dependencies:
     - yarn-project
   runDependencies:
-    - aztec-sandbox
+    - aztec
 
 mainnet-fork:
   buildDir: iac/mainnet-fork

docs/docs/developers/cli/sandbox-reference.md

@@ -56,13 +56,24 @@ cd ~/.aztec && docker-compose up
 
 ## Running Aztec PXE / Node / P2P-Bootstrap node
 
-If you wish to run components of the Aztec network stack separately, you can still use the Sandbox by including a `MODE` variable.
-The values for `MODE` can be:
+If you wish to run components of the Aztec network stack separately, you can use the `aztec start` command with various options for enabling components.
 
-- sandbox (default)
-- node
-- pxe
-- p2p-bootstrap
+```bash
+aztec start --node [nodeOptions] --pxe [pxeOptions] --archiver [archiverOptions] --sequencer [sequencerOptions] ----p2p-bootstrap [p2pOptions]
+```
+
+Starting the aztec node alongside a PXE, sequencer or archiver, will attach the components to the node. If you want to e.g. run a PXE separately to a node, you can:
+Start a node:
+
+```bash
+aztec start --node [node] --archiver [archiverOptions]
+```
+
+Then start a PXE on a separate terminal that connects to that node:
+
+```bash
+aztec start --pxe nodeUrl=http://localhost:8080
+```
 
 ## Environment Variables
 

docs/docs/developers/contracts/deploying.md

@@ -6,7 +6,7 @@ Once you have [compiled](./compiling.md) your contracts you can proceed to deplo
 
 - `aztec-cli` and `aztec-nargo` installed (go to [CLI main section](../cli/main.md) for installation instructions)
 - contract artifacts ready (go to [Compiling contracts section](./compiling.md) for instructions on how to compile contracts)
-- aztec-sandbox running (go to [Sandbox section](../getting_started/quickstart.md) for instructions on how to install and run the sandbox)
+- Aztec Sandbox running (go to [Sandbox section](../getting_started/quickstart.md) for instructions on how to install and run the sandbox)
 
 ## Deploy
 

yarn-project/Dockerfile.prod

@@ -11,13 +11,13 @@ WORKDIR /usr/src/yarn-project
 ARG COMMIT_TAG=""
 RUN ./scripts/version_packages.sh
 # Productionify. See comment in yarn-project-base/Dockerfile.
-RUN yarn workspaces focus @aztec/cli @aztec/aztec-sandbox @aztec/aztec-faucet @aztec/aztec.js --production && \
-    yarn cache clean && \
-    rm -rf /usr/src/barretenberg/ts/src && \
-    # TODO: Fix by extracting noir code out of yarn-project.
-    # This was just a "rm -rf ./**/src".
-    # Due to the mess of us needing to find noir code in noir-protocol-circuits/src/crates we have to do this...
-    find . -maxdepth 2 -name src -type d | grep -v "./noir-protocol-circuits" | xargs rm -rf
+RUN yarn workspaces focus @aztec/cli @aztec/aztec @aztec/aztec-faucet @aztec/aztec.js --production && \
+  yarn cache clean && \
+  rm -rf /usr/src/barretenberg/ts/src && \
+  # TODO: Fix by extracting noir code out of yarn-project.
+  # This was just a "rm -rf ./**/src".
+  # Due to the mess of us needing to find noir code in noir-protocol-circuits/src/crates we have to do this...
+  find . -maxdepth 2 -name src -type d | grep -v "./noir-protocol-circuits" | xargs rm -rf
 
 # We no longer need nargo.
 RUN rm -rf /usr/src/noir/target
@@ -34,32 +34,32 @@ ENV COMMIT_TAG=$COMMIT_TAG
 RUN apt update && apt install -y curl && rm -rf /var/lib/apt/lists/* && apt-get clean
 ENV NODE_VERSION=18.19.0
 RUN ARCH= && \
-    dpkgArch="$(dpkg --print-architecture)" && \
-    case "${dpkgArch##*-}" in \
-        amd64) ARCH='x64';; \
-        arm64) ARCH='arm64';; \
-        *) echo "unsupported architecture"; exit 1 ;; \
-    esac && \
-    curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \
-    tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \
-    --exclude "*/share/*" \
-    --exclude "*/bin/corepack" \
-    --exclude "*/bin/npx" \
-    --exclude "*/bin/npm" \
-    --exclude "*/corepack/*" \
-    --exclude "*/npm/man/*" \
-    --exclude "*/npm/docs/*" \
-    --exclude "*/include/*" && \
-    rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \
-    node --version
+  dpkgArch="$(dpkg --print-architecture)" && \
+  case "${dpkgArch##*-}" in \
+  amd64) ARCH='x64';; \
+  arm64) ARCH='arm64';; \
+  *) echo "unsupported architecture"; exit 1 ;; \
+  esac && \
+  curl -fsSLO --compressed "https://nodejs.org/dist/v$NODE_VERSION/node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \
+  tar zxf "node-v$NODE_VERSION-linux-$ARCH.tar.gz" -C /usr --strip-components=1 --no-same-owner \
+  --exclude "*/share/*" \
+  --exclude "*/bin/corepack" \
+  --exclude "*/bin/npx" \
+  --exclude "*/bin/npm" \
+  --exclude "*/corepack/*" \
+  --exclude "*/npm/man/*" \
+  --exclude "*/npm/docs/*" \
+  --exclude "*/include/*" && \
+  rm "node-v$NODE_VERSION-linux-$ARCH.tar.gz" && \
+  node --version
 # Yarn is used for unboxing.
-ENV YARN_VERSION=1.22.19 
+ENV YARN_VERSION=1.22.19
 RUN curl -fsSLO --compressed "https://yarnpkg.com/downloads/$YARN_VERSION/yarn-v$YARN_VERSION.tar.gz" && \
-    mkdir -p /opt && \
-    tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && \
-    ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && \
-    ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && \
-    rm yarn-v$YARN_VERSION.tar.gz && \
-    yarn --version
+  mkdir -p /opt && \
+  tar -xzf yarn-v$YARN_VERSION.tar.gz -C /opt/ && \
+  ln -s /opt/yarn-v$YARN_VERSION/bin/yarn /usr/local/bin/yarn && \
+  ln -s /opt/yarn-v$YARN_VERSION/bin/yarnpkg /usr/local/bin/yarnpkg && \
+  rm yarn-v$YARN_VERSION.tar.gz && \
+  yarn --version
 COPY --from=builder /usr/src /usr/src
 ENTRYPOINT ["/usr/bin/node"]

yarn-project/archiver/package.json

@@ -43,7 +43,7 @@
     "@aztec/l1-artifacts": "workspace:^",
     "@aztec/types": "workspace:^",
     "debug": "^4.3.4",
-    "lmdb": "^2.9.1",
+    "lmdb": "^2.9.2",
     "lodash.omit": "^4.5.0",
     "tsc-watch": "^6.0.0",
     "tslib": "^2.5.0",

yarn-project/archiver/src/archiver/archiver.ts

@@ -42,12 +42,17 @@ import {
   retrieveNewPendingL1ToL2Messages,
 } from './data_retrieval.js';
 
+/**
+ * Helper interface to combine all sources this archiver implementation provides.
+ */
+export type ArchiveSource = L2BlockSource & L2LogsSource & ContractDataSource & L1ToL2MessageSource;
+
 /**
  * Pulls L2 blocks in a non-blocking manner and provides interface for their retrieval.
  * Responsible for handling robust L1 polling so that other components do not need to
  * concern themselves with it.
  */
-export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource, L1ToL2MessageSource {
+export class Archiver implements ArchiveSource {
   /**
    * A promise in which we will be continually fetching new L2 blocks.
    */

yarn-project/archiver/src/index.ts

@@ -8,6 +8,7 @@ import { Archiver, getConfigEnvVars } from './archiver/index.js';
 import { MemoryArchiverStore } from './archiver/memory_archiver_store/memory_archiver_store.js';
 
 export * from './archiver/index.js';
+export * from './rpc/index.js';
 
 const log = createDebugLogger('aztec:archiver');