diff --git a/.github/workflows/devnet-deploys.yml b/.github/workflows/devnet-deploys.yml index 89166794d25..496f4f72833 100644 --- a/.github/workflows/devnet-deploys.yml +++ b/.github/workflows/devnet-deploys.yml @@ -515,7 +515,7 @@ jobs: run: | env terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/mainnet-fork" - terraform apply -input=false -auto-approve -replace="aws_efs_file_system.aztec_mainnet_fork_data_store" + terraform apply -input=false -auto-approve - name: Wait for mainnet fork deployment run: | @@ -567,13 +567,13 @@ jobs: working-directory: ./yarn-project/aztec/terraform/node run: | terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-node" - terraform apply -input=false -auto-approve -replace="aws_efs_file_system.node_data_store" -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}" + terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.node_udp_range_start }}" - name: Deploy Aztec Prover Nodes working-directory: ./yarn-project/aztec/terraform/prover-node run: | terraform init -input=false -backend-config="key=${{ env.DEPLOY_TAG }}/aztec-prover-node" - terraform apply -input=false -auto-approve -replace="aws_efs_file_system.prover_node_data_store" -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}" + terraform apply -input=false -auto-approve -var="NODE_P2P_TCP_PORT=${{ needs.set-network.outputs.prover_node_tcp_range_start }}" -var="NODE_P2P_UDP_PORT=${{ needs.set-network.outputs.prover_node_udp_range_start }}" - name: Deploy Provers working-directory: ./yarn-project/aztec/terraform/prover diff --git a/yarn-project/aztec/terraform/bot/main.tf b/yarn-project/aztec/terraform/bot/main.tf index 4dfe0641bf4..f49e44b334e 100644 --- a/yarn-project/aztec/terraform/bot/main.tf +++ b/yarn-project/aztec/terraform/bot/main.tf @@ -150,7 +150,7 @@ resource "aws_ecs_task_definition" "aztec-bot" { command = ["start", "--bot", "--pxe"] essential = true cpu = 8192 - memoryReservation = 14336 + memoryReservation = 15685 portMappings = [ { containerPort = 80 diff --git a/yarn-project/aztec/terraform/node/main.tf b/yarn-project/aztec/terraform/node/main.tf index 6304b37ab45..81686edc890 100644 --- a/yarn-project/aztec/terraform/node/main.tf +++ b/yarn-project/aztec/terraform/node/main.tf @@ -126,14 +126,78 @@ resource "aws_efs_mount_target" "public_az2" { security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] } + +data "template_file" "user_data" { + count = local.node_count + template = <> /etc/ecs/ecs.config +echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}"}' >> /etc/ecs/ecs.config +EOF +} + +# Launch template for our prover agents +# 4 cores and 8 GB memory +resource "aws_launch_template" "aztec-node-launch-template" { + count = local.node_count + name = "${var.DEPLOY_TAG}-aztec-node-launch-template-${count.index + 1}" + image_id = "ami-0cd4858f2b923aa6b" + instance_type = "c6a.xlarge" + vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + + iam_instance_profile { + name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name + } + + key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name + + user_data = base64encode(data.template_file.user_data[count.index].rendered) + + tag_specifications { + resource_type = "instance" + tags = { + Name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" + prometheus = "" + } + } +} + +resource "aws_ec2_fleet" "aztec_node_fleet" { + count = local.node_count + launch_template_config { + launch_template_specification { + launch_template_id = aws_launch_template.aztec-node-launch-template[count.index].id + version = aws_launch_template.aztec-node-launch-template[count.index].latest_version + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id + availability_zone = "eu-west-2a" + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + availability_zone = "eu-west-2b" + } + } + + target_capacity_specification { + default_target_capacity_type = "on-demand" + total_target_capacity = 1 + spot_target_capacity = 0 + on_demand_target_capacity = 1 + } + + terminate_instances = true + terminate_instances_with_expiration = true +} + # Define task definitions for each node. resource "aws_ecs_task_definition" "aztec-node" { count = local.node_count family = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" - requires_compatibilities = ["FARGATE"] + requires_compatibilities = ["EC2"] network_mode = "awsvpc" - cpu = "2048" - memory = "4096" execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn @@ -151,7 +215,8 @@ resource "aws_ecs_task_definition" "aztec-node" { image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" command = ["start", "--node", "--archiver", "--sequencer"] essential = true - memoryReservation = 3776 + cpu = 4096 + memoryReservation = 7790 portMappings = [ { containerPort = 80 @@ -390,19 +455,20 @@ resource "aws_ecs_service" "aztec-node" { count = local.node_count name = "${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id - launch_type = "FARGATE" + launch_type = "EC2" desired_count = 1 deployment_maximum_percent = 100 deployment_minimum_healthy_percent = 0 - platform_version = "1.4.0" - force_new_deployment = true - enable_execute_command = true + #platform_version = "1.4.0" + force_new_deployment = true + enable_execute_command = true network_configuration { - assign_public_ip = true + #assign_public_ip = true subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, + data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id ] security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] } @@ -419,6 +485,11 @@ resource "aws_ecs_service" "aztec-node" { container_port = 80 } + placement_constraints { + type = "memberOf" + expression = "attribute:group == ${var.DEPLOY_TAG}-aztec-node-${count.index + 1}" + } + task_definition = aws_ecs_task_definition.aztec-node[count.index].family } diff --git a/yarn-project/aztec/terraform/prover-node/main.tf b/yarn-project/aztec/terraform/prover-node/main.tf index 9ea84547ac0..ebdf4656309 100644 --- a/yarn-project/aztec/terraform/prover-node/main.tf +++ b/yarn-project/aztec/terraform/prover-node/main.tf @@ -103,7 +103,7 @@ resource "aws_service_discovery_service" "aztec-prover-node" { # Configure an EFS filesystem. resource "aws_efs_file_system" "prover_node_data_store" { - creation_token = "${var.DEPLOY_TAG}-prover-node-data" + creation_token = "${var.DEPLOY_TAG}-prover-node-data" tags = { Name = "${var.DEPLOY_TAG}-prover-node-data" @@ -126,14 +126,77 @@ resource "aws_efs_mount_target" "public_az2" { security_groups = [data.terraform_remote_state.setup_iac.outputs.security_group_public_id] } +data "template_file" "user_data" { + count = local.node_count + template = <> /etc/ecs/ecs.config +echo 'ECS_INSTANCE_ATTRIBUTES={"group": "${var.DEPLOY_TAG}-prover-node-${count.index + 1}"}' >> /etc/ecs/ecs.config +EOF +} + +# Launch template for our prover agents +# 4 cores and 8 GB memory +resource "aws_launch_template" "aztec-prover-node-launch-template" { + count = local.node_count + name = "${var.DEPLOY_TAG}-prover-node-launch-template-${count.index + 1}" + image_id = "ami-0cd4858f2b923aa6b" + instance_type = "c6a.xlarge" + vpc_security_group_ids = [data.terraform_remote_state.setup_iac.outputs.security_group_private_id] + + iam_instance_profile { + name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_profile_name + } + + key_name = data.terraform_remote_state.setup_iac.outputs.ecs_instance_key_pair_name + + user_data = base64encode(data.template_file.user_data[count.index].rendered) + + tag_specifications { + resource_type = "instance" + tags = { + Name = "${var.DEPLOY_TAG}-prover-node-${count.index + 1}" + prometheus = "" + } + } +} + +resource "aws_ec2_fleet" "aztec_prover_node_fleet" { + count = local.node_count + launch_template_config { + launch_template_specification { + launch_template_id = aws_launch_template.aztec-prover-node-launch-template[count.index].id + version = aws_launch_template.aztec-prover-node-launch-template[count.index].latest_version + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id + availability_zone = "eu-west-2a" + } + + override { + subnet_id = data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id + availability_zone = "eu-west-2b" + } + } + + target_capacity_specification { + default_target_capacity_type = "on-demand" + total_target_capacity = 1 + spot_target_capacity = 0 + on_demand_target_capacity = 1 + } + + terminate_instances = true + terminate_instances_with_expiration = true +} + # Define task definitions for each node. resource "aws_ecs_task_definition" "aztec-prover-node" { count = local.node_count family = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" - requires_compatibilities = ["FARGATE"] + requires_compatibilities = ["EC2"] network_mode = "awsvpc" - cpu = "2048" - memory = "4096" execution_role_arn = data.terraform_remote_state.setup_iac.outputs.ecs_task_execution_role_arn task_role_arn = data.terraform_remote_state.aztec2_iac.outputs.cloudwatch_logging_ecs_role_arn @@ -151,7 +214,8 @@ resource "aws_ecs_task_definition" "aztec-prover-node" { image = "${var.DOCKERHUB_ACCOUNT}/aztec:${var.IMAGE_TAG}" command = ["start", "--prover-node", "--archiver"] essential = true - memoryReservation = 3776 + cpu = 4096 + memoryReservation = 7790 portMappings = [ { containerPort = 80 @@ -250,18 +314,19 @@ resource "aws_ecs_service" "aztec-prover-node" { count = local.node_count name = "${var.DEPLOY_TAG}-aztec-prover-node-${count.index + 1}" cluster = data.terraform_remote_state.setup_iac.outputs.ecs_cluster_id - launch_type = "FARGATE" + launch_type = "EC2" desired_count = 1 deployment_maximum_percent = 100 deployment_minimum_healthy_percent = 0 - platform_version = "1.4.0" - force_new_deployment = true - enable_execute_command = true + #platform_version = "1.4.0" + force_new_deployment = true + enable_execute_command = true network_configuration { - assign_public_ip = true + #assign_public_ip = true subnets = [ - data.terraform_remote_state.setup_iac.outputs.subnet_az1_id + data.terraform_remote_state.setup_iac.outputs.subnet_az1_private_id, + data.terraform_remote_state.setup_iac.outputs.subnet_az2_private_id ] security_groups = [data.terraform_remote_state.aztec-network_iac.outputs.p2p_security_group_id, data.terraform_remote_state.setup_iac.outputs.security_group_private_id] } @@ -278,6 +343,11 @@ resource "aws_ecs_service" "aztec-prover-node" { container_port = 80 } + placement_constraints { + type = "memberOf" + expression = "attribute:group == ${var.DEPLOY_TAG}-prover-node-${count.index + 1}" + } + task_definition = aws_ecs_task_definition.aztec-prover-node[count.index].family } diff --git a/yarn-project/aztec/terraform/prover/main.tf b/yarn-project/aztec/terraform/prover/main.tf index 537bc7e73f0..72b48ff520b 100644 --- a/yarn-project/aztec/terraform/prover/main.tf +++ b/yarn-project/aztec/terraform/prover/main.tf @@ -238,7 +238,7 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" { "command": ["start", "--prover"], "essential": true, "cpu": 16384, - "memoryReservation": 122880, + "memoryReservation": 127800, "portMappings": [ { "containerPort": 80 @@ -285,9 +285,9 @@ resource "aws_ecs_task_definition" "aztec-proving-agent" { "name": "NETWORK_NAME", "value": "${var.DEPLOY_TAG}" }, - { - "name": "LOG_JSON", - "value": "1" + { + "name": "LOG_JSON", + "value": "1" } ], "logConfiguration": { diff --git a/yarn-project/cli/src/cmds/devnet/bootstrap_network.ts b/yarn-project/cli/src/cmds/devnet/bootstrap_network.ts index fe37e56979a..5d5ef4eff6b 100644 --- a/yarn-project/cli/src/cmds/devnet/bootstrap_network.ts +++ b/yarn-project/cli/src/cmds/devnet/bootstrap_network.ts @@ -13,7 +13,7 @@ import { type DebugLogger, type LogFn } from '@aztec/foundation/log'; import { getContract } from 'viem'; import { privateKeyToAccount } from 'viem/accounts'; -// import { FeeJuicePortalManager } from '../../utils/portal_manager.js'; +import { FeeJuicePortalManager } from '../../utils/portal_manager.js'; type ContractDeploymentInfo = { address: AztecAddress; @@ -53,7 +53,7 @@ export async function bootstrapNetwork( const counter = await deployCounter(wallet); // NOTE: Disabling for now in order to get devnet running - // await fundFPC(counter.address, wallet, l1Clients, fpc.address, debugLog); + await fundFPC(counter.address, wallet, l1Clients, fpc.address, debugLog); if (json) { log( @@ -227,47 +227,47 @@ async function deployCounter(wallet: Wallet): Promise { } // NOTE: Disabling for now in order to get devnet running -// async function fundFPC( -// counterAddress: AztecAddress, -// wallet: Wallet, -// l1Clients: L1Clients, -// fpcAddress: AztecAddress, -// debugLog: DebugLogger, -// ) { -// // eslint-disable-next-line @typescript-eslint/ban-ts-comment -// // @ts-ignore - Importing noir-contracts.js even in devDeps results in a circular dependency error. Need to ignore because this line doesn't cause an error in a dev environment -// const { FeeJuiceContract, CounterContract } = await import('@aztec/noir-contracts.js'); -// const { -// protocolContractAddresses: { feeJuice }, -// } = await wallet.getPXEInfo(); - -// const feeJuiceContract = await FeeJuiceContract.at(feeJuice, wallet); - -// const feeJuicePortal = await FeeJuicePortalManager.new( -// wallet, -// l1Clients.publicClient, -// l1Clients.walletClient, -// debugLog, -// ); - -// const amount = 10n ** 21n; -// const { claimAmount, claimSecret } = await feeJuicePortal.bridgeTokensPublic(fpcAddress, amount, true); - -// const counter = await CounterContract.at(counterAddress, wallet); - -// // TODO (alexg) remove this once sequencer builds blocks continuously -// // advance the chain -// await counter.methods -// .increment(wallet.getAddress(), wallet.getAddress()) -// .send() -// .wait({ proven: true, provenTimeout: 600 }); -// await counter.methods -// .increment(wallet.getAddress(), wallet.getAddress()) -// .send() -// .wait({ proven: true, provenTimeout: 600 }); - -// await feeJuiceContract.methods -// .claim(fpcAddress, claimAmount, claimSecret) -// .send() -// .wait({ proven: true, provenTimeout: 600 }); -// } +async function fundFPC( + counterAddress: AztecAddress, + wallet: Wallet, + l1Clients: L1Clients, + fpcAddress: AztecAddress, + debugLog: DebugLogger, +) { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore - Importing noir-contracts.js even in devDeps results in a circular dependency error. Need to ignore because this line doesn't cause an error in a dev environment + const { FeeJuiceContract, CounterContract } = await import('@aztec/noir-contracts.js'); + const { + protocolContractAddresses: { feeJuice }, + } = await wallet.getPXEInfo(); + + const feeJuiceContract = await FeeJuiceContract.at(feeJuice, wallet); + + const feeJuicePortal = await FeeJuicePortalManager.new( + wallet, + l1Clients.publicClient, + l1Clients.walletClient, + debugLog, + ); + + const amount = 10n ** 21n; + const { claimAmount, claimSecret } = await feeJuicePortal.bridgeTokensPublic(fpcAddress, amount, true); + + const counter = await CounterContract.at(counterAddress, wallet); + + // TODO (alexg) remove this once sequencer builds blocks continuously + // advance the chain + await counter.methods + .increment(wallet.getAddress(), wallet.getAddress()) + .send() + .wait({ proven: true, provenTimeout: 600 }); + await counter.methods + .increment(wallet.getAddress(), wallet.getAddress()) + .send() + .wait({ proven: true, provenTimeout: 600 }); + + await feeJuiceContract.methods + .claim(fpcAddress, claimAmount, claimSecret) + .send() + .wait({ proven: true, provenTimeout: 600 }); +} diff --git a/yarn-project/end-to-end/src/fixtures/utils.ts b/yarn-project/end-to-end/src/fixtures/utils.ts index 17ca30c3300..9c81ec82977 100644 --- a/yarn-project/end-to-end/src/fixtures/utils.ts +++ b/yarn-project/end-to-end/src/fixtures/utils.ts @@ -38,7 +38,7 @@ import { computeContractAddressFromInstance, getContractClassFromArtifact, } from '@aztec/circuits.js'; -import { NULL_KEY } from '@aztec/ethereum'; +import { NULL_KEY, isAnvilTestChain } from '@aztec/ethereum'; import { bufferAsFields } from '@aztec/foundation/abi'; import { makeBackoff, retry, retryUntil } from '@aztec/foundation/retry'; import { @@ -347,7 +347,7 @@ export async function setup( let anvil: Anvil | undefined; if (!config.l1RpcUrl) { - if (chain.id != foundry.id) { + if (!isAnvilTestChain(chain.id)) { throw new Error(`No ETHEREUM_HOST set but non anvil chain requested`); } if (PXE_URL) { diff --git a/yarn-project/ethereum/src/constants.ts b/yarn-project/ethereum/src/constants.ts index 92e0b01c9d3..c1f4b34d732 100644 --- a/yarn-project/ethereum/src/constants.ts +++ b/yarn-project/ethereum/src/constants.ts @@ -1,3 +1,4 @@ import { type Hex } from 'viem'; export const NULL_KEY: Hex = `0x${'0000000000000000000000000000000000000000000000000000000000000000'}`; +export const AZTEC_TEST_CHAIN_ID = 677692; diff --git a/yarn-project/ethereum/src/deploy_l1_contracts.ts b/yarn-project/ethereum/src/deploy_l1_contracts.ts index f09c1963d70..969cb67ba0e 100644 --- a/yarn-project/ethereum/src/deploy_l1_contracts.ts +++ b/yarn-project/ethereum/src/deploy_l1_contracts.ts @@ -26,6 +26,7 @@ import { import { type HDAccount, type PrivateKeyAccount, mnemonicToAccount, privateKeyToAccount } from 'viem/accounts'; import { foundry } from 'viem/chains'; +import { isAnvilTestChain } from './ethereum_chain.js'; import { type L1ContractAddresses } from './l1_contract_addresses.js'; /** @@ -167,7 +168,7 @@ export const deployL1Contracts = async ( }; return await (await fetch(rpcUrl, content)).json(); }; - if (chain.id == foundry.id) { + if (isAnvilTestChain(chain.id)) { const interval = 12; const res = await rpcCall('anvil_setBlockTimestampInterval', [interval]); if (res.error) { @@ -257,7 +258,7 @@ export const deployL1Contracts = async ( `Initialized Gas Portal at ${feeJuicePortalAddress} to bridge between L1 ${feeJuiceAddress} to L2 ${args.l2FeeJuiceAddress}`, ); - if (chain.id == foundry.id) { + if (isAnvilTestChain(chain.id)) { // @note We make a time jump PAST the very first slot to not have to deal with the edge case of the first slot. // The edge case being that the genesis block is already occupying slot 0, so we cannot have another block. try { diff --git a/yarn-project/ethereum/src/ethereum_chain.ts b/yarn-project/ethereum/src/ethereum_chain.ts index 0f8d401a01d..7fc9dd092cf 100644 --- a/yarn-project/ethereum/src/ethereum_chain.ts +++ b/yarn-project/ethereum/src/ethereum_chain.ts @@ -1,4 +1,7 @@ import { type Chain } from 'viem'; +import { foundry } from 'viem/chains'; + +import { AZTEC_TEST_CHAIN_ID } from './constants.js'; /** * Interface containing the connection and chain properties to interact with a blockchain. @@ -14,3 +17,55 @@ export interface EthereumChain { */ rpcUrl: string; } + +/** + * Helper function to create an instance of Aztec Chain from an rpc url and api key. + * @param rpcUrl - The rpc url of the chain or a chain identifier (e.g. 'testnet') + * @param apiKey - An optional API key for the chain client. + */ +export function createEthereumChain(rpcUrl: string, _chainId: number | string): EthereumChain { + let chainId: number; + if (typeof _chainId === 'string') { + chainId = +_chainId; + } else { + chainId = _chainId; + } + if (chainId) { + return { + chainInfo: { + id: chainId, + name: 'Ethereum', + rpcUrls: { + default: { + http: [rpcUrl], + }, + }, + nativeCurrency: { + decimals: 18, + name: 'Ether', + symbol: 'ETH', + }, + }, + rpcUrl, + }; + } else { + return { + chainInfo: foundry, + rpcUrl, + }; + } +} + +/** + * Helper function to determine if a chain id is an instance of Anvil + */ +export function isAnvilTestChain(_chainId: number | string): boolean { + let chainId: number; + if (typeof _chainId === 'string') { + chainId = +_chainId; + } else { + chainId = _chainId; + } + const testChains = [foundry.id, AZTEC_TEST_CHAIN_ID]; + return testChains.includes(chainId); +} diff --git a/yarn-project/ethereum/src/index.ts b/yarn-project/ethereum/src/index.ts index c136dcdb263..aac126e1108 100644 --- a/yarn-project/ethereum/src/index.ts +++ b/yarn-project/ethereum/src/index.ts @@ -1,46 +1,5 @@ -import { foundry } from 'viem/chains'; - -import { type EthereumChain } from './ethereum_chain.js'; - export * from './constants.js'; export * from './deploy_l1_contracts.js'; export * from './l1_contract_addresses.js'; export * from './l1_reader.js'; - -/** - * Helper function to create an instance of Aztec Chain from an rpc url and api key. - * @param rpcUrl - The rpc url of the chain or a chain identifier (e.g. 'testnet') - * @param apiKey - An optional API key for the chain client. - */ -export function createEthereumChain(rpcUrl: string, _chainId: number | string): EthereumChain { - let chainId: number; - if (typeof _chainId === 'string') { - chainId = +_chainId; - } else { - chainId = _chainId; - } - if (chainId) { - return { - chainInfo: { - id: chainId, - name: 'Ethereum', - rpcUrls: { - default: { - http: [rpcUrl], - }, - }, - nativeCurrency: { - decimals: 18, - name: 'Ether', - symbol: 'ETH', - }, - }, - rpcUrl, - }; - } else { - return { - chainInfo: foundry, - rpcUrl, - }; - } -} +export * from './ethereum_chain.js';