diff --git a/.github/workflows/network-deploy.yml b/.github/workflows/network-deploy.yml index 8ceba615141..e964d26f650 100644 --- a/.github/workflows/network-deploy.yml +++ b/.github/workflows/network-deploy.yml @@ -12,6 +12,14 @@ on: aztec_docker_image: description: The Aztec Docker image to use, e.g. aztecprotocol/aztec:da809c58290f9590836f45ec59376cbf04d3c4ce-x86_64 required: true + deployment_mnemonic_secret_name: + description: The name of the secret which holds the boot node's contract deployment mnemonic + required: true + default: testnet-deployment-mnemonic + respect_tf_lock: + description: Whether to respect the Terraform lock + required: false + default: "true" jobs: network_deployment: @@ -26,6 +34,7 @@ jobs: AZTEC_DOCKER_IMAGE: ${{ inputs.aztec_docker_image }} NAMESPACE: ${{ inputs.namespace }} VALUES_FILE: ${{ inputs.values_file }} + DEPLOYMENT_MNEMONIC_SECRET_NAME: ${{ inputs.deployment_mnemonic_secret_name }} CHART_PATH: ./spartan/aztec-network CLUSTER_NAME: aztec-gke REGION: us-west1-a @@ -62,6 +71,12 @@ jobs: echo "Terraform state bucket already exists" fi + - name: Grab the boot node deployment mnemonic + id: get-mnemonic + run: | + echo "::add-mask::$(gcloud secrets versions access latest --secret=${{ env.DEPLOYMENT_MNEMONIC_SECRET_NAME }})" + echo "mnemonic=$(gcloud secrets versions access latest --secret=${{ env.DEPLOYMENT_MNEMONIC_SECRET_NAME }})" >> "$GITHUB_OUTPUT" + - name: Setup Terraform uses: hashicorp/setup-terraform@v2 with: @@ -82,8 +97,10 @@ jobs: -var="values_file=${{ env.VALUES_FILE }}" \ -var="gke_cluster_context=${{ env.GKE_CLUSTER_CONTEXT }}" \ -var="aztec_docker_image=${{ env.AZTEC_DOCKER_IMAGE }}" \ - -out=tfplan + -var="l1_deployment_mnemonic=${{ steps.get-mnemonic.outputs.mnemonic }}" \ + -out=tfplan \ + -lock=${{ inputs.respect_tf_lock }} - name: Terraform Apply working-directory: ./spartan/terraform/deploy-release - run: terraform apply -auto-approve tfplan + run: terraform apply -lock=${{ inputs.respect_tf_lock }} -auto-approve tfplan diff --git a/spartan/aztec-network/files/config/deploy-l1-contracts.sh b/spartan/aztec-network/files/config/deploy-l1-contracts.sh index 4d976821f04..f84cc6bcb51 100644 --- a/spartan/aztec-network/files/config/deploy-l1-contracts.sh +++ b/spartan/aztec-network/files/config/deploy-l1-contracts.sh @@ -4,16 +4,13 @@ set -exu CHAIN_ID=$1 -# Use default account, it is funded on our dev machine -export PRIVATE_KEY="0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80" - # Run the deploy-l1-contracts command and capture the output output="" # if INIT_VALIDATORS is true, then we need to pass the validators flag to the deploy-l1-contracts command if [ "$INIT_VALIDATORS" = "true" ]; then - output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --validators $2 --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --validators $2 --l1-chain-id $CHAIN_ID) else - output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --l1-chain-id $CHAIN_ID) + output=$(node --no-warnings /usr/src/yarn-project/aztec/dest/bin/index.js deploy-l1-contracts --mnemonic "$MNEMONIC" --l1-chain-id $CHAIN_ID) fi echo "$output" diff --git a/spartan/aztec-network/templates/boot-node.yaml b/spartan/aztec-network/templates/boot-node.yaml index 022d344ebe4..cd8fe41aa68 100644 --- a/spartan/aztec-network/templates/boot-node.yaml +++ b/spartan/aztec-network/templates/boot-node.yaml @@ -72,6 +72,8 @@ spec: env: - name: INIT_VALIDATORS value: "true" + - name: MNEMONIC + value: "{{ .Values.aztec.l1DeploymentMnemonic }}" - name: ETHEREUM_SLOT_DURATION value: "{{ .Values.ethereum.blockTime }}" - name: AZTEC_SLOT_DURATION diff --git a/spartan/aztec-network/templates/reth.yaml b/spartan/aztec-network/templates/reth.yaml index d6230ecf0ad..8a3e28728bf 100644 --- a/spartan/aztec-network/templates/reth.yaml +++ b/spartan/aztec-network/templates/reth.yaml @@ -19,6 +19,43 @@ spec: {{- if .Values.network.public }} hostNetwork: true {{- end }} + initContainers: + - name: prepare-genesis + image: node:18-alpine + command: ["/bin/sh", "-c"] + args: + - | + cd /tmp + npm init -y + npm install ethers@6 + cat > derive.js << 'EOF' + const { ethers } = require('ethers'); + const fs = require('fs'); + + async function main() { + const mnemonic = process.env.DEPLOYMENT_MNEMONIC; + const wallet = ethers.Wallet.fromPhrase(mnemonic); + + const genesis = JSON.parse(fs.readFileSync('/genesis-template/genesis.json', 'utf8')); + + genesis.alloc[wallet.address] = { + balance: '0x3635c9adc5dea00000' // 1000 ETH in wei + }; + + fs.writeFileSync('/genesis-output/genesis.json', JSON.stringify(genesis, null, 2)); + } + + main().catch(console.error); + EOF + node derive.js + env: + - name: DEPLOYMENT_MNEMONIC + value: {{ .Values.aztec.l1DeploymentMnemonic }} + volumeMounts: + - name: genesis-template + mountPath: /genesis-template + - name: genesis-output + mountPath: /genesis-output containers: - name: ethereum image: "{{ .Values.images.reth.image }}" @@ -40,7 +77,7 @@ spec: volumeMounts: - name: shared-volume mountPath: /data - - name: genesis + - name: genesis-output mountPath: /genesis resources: {{- toYaml .Values.ethereum.resources | nindent 12 }} @@ -48,9 +85,11 @@ spec: - name: shared-volume persistentVolumeClaim: claimName: {{ include "aztec-network.fullname" . }}-ethereum-pvc - - name: genesis + - name: genesis-template configMap: name: {{ include "aztec-network.fullname" . }}-reth-genesis + - name: genesis-output + emptyDir: {} {{if not .Values.network.public }} --- apiVersion: v1 diff --git a/spartan/aztec-network/values.yaml b/spartan/aztec-network/values.yaml index fbc957e802f..3d1815d41b4 100644 --- a/spartan/aztec-network/values.yaml +++ b/spartan/aztec-network/values.yaml @@ -34,6 +34,7 @@ aztec: epochDuration: 16 # how many L2 slots in an epoch epochProofClaimWindow: 13 # in L2 slots realProofs: false + l1DeploymentMnemonic: "test test test test test test test test test test test junk" # the mnemonic used when deploying contracts bootNode: peerIdPrivateKey: "" diff --git a/spartan/terraform/deploy-release/deploy.sh b/spartan/terraform/deploy-release/deploy.sh deleted file mode 100755 index ac13466745f..00000000000 --- a/spartan/terraform/deploy-release/deploy.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash - -# Usage: ./deploy.sh -# Example: ./deploy.sh rough-rhino aztecprotocol/aztec:698cd3d62680629a3f1bfc0f82604534cedbccf3-x86_64 - -set -eu - -RELEASE_NAME=$1 -AZTEC_DOCKER_IMAGE=$2 - -terraform init -backend-config="key=deploy-network/${RELEASE_NAME}/terraform.tfstate" -terraform apply -var-file="release.tfvars" -var="RELEASE_NAME=${RELEASE_NAME}" -var="AZTEC_DOCKER_IMAGE=${AZTEC_DOCKER_IMAGE}" diff --git a/spartan/terraform/deploy-release/main.tf b/spartan/terraform/deploy-release/main.tf index 458f36795ef..4f525c1388f 100644 --- a/spartan/terraform/deploy-release/main.tf +++ b/spartan/terraform/deploy-release/main.tf @@ -46,6 +46,11 @@ resource "helm_release" "aztec-gke-cluster" { value = var.AZTEC_DOCKER_IMAGE } + set { + name = "aztec.l1DeploymentMnemonic" + value = var.l1_deployment_mnemonic + } + # Setting timeout and wait conditions timeout = 1200 # 20 minutes in seconds wait = true diff --git a/spartan/terraform/deploy-release/release.tfvars b/spartan/terraform/deploy-release/release.tfvars deleted file mode 100644 index 916a85db918..00000000000 --- a/spartan/terraform/deploy-release/release.tfvars +++ /dev/null @@ -1 +0,0 @@ -VALUES_FILE = "release.yaml" diff --git a/spartan/terraform/deploy-release/variables.tf b/spartan/terraform/deploy-release/variables.tf index 03930fa3d65..369218d3ede 100644 --- a/spartan/terraform/deploy-release/variables.tf +++ b/spartan/terraform/deploy-release/variables.tf @@ -18,3 +18,9 @@ variable "AZTEC_DOCKER_IMAGE" { description = "Docker image to use for the aztec network" type = string } + +variable "l1_deployment_mnemonic" { + description = "Mnemonic to use for the L1 contract deployments" + type = string + sensitive = true +} diff --git a/spartan/terraform/gke-cluster-old/firewall.tf b/spartan/terraform/gke-cluster-old/firewall.tf new file mode 100644 index 00000000000..0dc4b406ce3 --- /dev/null +++ b/spartan/terraform/gke-cluster-old/firewall.tf @@ -0,0 +1,51 @@ +# Create ingress firewall rules for UDP +resource "google_compute_firewall" "udp_ingress" { + name = "allow-udp-ingress-custom" + network = "default" + allow { + protocol = "udp" + ports = ["40400-40499", "8080", "8545"] + } + direction = "INGRESS" + source_ranges = ["0.0.0.0/0"] + target_tags = ["gke-node", "aztec-gke-node"] +} + +# Create egress firewall rules for UDP +resource "google_compute_firewall" "udp_egress" { + name = "allow-udp-egress-custom" + network = "default" + allow { + protocol = "udp" + ports = ["40400-40499", "8080", "8545"] + } + direction = "EGRESS" + destination_ranges = ["0.0.0.0/0"] + target_tags = ["gke-node", "aztec-gke-node"] +} + +# Create ingress firewall rules for TCP +resource "google_compute_firewall" "tcp_ingress" { + name = "allow-tcp-ingress-custom" + network = "default" + allow { + protocol = "tcp" + ports = ["40400-40499", "8080", "8545"] + } + direction = "INGRESS" + source_ranges = ["0.0.0.0/0"] + target_tags = ["gke-node", "aztec-gke-node"] +} + +# Create egress firewall rules for TCP +resource "google_compute_firewall" "tcp_egress" { + name = "allow-tcp-egress-custom" + network = "default" + allow { + protocol = "tcp" + ports = ["40400-40499", "8080", "8545"] + } + direction = "EGRESS" + destination_ranges = ["0.0.0.0/0"] + target_tags = ["gke-node", "aztec-gke-node"] +} diff --git a/spartan/terraform/gke-cluster-old/main.tf b/spartan/terraform/gke-cluster-old/main.tf new file mode 100644 index 00000000000..6055ca52cc3 --- /dev/null +++ b/spartan/terraform/gke-cluster-old/main.tf @@ -0,0 +1,193 @@ +terraform { + backend "s3" { + bucket = "aztec-terraform" + key = "spartan-gke-cluster/terraform.tfstate" + region = "eu-west-2" + } + required_providers { + google = { + source = "hashicorp/google" + version = "~> 5.0" + } + } +} + +# Configure the Google Cloud provider +provider "google" { + project = var.project + region = var.region +} + +# Create the service account +resource "google_service_account" "gke_sa" { + account_id = "gke-nodes-sa" + display_name = "GKE Nodes Service Account" + description = "Service account for GKE nodes" +} + +# Add IAM roles to the service account +resource "google_project_iam_member" "gke_sa_roles" { + for_each = toset([ + "roles/logging.logWriter", + "roles/monitoring.metricWriter", + "roles/monitoring.viewer", + "roles/artifactregistry.reader" + ]) + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.gke_sa.email}" +} + +# Create a new service account for Helm +resource "google_service_account" "helm_sa" { + account_id = "helm-sa" + display_name = "Helm Service Account" + description = "Service account for Helm operations" +} + +# Add IAM roles to the Helm service account +resource "google_project_iam_member" "helm_sa_roles" { + for_each = toset([ + "roles/container.admin", + "roles/storage.admin", + "roles/secretmanager.admin" + ]) + project = var.project + role = each.key + member = "serviceAccount:${google_service_account.helm_sa.email}" +} + +# Create a GKE cluster +resource "google_container_cluster" "primary" { + name = "spartan-gke" + location = var.zone + + initial_node_count = 1 + # Remove default node pool after cluster creation + remove_default_node_pool = true + + # Kubernetes version + min_master_version = "latest" + + # Network configuration + network = "default" + subnetwork = "default" + + # Master auth configuration + master_auth { + client_certificate_config { + issue_client_certificate = false + } + } +} + +# Create primary node pool with autoscaling +resource "google_container_node_pool" "primary_nodes" { + name = "primary-node-pool" + location = var.zone + cluster = google_container_cluster.primary.name + + # Enable autoscaling + autoscaling { + min_node_count = 1 + max_node_count = 5 + } + + # Node configuration + node_config { + machine_type = "t2d-standard-32" + + service_account = google_service_account.gke_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + env = "production" + } + tags = ["gke-node"] + } + + # Management configuration + management { + auto_repair = true + auto_upgrade = true + } +} + +# Create node pool for aztec nodes (validators, prover nodes, boot nodes) +resource "google_container_node_pool" "aztec_nodes" { + name = "aztec-node-pool" + location = var.zone + cluster = google_container_cluster.primary.name + + # Enable autoscaling + autoscaling { + min_node_count = 1 + max_node_count = 128 + } + + # Node configuration + node_config { + machine_type = "t2d-standard-8" + + service_account = google_service_account.gke_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + env = "production" + } + tags = ["gke-node", "aztec"] + } + + # Management configuration + management { + auto_repair = true + auto_upgrade = true + } +} + +# Create spot instance node pool with autoscaling +resource "google_container_node_pool" "spot_nodes" { + name = "spot-node-pool" + location = var.zone + cluster = google_container_cluster.primary.name + + # Enable autoscaling + autoscaling { + min_node_count = 0 + max_node_count = 10 + } + + # Node configuration + node_config { + machine_type = "t2d-standard-32" + spot = true + + service_account = google_service_account.gke_sa.email + oauth_scopes = [ + "https://www.googleapis.com/auth/cloud-platform" + ] + + labels = { + env = "production" + pool = "spot" + } + tags = ["gke-node", "spot"] + + # Spot instance termination handler + taint { + key = "cloud.google.com/gke-spot" + value = "true" + effect = "NO_SCHEDULE" + } + } + + # Management configuration + management { + auto_repair = true + auto_upgrade = true + } +} diff --git a/spartan/terraform/gke-cluster-old/outputs.tf b/spartan/terraform/gke-cluster-old/outputs.tf new file mode 100644 index 00000000000..befaa28092e --- /dev/null +++ b/spartan/terraform/gke-cluster-old/outputs.tf @@ -0,0 +1,17 @@ +output "cluster_endpoint" { + value = google_container_cluster.primary.endpoint +} + +output "service_account_email" { + value = google_service_account.gke_sa.email +} + +output "region" { + description = "Google cloud region" + value = var.region +} + +output "kubernetes_cluster_name" { + description = "GKE Cluster Name" + value = google_container_cluster.primary.name +} diff --git a/spartan/terraform/gke-cluster-old/variables.tf b/spartan/terraform/gke-cluster-old/variables.tf new file mode 100644 index 00000000000..555458daa5d --- /dev/null +++ b/spartan/terraform/gke-cluster-old/variables.tf @@ -0,0 +1,11 @@ +variable "project" { + default = "testnet-440309" +} + +variable "region" { + default = "us-east4" +} + +variable "zone" { + default = "us-east4-a" +} diff --git a/spartan/terraform/gke-cluster/firewall.tf b/spartan/terraform/gke-cluster/firewall.tf index 0c5741c8506..0dc4b406ce3 100644 --- a/spartan/terraform/gke-cluster/firewall.tf +++ b/spartan/terraform/gke-cluster/firewall.tf @@ -8,7 +8,7 @@ resource "google_compute_firewall" "udp_ingress" { } direction = "INGRESS" source_ranges = ["0.0.0.0/0"] - target_tags = ["gke-node"] + target_tags = ["gke-node", "aztec-gke-node"] } # Create egress firewall rules for UDP @@ -21,7 +21,7 @@ resource "google_compute_firewall" "udp_egress" { } direction = "EGRESS" destination_ranges = ["0.0.0.0/0"] - target_tags = ["gke-node"] + target_tags = ["gke-node", "aztec-gke-node"] } # Create ingress firewall rules for TCP @@ -34,7 +34,7 @@ resource "google_compute_firewall" "tcp_ingress" { } direction = "INGRESS" source_ranges = ["0.0.0.0/0"] - target_tags = ["gke-node"] + target_tags = ["gke-node", "aztec-gke-node"] } # Create egress firewall rules for TCP @@ -47,5 +47,5 @@ resource "google_compute_firewall" "tcp_egress" { } direction = "EGRESS" destination_ranges = ["0.0.0.0/0"] - target_tags = ["gke-node"] + target_tags = ["gke-node", "aztec-gke-node"] } diff --git a/spartan/terraform/gke-cluster/main.tf b/spartan/terraform/gke-cluster/main.tf index 971a4aacdbc..2c169144485 100644 --- a/spartan/terraform/gke-cluster/main.tf +++ b/spartan/terraform/gke-cluster/main.tf @@ -1,7 +1,7 @@ terraform { backend "s3" { bucket = "aztec-terraform" - key = "spartan-gke-cluster/terraform.tfstate" + key = "aztec-gke-cluster/terraform.tfstate" region = "eu-west-2" } required_providers { @@ -20,9 +20,9 @@ provider "google" { # Create the service account resource "google_service_account" "gke_sa" { - account_id = "gke-nodes-sa" - display_name = "GKE Nodes Service Account" - description = "Service account for GKE nodes" + account_id = "aztec-gke-nodes-sa" + display_name = "Aztec GKE Nodes Service Account" + description = "Service account for aztec GKE nodes" } # Add IAM roles to the service account @@ -49,7 +49,8 @@ resource "google_service_account" "helm_sa" { resource "google_project_iam_member" "helm_sa_roles" { for_each = toset([ "roles/container.admin", - "roles/storage.admin" + "roles/storage.admin", + "roles/secretmanager.admin" ]) project = var.project role = each.key @@ -58,7 +59,7 @@ resource "google_project_iam_member" "helm_sa_roles" { # Create a GKE cluster resource "google_container_cluster" "primary" { - name = "spartan-gke" + name = var.cluster_name location = var.zone initial_node_count = 1 @@ -89,7 +90,7 @@ resource "google_container_node_pool" "primary_nodes" { # Enable autoscaling autoscaling { min_node_count = 1 - max_node_count = 5 + max_node_count = 2 } # Node configuration @@ -104,7 +105,7 @@ resource "google_container_node_pool" "primary_nodes" { labels = { env = "production" } - tags = ["gke-node"] + tags = ["aztec-gke-node"] } # Management configuration @@ -128,7 +129,7 @@ resource "google_container_node_pool" "aztec_nodes" { # Node configuration node_config { - machine_type = "t2d-standard-8" + machine_type = "t2d-standard-4" service_account = google_service_account.gke_sa.email oauth_scopes = [ @@ -138,7 +139,7 @@ resource "google_container_node_pool" "aztec_nodes" { labels = { env = "production" } - tags = ["gke-node", "aztec"] + tags = ["aztec-gke-node", "aztec"] } # Management configuration @@ -150,7 +151,7 @@ resource "google_container_node_pool" "aztec_nodes" { # Create spot instance node pool with autoscaling resource "google_container_node_pool" "spot_nodes" { - name = "spot-node-pool" + name = "aztec-spot-node-pool" location = var.zone cluster = google_container_cluster.primary.name @@ -174,7 +175,7 @@ resource "google_container_node_pool" "spot_nodes" { env = "production" pool = "spot" } - tags = ["gke-node", "spot"] + tags = ["aztec-gke-node", "spot"] # Spot instance termination handler taint { diff --git a/spartan/terraform/gke-cluster/variables.tf b/spartan/terraform/gke-cluster/variables.tf index 555458daa5d..83e1925cbd4 100644 --- a/spartan/terraform/gke-cluster/variables.tf +++ b/spartan/terraform/gke-cluster/variables.tf @@ -3,9 +3,13 @@ variable "project" { } variable "region" { - default = "us-east4" + default = "us-west1" } variable "zone" { - default = "us-east4-a" + default = "us-west1-a" +} + +variable "cluster_name" { + default = "aztec-gke" }