diff --git a/datastores/gossip_kv/Makefile b/datastores/gossip_kv/Makefile index 10783c1ce93..81f4688d2c3 100644 --- a/datastores/gossip_kv/Makefile +++ b/datastores/gossip_kv/Makefile @@ -1,15 +1,35 @@ # Makefile ## Minikube Options. -MINIKUBE_DISK_SIZE=100g -MINIKUBE_CPUS=16 -MINIKUBE_MEMORY=32768 +MINIKUBE_DISK_SIZE:=100g +MINIKUBE_CPUS:=16 +MINIKUBE_MEMORY:=32768 + +BASE_IMAGE_VERSION:=latest +SERVER_IMAGE_VERSION:=latest +CLI_IMAGE_VERSION:=latest # Docker Image Tags -BASE_IMAGE_TAG=hydroflow/gossip-kv-base-image:latest -SERVER_IMAGE_TAG=hydroflow/gossip-kv-server:latest -CLI_IMAGE_TAG=hydroflow/gossip-kv-cli:latest -DUMMY_IMAGE_TAG=hydroflow/gossip-kv-dummy:latest +BASE_IMAGE_TAG:=hydroflow-gossip-kv-base-image:$(BASE_IMAGE_VERSION) +SERVER_IMAGE_TAG:=hydroflow-gossip-kv-server:$(SERVER_IMAGE_VERSION) +CLI_IMAGE_TAG:=hydroflow-gossip-kv-cli:$(CLI_IMAGE_VERSION) + +AWS_TERRAFORM_PATH=../../datastores/gossip_kv/server/deployment/aws/terraform + +# Define color variables +COLOR_RESET = \033[0m +COLOR_GREEN = \033[1;32m +COLOR_BLUE = \033[1;34m +COLOR_RED = \033[1;31m + +# Define custom echo functions for different colors +greentext = @echo "$(COLOR_GREEN)$(1)$(COLOR_RESET)" +bluetext = @echo "$(COLOR_BLUE)$(1)$(COLOR_RESET)" +redtext = @echo "$(COLOR_RED)$(1)$(COLOR_RESET)" + +aws_setup_cli: + $(call cecho_green, Setting up kubectl to work with AWS EKS Cluster) + aws eks update-kubeconfig --region $$(terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw region) --name $$(terraform -chdir=$(AWS_TERRAFORM​⬤ # Default target when you run 'make' @@ -39,4 +59,25 @@ deploy_local: kubectl apply -f ../../datastores/gossip_kv/server/local # Target to delete the Minikube cluster and build again -rebuild_local: clean_local start_minikube build_docker_images \ No newline at end of file +rebuild_local: clean_local start_minikube build_docker_images + +aws_terraform_init: + terraform -chdir="$(AWS_TERRAFORM_PATH)" init + +aws_terraform_apply: + terraform -chdir="$(AWS_TERRAFORM_PATH)" apply + +aws_setup_kubectl: + @echo "Setting up kubectl to work with AWS EKS Cluster" + aws eks update-kubeconfig --region $$(terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw region) --name $$(terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw cluster_name) + +aws_upload_docker_images: build_docker_images + $(eval SERVER_REPO_URL := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -json repository_urls | jq -r '.["gossip_kv_server"]')) + $(eval CLI_REPO_URL := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -json repository_urls | jq -r '.["gossip_kv_cli"]')) + $(eval REGION := $(shell terraform -chdir=$(AWS_TERRAFORM_PATH) output -raw region)) + docker tag $(SERVER_IMAGE_TAG) $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) + docker tag $(CLI_IMAGE_TAG) $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) + aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(SERVER_REPO_URL) + docker push $(SERVER_REPO_URL):$(SERVER_IMAGE_VERSION) + aws ecr get-login-password --region $(REGION) | docker login --username AWS --password-stdin $(SERVER_REPO_URL) + docker push $(CLI_REPO_URL):$(CLI_IMAGE_VERSION) \ No newline at end of file diff --git a/datastores/gossip_kv/server/deployment/aws/terraform/.gitignore b/datastores/gossip_kv/server/deployment/aws/terraform/.gitignore new file mode 100644 index 00000000000..3fa8c86b7b0 --- /dev/null +++ b/datastores/gossip_kv/server/deployment/aws/terraform/.gitignore @@ -0,0 +1 @@ +.terraform diff --git a/datastores/gossip_kv/server/deployment/aws/terraform/main.tf b/datastores/gossip_kv/server/deployment/aws/terraform/main.tf new file mode 100644 index 00000000000..009a7b36b6c --- /dev/null +++ b/datastores/gossip_kv/server/deployment/aws/terraform/main.tf @@ -0,0 +1,295 @@ + +provider "aws" { + region = var.region +} + +# Filter out local zones, which are not currently supported +# with managed node groups +data "aws_availability_zones" "available" { + filter { + name = "opt-in-status" + values = ["opt-in-not-required"] + } +} + +data "aws_caller_identity" "current" {} + +locals { + cluster_name = "anna-load-test-${random_string.suffix.result}" + account_id = data.aws_caller_identity.current.account_id +} + +resource "random_string" "suffix" { + length = 8 + special = false +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + version = "5.8.1" + + name = "anna-load-test-vpc" + + cidr = "10.0.0.0/16" + azs = slice(data.aws_availability_zones.available.names, 0, 3) + + map_public_ip_on_launch = true + public_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"] + + enable_dns_hostnames = true + + public_subnet_tags = { + "kubernetes.io/role/elb" = 1 + } +} + +module "eks_cluster" { + source = "terraform-aws-modules/eks/aws" + version = "20.24.3" + + cluster_name = local.cluster_name + cluster_version = "1.31" + + cluster_endpoint_public_access = true + enable_cluster_creator_admin_permissions = true + + cluster_addons = { + aws-ebs-csi-driver = { + service_account_role_arn = module.irsa-ebs-csi.iam_role_arn + } + } + + vpc_id = module.vpc.vpc_id + subnet_ids = module.vpc.public_subnets + + eks_managed_node_group_defaults = { + ami_type = "AL2_x86_64" + } + + eks_managed_node_groups = { + one = { + name = "servers" + + instance_types = [var.instance_type] + + min_size = 1 + max_size = 3 + desired_size = 2 + } + } +} + +# https://aws.amazon.com/blogs/containers/amazon-ebs-csi-driver-is-now-generally-available-in-amazon-eks-add-ons/ +data "aws_iam_policy" "ebs_csi_policy" { + arn = "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy" +} + +module "irsa-ebs-csi" { + source = "terraform-aws-modules/iam/aws//modules/iam-assumable-role-with-oidc" + version = "5.39.0" + + create_role = true + role_name = "AmazonEKSTFEBSCSIRole-${module.eks_cluster.cluster_name}" + provider_url = module.eks_cluster.oidc_provider + role_policy_arns = [data.aws_iam_policy.ebs_csi_policy.arn] + oidc_fully_qualified_subjects = ["system:serviceaccount:kube-system:ebs-csi-controller-sa"] +} + +variable "ecr_repositories" { + description = "List of ECR repository names" + type = list(string) + default = ["gossip_kv_server", "gossip_kv_cli"] +} + +module "ecr" { + source = "terraform-aws-modules/ecr/aws" + version = "2.3.0" + + for_each = { for repo in var.ecr_repositories : repo => repo } + repository_name = each.value + + repository_read_write_access_arns = [data.aws_caller_identity.current.arn] + repository_lifecycle_policy = jsonencode({ + rules = [ + { + rulePriority = 1, + description = "Keep last 30 images", + selection = { + tagStatus = "tagged", + tagPrefixList = ["v"], + countType = "imageCountMoreThan", + countNumber = 30 + }, + action = { + type = "expire" + } + } + ] + }) + + repository_image_tag_mutability = "MUTABLE" + tags = { + Terraform = "true" + Environment = "dev" + } +} + +provider "kubernetes" { + host = module.eks_cluster.cluster_endpoint + cluster_ca_certificate = base64decode(module.eks_cluster.cluster_certificate_authority_data) + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "aws" + args = [ + "eks", + "get-token", + "--cluster-name", + module.eks_cluster.cluster_name, + ] + } +} + +resource "kubernetes_stateful_set" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + service_name = "gossip-kv-seed-nodes" + replicas = 3 + + selector { + match_labels = { + app = "gossip-kv-seed-nodes" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + termination_grace_period_seconds = 5 + + container { + name = "gossip-kv-server" + image = "${module.ecr.gossip_kv_server.repository_url}:latest" + image_pull_policy = "IfNotPresent" + + env { + name = "RUST_LOG" + value = "trace" + } + + env { + name = "RUST_BACKTRACE" + value = "full" + } + + port { + container_port = 3001 + protocol = "UDP" + } + + volume_mount { + name = "gossip-kv-dynamic-config" + mount_path = "/config/dynamic" + } + } + + volume { + name = "gossip-kv-dynamic-config" + + config_map { + name = "gossip-kv-dynamic-config" + } + } + } + } + } +} + +resource "kubernetes_deployment" "gossip_kv_cli" { + metadata { + name = "gossip-kv-cli" + labels = { + app = "gossip-kv-cli" + } + } + + spec { + replicas = 1 + + selector { + match_labels = { + app = "gossip-kv-cli" + } + } + + template { + metadata { + labels = { + app = "gossip-kv-cli" + } + } + + spec { + termination_grace_period_seconds = 5 + + container { + name = "gossip-kv-cli" + image = "${module.ecr.gossip_kv_cli.repository_url}:latest" + image_pull_policy = "IfNotPresent" + command = ["/bin/sh"] + args = ["-c", "while true; do sleep 3600; done"] + tty = true + + env { + name = "RUST_LOG" + value = "info" + } + } + } + } + } +} + +resource "kubernetes_service" "gossip_kv_seed_nodes" { + metadata { + name = "gossip-kv-seed-nodes" + labels = { + app = "gossip-kv-seed-nodes" + } + } + + spec { + cluster_ip = "None" + selector = { + app = "gossip-kv-seed-nodes" + } + + port { + port = 3001 + target_port = 3001 + protocol = "UDP" + } + } +} + +resource "kubernetes_config_map" "gossip_kv_dynamic_config" { + metadata { + name = "gossip-kv-dynamic-config" + } + + data = { + "dynamic.toml" = < details.repository_url } +} \ No newline at end of file diff --git a/datastores/gossip_kv/server/deployment/aws/terraform/terraform.tf b/datastores/gossip_kv/server/deployment/aws/terraform/terraform.tf new file mode 100644 index 00000000000..6b54c550a22 --- /dev/null +++ b/datastores/gossip_kv/server/deployment/aws/terraform/terraform.tf @@ -0,0 +1,30 @@ +terraform { + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.70.0" + } + + random = { + source = "hashicorp/random" + version = "~> 3.6.1" + } + + tls = { + source = "hashicorp/tls" + version = "~> 4.0.5" + } + + cloudinit = { + source = "hashicorp/cloudinit" + version = "~> 2.3.4" + } + + kubernetes = { + source = "hashicorp/kubernetes" + version = ">= 2.32.0" + } + } + + required_version = "~> 1.3" +} \ No newline at end of file diff --git a/datastores/gossip_kv/server/deployment/aws/terraform/variables.tf b/datastores/gossip_kv/server/deployment/aws/terraform/variables.tf new file mode 100644 index 00000000000..96800b84190 --- /dev/null +++ b/datastores/gossip_kv/server/deployment/aws/terraform/variables.tf @@ -0,0 +1,11 @@ +variable "region" { + description = "AWS region where the resources will be created" + type = string + default = "us-east-2" +} + +variable "instance_type" { + description = "Instance type for the EC2 instances" + type = string + default = "t3.small" +} \ No newline at end of file diff --git a/datastores/gossip_kv/server/deployment/local/objects.yaml b/datastores/gossip_kv/server/deployment/local/objects.yaml index 7421df268e1..2905e44ca86 100644 --- a/datastores/gossip_kv/server/deployment/local/objects.yaml +++ b/datastores/gossip_kv/server/deployment/local/objects.yaml @@ -67,7 +67,7 @@ spec: - name: RUST_LOG value: "info" --- -apiVersion: v1 +apiVersion: vw1 kind: Service metadata: name: gossip-kv-seed-nodes