diff --git a/.gitignore b/.gitignore index 0f5a9b892e1..4a296ac9077 100644 --- a/.gitignore +++ b/.gitignore @@ -52,3 +52,20 @@ gateway/testdata/recordings/ # Docs /docs/_site/ + +# terraform +# Local .terraform directories +**/.terraform/* +# .tfstate files +*.tfstate +*.tfstate.* +# Crash log files +crash.log +# Exclude all .tfvars files, which are likely to contain sentitive data, such as +# password, private keys, and other secrets. These should not be part of version +# control as they are data points which are potentially sensitive and subject +# to change depending on the environment. +*.tfvars +# Ignore CLI configuration files +.terraformrc +terraform.rc \ No newline at end of file diff --git a/benchmarks/terraform/lb.tf b/benchmarks/terraform/lb.tf new file mode 100644 index 00000000000..abe1ff7c3ab --- /dev/null +++ b/benchmarks/terraform/lb.tf @@ -0,0 +1,55 @@ +# ALB Security Group: Edit to restrict access to the application +resource "aws_security_group" "aws-lb" { + name = "benchmark-load-balancer" + description = "Controls access to the ALB" + vpc_id = aws_security_group.benchmark_sg.vpc_id + + ingress { + protocol = "tcp" + from_port = 8000 + to_port = 8000 + cidr_blocks = [for s in data.aws_subnet.all : s.cidr_block] + } + + egress { + protocol = "-1" + from_port = 0 + to_port = 0 + cidr_blocks = ["0.0.0.0/0"] + } + tags = { + Name = "benchmark-load-balancer" + } +} + +resource "aws_alb" "main" { + name = "benchmark-load-balancer" + subnets = [for s in data.aws_subnet.all : s.id] + security_groups = [aws_security_group.aws-lb.id] + internal = true + tags = { + Name = "benchmark-alb" + } +} + +resource "aws_alb_target_group" "benchmark" { + name = "benchmark-target-group" + port = 8000 + protocol = "HTTP" + vpc_id = aws_security_group.benchmark_sg.vpc_id + target_type = "ip" + tags = { + Name = "benchmark-alb-target-group" + } +} + +# Redirect all traffic from the ALB to the target group +resource "aws_alb_listener" "benchmark" { + load_balancer_arn = aws_alb.main.id + port = 8000 + protocol = "HTTP" + default_action { + target_group_arn = aws_alb_target_group.benchmark.id + type = "forward" + } +} diff --git a/benchmarks/terraform/main.tf b/benchmarks/terraform/main.tf new file mode 100644 index 00000000000..14fe9c62217 --- /dev/null +++ b/benchmarks/terraform/main.tf @@ -0,0 +1,238 @@ +variable "password" { + type = string + description = "The password to the postgres DB." +} + +variable "dockerReg" { + type = string + description = "docker registry to pull the image from" +} + +variable "tag" { + type = string + description = "lakeFS docker image tag" +} + +provider "aws" { + region = "us-east-1" +} + +//############################################################## +//# Data sources to get VPC, subnets and security group details +//############################################################## +data "aws_subnet_ids" "all" { + vpc_id = "vpc-04b176d1264698ffc" + tags = {"Type":"private"} +} + +data "aws_subnet" "all" { + for_each = data.aws_subnet_ids.all.ids + id = each.value +} + +data "aws_ami" "ubuntu" { + most_recent = true + + filter { + name = "name" + values = ["ubuntu/images/hvm-ssd/ubuntu-trusty-14.04-amd64-server-*"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } + + owners = ["099720109477"] # Canonical +} + +data "aws_arn" "BENCHMARK_VM" { + arn = "arn:aws:iam::977611293394:role/BENCHMARK_VM" +} + +//############################################################## +//# Resources to create sg, postgres db, and Fargate service +//############################################################## +resource "aws_security_group" "benchmark_sg" { + name = "benchmark_sg-${var.tag}" + description = "Allow benchmark traffic" + vpc_id = "vpc-04b176d1264698ffc" + + ingress { + description = "postgres" + from_port = 5432 + to_port = 5432 + protocol = "tcp" + cidr_blocks = [for s in data.aws_subnet.all : s.cidr_block] + } + + ingress { + description = "lakeFS" + from_port = 8000 + to_port = 8000 + protocol = "tcp" + cidr_blocks = [for s in data.aws_subnet.all : s.cidr_block] + } + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = "allow_benchmark" + } +} + +##### +# DB +##### +module "db" { + source = "github.com/terraform-aws-modules/terraform-aws-rds" + + identifier = "benchmarks-postgres-${var.tag}" + + engine = "postgres" + engine_version = "11" + instance_class = "db.t2.medium" + allocated_storage = 5 + storage_encrypted = false + + name = "BenchmarksDB" + + # Do NOT use 'user' as the value for 'username' as it throws: + # "Error creating DB Instance: InvalidParameterValue: MasterUsername + # user cannot be used as it is a reserved word used by the engine" + username = "benchmarks" + + password = var.password + port = "5432" + + vpc_security_group_ids = [aws_security_group.benchmark_sg.id] + + maintenance_window = "Mon:00:00-Mon:03:00" + backup_window = "03:00-06:00" + + # disable backups to create DB faster + backup_retention_period = 0 + + enabled_cloudwatch_logs_exports = ["postgresql", "upgrade"] + + # DB subnet group + subnet_ids = data.aws_subnet_ids.all.ids + + # DB parameter group + family = "postgres11" + + # DB option group + major_engine_version = "11" + + # Snapshot name upon DB deletion + final_snapshot_identifier = "demodb" + + # Database Deletion Protection + deletion_protection = false +} + +resource "random_string" "default" { + length = 16 +} + +resource "aws_launch_configuration" "benchmark" { + name = "benchmark-launch-${var.tag}" + image_id = data.aws_ami.ubuntu.id + instance_type = "t2.micro" +} + +resource "aws_autoscaling_group" "benchmark" { + vpc_zone_identifier = [for s in data.aws_subnet.all : s.id] + min_size = 1 + max_size = 1 + launch_configuration = aws_launch_configuration.benchmark.id + + tag { + key = "AmazonECSManaged" + value = "true" + propagate_at_launch = true + } +} + +resource "aws_ecs_cluster" "benchmark" { + name = "benchmark-${var.tag}" + capacity_providers = ["FARGATE"] +} + +resource "aws_cloudwatch_log_group" "benchmark" { + name = "/ecs/benchmark/${var.tag}" + + tags = { + Benchmark = var.tag + } +} + +resource "aws_ecs_task_definition" "benchmark" { + family = "benchmark-${var.tag}" + requires_compatibilities = ["FARGATE"] + cpu = "2048" + memory = "8192" + network_mode = "awsvpc" + execution_role_arn = data.aws_arn.BENCHMARK_VM.arn + task_role_arn = data.aws_arn.BENCHMARK_VM.arn + + container_definitions = <