Add EC2 Instance Connect based bastion for RDS (#669)

* Add EC2 Instance Connect based bastion for RDS

Signed-off-by: Carl Gieringer <78054+carlgieringer@users.noreply.github.com>

* Remove instance of Guimove/bastion/aws now that EC2 Instance Connect works

Signed-off-by: Carl Gieringer <78054+carlgieringer@users.noreply.github.com>

---------

Signed-off-by: Carl Gieringer <78054+carlgieringer@users.noreply.github.com>

config/example.env

@@ -9,3 +9,5 @@ NODE_ENV=development
 DEBUG=premiser-api:*
 DO_ASSERT=true
 LOCAL_API_SERVER_ARTIFICIAL_LATENCY_MS=100
+BASTION_INSTANCE_ID=i-0123abc
+RDS_ADDRESS=db-instance.abc123.region.rds.amazonaws.com

docs/Development.md

@@ -694,7 +694,7 @@ mfa_serial = arn:aws:iam::007899441171:mfa/username
 Logging in:
 
 ```shell
-aws-vault login username@howdju --duration 2h
+AWS_FEDERATION_TOKEN_TTL=6h aws-vault login username@howdju
 ```
 
 Running commands with your credentials:

infra/howdju.tf

@@ -24,17 +24,22 @@ module "s3_backend" {
   source = "./modules/s3_backend"
 }
 
-module "bastion" {
-  source              = "./modules/bastion"
-  instance_count      = 1
-  aws_region          = var.aws_region
-  vpc_id              = aws_vpc.default.id
-  key_pair_name       = local.key_name_bastion
-  hosted_zone_id      = data.aws_route53_zone.howdju.id
-  bastion_record_name = "bastion.howdju.com."
-  logs_bucket_name    = "howdju-bastion"
-  subnet_ids          = data.aws_subnets.default.ids
-  tags                = var.default_tags
+data "aws_ami" "bastion" {
+  most_recent = true
+  owners      = ["amazon"]
+  filter {
+    name   = "name"
+    values = ["al2023-ami-2023.*-arm64"]
+  }
+}
+
+module "bastion_connect" {
+  source         = "./modules/bastion_connect"
+  aws_region     = var.aws_region
+  aws_account_id = data.aws_caller_identity.current.account_id
+  vpc_id         = aws_vpc.default.id
+  instance_ami   = data.aws_ami.bastion.id
+  subnet_id      = data.aws_subnet.default_private_subnet_b.id
 }
 
 module "messages" {
@@ -123,12 +128,3 @@ resource "aws_eip" "elasticstack_instance" {
   instance   = module.elasticstack.instance_ids[count.index]
   depends_on = [aws_internet_gateway.default]
 }
-
-locals {
-  // The aws_key_pair resource is import-only and requires a public_key argument, which must be manually entered into
-  // the state file or Terraform will try to replace the key, which it can't.
-  // https://github.com/hashicorp/terraform-provider-aws/issues/1092
-  // Since our state is now stored in S3, and it's inconvenient to edit it manually, and because the only thing we need
-  // to reference is the key name, just use that directly.
-  key_name_bastion = "bastion"
-}

infra/modules/bastion_connect/main.tf

@@ -0,0 +1,92 @@
+resource "aws_ec2_instance_connect_endpoint" "bastion" {
+  security_group_ids = [aws_security_group.endpoint.id]
+  subnet_id          = var.subnet_id
+  // When client IP preservation is enabled, the instance to connect to must be in the same VPC as
+  // the EC2 Instance Connect Endpoint.
+  // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/connect-using-eice.html#ec2-instance-connect-endpoint-limitations
+  preserve_client_ip = false
+}
+
+resource "aws_security_group" "endpoint" {
+  vpc_id = var.vpc_id
+  name   = "instance-connect-endpoint"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "internet_to_endpoint" {
+  security_group_id = aws_security_group.endpoint.id
+  ip_protocol       = "tcp"
+  from_port         = 22
+  to_port           = 22
+  cidr_ipv4         = "0.0.0.0/0"
+  description       = "Allow someone to connect to the endpoint from any IPv4 address."
+}
+
+resource "aws_vpc_security_group_egress_rule" "endpoint_to_bastion" {
+  security_group_id            = aws_security_group.endpoint.id
+  ip_protocol                  = "tcp"
+  from_port                    = 22
+  to_port                      = 22
+  referenced_security_group_id = aws_security_group.bastion.id
+  description                  = "Allow instance connect endpoint to connect to the bastion instance."
+}
+
+resource "aws_instance" "bastion" {
+  ami                    = var.instance_ami
+  instance_type          = var.instance_type
+  subnet_id              = var.subnet_id
+  vpc_security_group_ids = [aws_security_group.bastion.id]
+  tags = {
+    Name = "bastion"
+  }
+}
+
+resource "aws_security_group" "bastion" {
+  vpc_id = var.vpc_id
+  name   = "bastion-instance"
+}
+
+resource "aws_vpc_security_group_ingress_rule" "endpoint_to_bastion" {
+  security_group_id            = aws_security_group.bastion.id
+  ip_protocol                  = "tcp"
+  from_port                    = 22
+  to_port                      = 22
+  referenced_security_group_id = aws_security_group.endpoint.id
+  description                  = "Allow endpoint to connect to bastion."
+}
+
+resource "aws_vpc_security_group_egress_rule" "bastion_to_rds_postgres" {
+  security_group_id            = aws_security_group.bastion.id
+  ip_protocol                  = "tcp"
+  from_port                    = 5432
+  to_port                      = 5432
+  referenced_security_group_id = aws_security_group.db_instances.id
+  description                  = "Allow bastion to tunnel to RDS on Postgres port."
+}
+
+resource "aws_security_group" "db_instances" {
+  vpc_id      = var.vpc_id
+  name        = "instance-connect-db-instances"
+  description = "Security group for RDS instances that the bastion host can connect to."
+}
+
+resource "aws_iam_policy" "tunnel_ssh_to_bastion" {
+  // See
+  // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/permissions-for-ec2-instance-connect-endpoint.html#iam-OpenTunnel
+  // and
+  // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-connect-configure-IAM-role.html#eic-permissions-allow-users-to-connect-to-specific-instances
+  name        = "bastion-ec2-instance-connect"
+  description = "Allow a principal to use the bastion EC2 Instance Connect to tunnel SSH connections to the bastion host"
+  policy = templatefile(
+    "${path.module}/policy.json.tftpl",
+    {
+      aws_region          = var.aws_region
+      aws_account_id      = var.aws_account_id
+      eice_id             = aws_ec2_instance_connect_endpoint.bastion.id
+      subnet_address      = jsonencode(data.aws_subnet.bastion.cidr_block)
+      bastion_instance_id = aws_instance.bastion.id
+  })
+}
+
+data "aws_subnet" "bastion" {
+  id = var.subnet_id
+}

infra/modules/bastion_connect/outputs.tf

@@ -0,0 +1,11 @@
+output "db_instances_security_group" {
+  value = aws_security_group.db_instances
+}
+
+output "bastion_instance_connect_endpoint" {
+  value = aws_ec2_instance_connect_endpoint.bastion
+}
+
+output "bastion_instance" {
+  value = aws_instance.bastion
+}

infra/modules/bastion_connect/policy.json.tftpl

@@ -0,0 +1,41 @@
+{
+  "Version": "2012-10-17",
+  "Statement": [
+      {
+        "Sid": "EC2InstanceConnect",
+        "Effect": "Allow",
+        "Resource": "arn:aws:ec2:${aws_region}:${aws_account_id}:instance-connect-endpoint/eice-${eice_id}",
+        "Action": "ec2-instance-connect:OpenTunnel",
+        "Condition": {
+            "NumericEquals": {
+                "ec2-instance-connect:remotePort": "22"
+            },
+            "IpAddress": {
+                "ec2-instance-connect:privateIpAddress": [${subnet_address}]
+            }
+        }
+      },
+      {
+        "Sid": "SSHPublicKey",
+        "Effect": "Allow",
+        "Resource": [
+          "arn:aws:ec2:${aws_region}:${aws_account_id}:instance/${bastion_instance_id}"
+        ],
+        "Action": "ec2-instance-connect:SendSSHPublicKey",
+        "Condition": {
+          "StringEquals": {
+            "ec2:osuser": "ec2-user"
+          }
+        }
+      },
+      {
+        "Sid": "EC2Describe",
+        "Effect": "Allow",
+        "Resource": "*",
+        "Action": [
+            "ec2:DescribeInstances",
+            "ec2:DescribeInstanceConnectEndpoints"
+        ]
+      }
+  ]
+}

infra/modules/bastion_connect/variables.tf

@@ -0,0 +1,20 @@
+variable "instance_type" {
+  default     = "t4g.nano"
+  description = "The type of bastion EC2 instance to start"
+}
+
+variable "instance_ami" {
+  description = "The AMI ID to use for the bastion EC2 instance"
+}
+
+variable "vpc_id" {
+  description = "The VPC to launch the bastion instance in"
+}
+
+variable "subnet_id" {
+  description = "The subnet to launch the bastion instance in. Should be private."
+}
+
+variable "aws_region" {}
+
+variable "aws_account_id" {}

infra/outputs.tf

@@ -0,0 +1,14 @@
+output "bastion_instance_id" {
+  value       = module.bastion_connect.bastion_instance.id
+  description = "The ID of the bastion instance"
+}
+
+output "bastion_instance_connect_endpoint_id" {
+  value       = module.bastion_connect.bastion_instance_connect_endpoint.id
+  description = "The ID of the bastion instance connect endpoint"
+}
+
+output "bastion_db_instances_security_group_id" {
+  value       = module.bastion_connect.db_instances_security_group.id
+  description = "The ID of the security group for RDS instances that the bastion host can connect to"
+}

premiser-api/README.md

@@ -0,0 +1,42 @@
+# Howdju API package
+
+This package contains the handler for Howdju's mono-lambda.
+
+## Connecting to DB
+
+(Admins only)
+
+To connect to the dbs, do the following.
+
+In one terminal do:
+
+```sh
+aws-vault exec user@howdju -- yarn run db:tunnel
+```
+
+In another terminal then do:
+
+```sh
+yarn run db:tunnel:shell:prod
+```
+
+You'll need env vars `BASTION_INSTANCE_ID` and `RDS_ADDRESS` set in your env. file.
+`BASTION_INSTANCE_ID` corresponds to `bastion_instance_id` from Terraform and
+`RDS_ADDRESS` must be looked up in the AWS console.
+
+And enter the Postgres password.
+
+### Too many authentication failures
+
+EC2 Instance Connect's ephemeral keys may build up, leading to:
+
+```text
+Received disconnect from UNKNOWN port 65535:2: Too many authentication failures
+Disconnected from UNKNOWN port 65535
+```
+
+To fix it, run the following to clear out your saved keys.
+
+```sh
+ssh-agent -D
+```

premiser-api/bin/db-tunnel.sh

@@ -1,11 +1,8 @@
 #!/usr/bin/env bash
 
-trap "exit" INT TERM
-trap "kill 0" EXIT
+set -e
 
-env=${1}
+# Sets up a tunnel between 5434 locally and port 5432 on RDS using an EC2 Instance Connect instance
+# as a bastion host.
 
-npm run db:tunnel &
-# The tunnel takes a few seconds to initialize
-sleep 10
-npm run db:tunnel:shell:${env}
+aws ec2-instance-connect ssh --instance-id $BASTION_INSTANCE_ID --local-forwarding 5434:$RDS_ADDRESS:5432

premiser-api/package.json

@@ -22,12 +22,9 @@
     "db:local:restart": "docker restart premiser_postgres",
     "db:local:shell": "psql -h localhost -U postgres premiser",
     "db:local": "yarn run db:local:restart && yarn run db:local:shell",
-    "db:tunnel": "ssh -N -L 5434:premiser.cc4gfeuy5z8h.us-east-1.rds.amazonaws.com:5432 bastion.howdju.com",
+    "db:tunnel": "env-cmd -f ../config/local.env bin/db-tunnel.sh",
     "db:tunnel:shell:prod": "psql -h 127.0.0.1 -p 5434 -d premiser -U premiser_rds",
     "db:tunnel:shell:pre-prod": "psql -h 127.0.0.1 -p 5434 -d howdju_pre_prod -U premiser_rds",
-    "db:connect": "aws ec2-instance-connect open-tunnel --instance-connect-endpoint-id eice-0527aed892af765ac --private-ip-address 172.31.69.62 --local-port 5434 --remote-port 5432",
-    "db:prod": "bin/db-tunnel.sh prod",
-    "db:pre-prod": "bin/db-tunnel.sh pre-prod",
     "dedupe-writs:prod": "env-cmd -f ../config/prod-local-tunnel.env ../bin/build-and-run-script.sh bin/dedupe-writs.js",
     "deploy:api:pre-prod": "bin/deploy.sh pre-prod",
     "deploy:api:prod": "bin/deploy.sh prod",