data/aws: use nlbs instead of elbs

We've noticed an elevated rate of installation failures recently. The
root cause appears to be 50-90 seconds of latency added to traffic going
through the internal ELB on port 49500. This was causing Ignition's
connections to timeout, resulting in the machines never provisioning.

AWS's NLBs don't seem to have this high latency, so we've decided to
move over to them instead. With the move to NLBs, we also get the
ability to add individual health checks for each port instead of just a
single health check for each load balancer. Also, NLBs are cheaper.

This commit drops support for ingress and the console. Since the console
and router aren't currently configured correctly, nobody should notice
that this is gone. It was easier to drop support in this commit rather
than continue to try to plumb through the existing implementation
knowing that it was going to have to change in the future. Once the
router has a strategy for ingress, we'll re-add this functionality using
the new NLBs.

This also drop support for the `<cluster-name>-k8s` DNS entry. We aren't
aware of any consumers and it was going to be tedious to keep this
working.

data/data/aws/bootstrap/main.tf

@@ -120,8 +120,16 @@ resource "aws_instance" "bootstrap" {
  volume_tags = "${var.tags}"
 }
 
-resource "aws_elb_attachment" "bootstrap" {
- count = "${var.elbs_length}"
- elb = "${var.elbs[count.index]}"
- instance = "${aws_instance.bootstrap.id}"
+resource "aws_lb_target_group_attachment" "public" {
+ count = "${var.public_target_group_arns_length}"
+
+ target_group_arn = "${var.public_target_group_arns[count.index]}"
+ target_id = "${aws_instance.bootstrap.private_ip}"
+}
+
+resource "aws_lb_target_group_attachment" "private" {
+ count = "${var.private_target_group_arns_length}"
+
+ target_group_arn = "${var.private_target_group_arns[count.index]}"
+ target_id = "${aws_instance.bootstrap.private_ip}"
 }

data/data/aws/bootstrap/variables.tf

@@ -18,16 +18,6 @@ variable "cluster_name" {
  description = "The name of the cluster."
 }
 
-variable "elbs" {
- type = "list"
- default = []
- description = "Elastic load balancer IDs to attach to the bootstrap node."
-}
-
-variable "elbs_length" {
- description = "The length of the 'elbs' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
-}
-
 variable "iam_role" {
  type = "string"
  default = ""
@@ -45,6 +35,26 @@ variable "instance_type" {
  description = "The EC2 instance type for the bootstrap node."
 }
 
+variable "private_target_group_arns" {
+ type = "list"
+ default = []
+ description = "The list of target group ARNs for the private load balancer."
+}
+
+variable "private_target_group_arns_length" {
+ description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
+}
+
+variable "public_target_group_arns" {
+ type = "list"
+ default = []
+ description = "The list of target group ARNs for the public load balancer."
+}
+
+variable "public_target_group_arns_length" {
+ description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
+}
+
 variable "subnet_id" {
  type = "string"
  description = "The subnet ID for the bootstrap node."

data/data/aws/main.tf

@@ -17,16 +17,18 @@ provider "aws" {
 module "bootstrap" {
  source = "./bootstrap"
 
- ami = "${var.tectonic_aws_ec2_ami_override}"
- associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
- bucket = "${aws_s3_bucket.bootstrap.id}"
- cluster_name = "${var.tectonic_cluster_name}"
- elbs = "${module.vpc.aws_lbs}"
- elbs_length = "${module.vpc.aws_lbs_length}"
- iam_role = "${var.tectonic_aws_master_iam_role_name}"
- ignition = "${var.ignition_bootstrap}"
- subnet_id = "${module.vpc.master_subnet_ids[0]}"
- vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]
+ ami = "${var.tectonic_aws_ec2_ami_override}"
+ associate_public_ip_address = "${var.tectonic_aws_endpoints != "private"}"
+ bucket = "${aws_s3_bucket.bootstrap.id}"
+ cluster_name = "${var.tectonic_cluster_name}"
+ public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
+ public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
+ private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
+ private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
+ iam_role = "${var.tectonic_aws_master_iam_role_name}"
+ ignition = "${var.ignition_bootstrap}"
+ subnet_id = "${module.vpc.master_subnet_ids[0]}"
+ vpc_security_group_ids = ["${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"]
 
  tags = "${merge(map(
  "Name", "${var.tectonic_cluster_name}-bootstrap",
@@ -37,25 +39,26 @@ module "bootstrap" {
 module "masters" {
  source = "./master"
 
- elb_api_internal_id = "${module.vpc.aws_elb_api_internal_id}"
- elb_api_external_id = "${module.vpc.aws_elb_api_external_id}"
- elb_console_id = "${module.vpc.aws_elb_console_id}"
- base_domain = "${var.tectonic_base_domain}"
- cluster_id = "${var.tectonic_cluster_id}"
- cluster_name = "${var.tectonic_cluster_name}"
- ec2_type = "${var.tectonic_aws_master_ec2_type}"
- extra_tags = "${var.tectonic_aws_extra_tags}"
- instance_count = "${var.tectonic_master_count}"
- master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
- master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
- private_endpoints = "${local.private_endpoints}"
- public_endpoints = "${local.public_endpoints}"
- root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
- root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
- root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
- subnet_ids = "${module.vpc.master_subnet_ids}"
- ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
- user_data_ign = "${var.ignition_master}"
+ public_target_group_arns = "${module.vpc.aws_lb_public_target_group_arns}"
+ public_target_group_arns_length = "${module.vpc.aws_lb_public_target_group_arns_length}"
+ private_target_group_arns = "${module.vpc.aws_lb_private_target_group_arns}"
+ private_target_group_arns_length = "${module.vpc.aws_lb_private_target_group_arns_length}"
+ base_domain = "${var.tectonic_base_domain}"
+ cluster_id = "${var.tectonic_cluster_id}"
+ cluster_name = "${var.tectonic_cluster_name}"
+ ec2_type = "${var.tectonic_aws_master_ec2_type}"
+ extra_tags = "${var.tectonic_aws_extra_tags}"
+ instance_count = "${var.tectonic_master_count}"
+ master_iam_role = "${var.tectonic_aws_master_iam_role_name}"
+ master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}"
+ private_endpoints = "${local.private_endpoints}"
+ public_endpoints = "${local.public_endpoints}"
+ root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}"
+ root_volume_size = "${var.tectonic_aws_master_root_volume_size}"
+ root_volume_type = "${var.tectonic_aws_master_root_volume_type}"
+ subnet_ids = "${module.vpc.master_subnet_ids}"
+ ec2_ami = "${var.tectonic_aws_ec2_ami_override}"
+ user_data_ign = "${var.ignition_master}"
 }
 
 module "iam" {
@@ -68,22 +71,19 @@ module "iam" {
 module "dns" {
  source = "./route53"
 
- api_external_elb_dns_name = "${module.vpc.aws_elb_api_external_dns_name}"
- api_external_elb_zone_id = "${module.vpc.aws_elb_api_external_zone_id}"
- api_internal_elb_dns_name = "${module.vpc.aws_elb_api_internal_dns_name}"
- api_internal_elb_zone_id = "${module.vpc.aws_elb_api_internal_zone_id}"
- api_ip_addresses = "${module.vpc.aws_lbs}"
- base_domain = "${var.tectonic_base_domain}"
- cluster_name = "${var.tectonic_cluster_name}"
- console_elb_dns_name = "${module.vpc.aws_console_dns_name}"
- console_elb_zone_id = "${module.vpc.aws_elb_console_zone_id}"
- elb_alias_enabled = true
- master_count = "${var.tectonic_master_count}"
- private_zone_id = "${local.private_zone_id}"
- external_vpc_id = "${module.vpc.vpc_id}"
- extra_tags = "${var.tectonic_aws_extra_tags}"
- private_endpoints = "${local.private_endpoints}"
- public_endpoints = "${local.public_endpoints}"
+ api_external_lb_dns_name = "${module.vpc.aws_lb_api_external_dns_name}"
+ api_external_lb_zone_id = "${module.vpc.aws_lb_api_external_zone_id}"
+ api_internal_lb_dns_name = "${module.vpc.aws_lb_api_internal_dns_name}"
+ api_internal_lb_zone_id = "${module.vpc.aws_lb_api_internal_zone_id}"
+ base_domain = "${var.tectonic_base_domain}"
+ cluster_name = "${var.tectonic_cluster_name}"
+ elb_alias_enabled = true
+ master_count = "${var.tectonic_master_count}"
+ private_zone_id = "${local.private_zone_id}"
+ external_vpc_id = "${module.vpc.vpc_id}"
+ extra_tags = "${var.tectonic_aws_extra_tags}"
+ private_endpoints = "${local.private_endpoints}"
+ public_endpoints = "${local.public_endpoints}"
 }
 
 module "vpc" {

data/data/aws/master/main.tf

@@ -113,20 +113,16 @@ resource "aws_instance" "master" {
  ), var.extra_tags)}"
 }
 
-resource "aws_elb_attachment" "masters_internal" {
- count = "${var.private_endpoints ? var.instance_count : 0}"
- elb = "${var.elb_api_internal_id}"
- instance = "${aws_instance.master.*.id[count.index]}"
-}
+resource "aws_lb_target_group_attachment" "public" {
+ count = "${var.public_endpoints ? var.instance_count * var.public_target_group_arns_length : 0}"
 
-resource "aws_elb_attachment" "masters_external" {
- count = "${var.public_endpoints ? var.instance_count : 0}"
- elb = "${var.elb_api_external_id}"
- instance = "${aws_instance.master.*.id[count.index]}"
+ target_group_arn = "${var.public_target_group_arns[count.index % var.public_target_group_arns_length]}"
+ target_id = "${aws_instance.master.*.private_ip[count.index / var.public_target_group_arns_length]}"
 }
 
-resource "aws_elb_attachment" "masters_console" {
- count = "${var.instance_count}"
- elb = "${var.elb_console_id}"
- instance = "${aws_instance.master.*.id[count.index]}"
+resource "aws_lb_target_group_attachment" "private" {
+ count = "${var.private_endpoints ? var.instance_count * var.private_target_group_arns_length : 0}"
+
+ target_group_arn = "${var.private_target_group_arns[count.index % var.private_target_group_arns_length]}"
+ target_id = "${aws_instance.master.*.private_ip[count.index / var.private_target_group_arns_length]}"
 }

data/data/aws/master/variables.tf

@@ -46,21 +46,29 @@ variable "private_endpoints" {
  default = true
 }
 
+variable "private_target_group_arns" {
+ type = "list"
+ default = []
+ description = "The list of target group ARNs for the private load balancer."
+}
+
+variable "private_target_group_arns_length" {
+ description = "The length of the 'private_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
+}
+
 variable "public_endpoints" {
  description = "If set to true, public-facing ingress resources are created."
  default = true
 }
 
-variable "elb_api_internal_id" {
- type = "string"
-}
-
-variable "elb_api_external_id" {
- type = "string"
+variable "public_target_group_arns" {
+ type = "list"
+ default = []
+ description = "The list of target group ARNs for the public load balancer."
 }
 
-variable "elb_console_id" {
- type = "string"
+variable "public_target_group_arns_length" {
+ description = "The length of the 'public_target_group_arns' variable, to work around https://github.com/hashicorp/terraform/issues/12570."
 }
 
 variable "root_volume_iops" {

data/data/aws/route53/tectonic.tf

@@ -10,103 +10,33 @@ data "aws_route53_zone" "tectonic" {
 locals {
  public_zone_id = "${join("", data.aws_route53_zone.tectonic.*.zone_id)}"
 
- zone_id = "${var.private_endpoints ?
- var.private_zone_id :
- local.public_zone_id}"
-}
-
-resource "aws_route53_record" "tectonic_api" {
- count = "${var.elb_alias_enabled ? 0 : 1}"
- zone_id = "${local.public_zone_id}"
- name = "${var.cluster_name}-k8s"
- type = "A"
- ttl = "60"
- records = ["${var.api_ip_addresses}"]
+ zone_id = "${var.private_endpoints ? var.private_zone_id : local.public_zone_id}"
 }
 
 resource "aws_route53_record" "tectonic_api_external" {
- count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
+ count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
+
  zone_id = "${local.public_zone_id}"
  name = "${var.cluster_name}-api.${var.base_domain}"
  type = "A"
 
  alias {
- name = "${var.api_external_elb_dns_name}"
- zone_id = "${var.api_external_elb_zone_id}"
+ name = "${var.api_external_lb_dns_name}"
+ zone_id = "${var.api_external_lb_zone_id}"
  evaluate_target_health = true
  }
 }
 
 resource "aws_route53_record" "tectonic_api_internal" {
- count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
- zone_id = "${var.private_zone_id}"
- name = "${var.cluster_name}-api.${var.base_domain}"
- type = "A"
-
- alias {
- name = "${var.api_internal_elb_dns_name}"
- zone_id = "${var.api_internal_elb_zone_id}"
- evaluate_target_health = true
- }
-}
-
-resource "aws_route53_record" "tectonic-console" {
- count = "${var.elb_alias_enabled ? 0 : 1}"
- zone_id = "${local.public_zone_id}"
- name = "${var.cluster_name}"
- type = "A"
- ttl = "60"
- records = ["${var.worker_ip_addresses}"]
-}
-
-resource "aws_route53_record" "tectonic_ingress_public" {
- count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
- zone_id = "${local.public_zone_id}"
- name = "${var.cluster_name}.${var.base_domain}"
- type = "A"
-
- alias {
- name = "${var.console_elb_dns_name}"
- zone_id = "${var.console_elb_zone_id}"
- evaluate_target_health = true
- }
-}
+ count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
 
-resource "aws_route53_record" "tectonic_ingress_private" {
- count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
  zone_id = "${var.private_zone_id}"
- name = "${var.cluster_name}.${var.base_domain}"
- type = "A"
-
- alias {
- name = "${var.console_elb_dns_name}"
- zone_id = "${var.console_elb_zone_id}"
- evaluate_target_health = true
- }
-}
-
-resource "aws_route53_record" "routes_ingress_public" {
- count = "${var.elb_alias_enabled ? local.public_endpoints_count : 0}"
- zone_id = "${local.public_zone_id}"
- name = "*.${var.cluster_name}.${var.base_domain}"
- type = "A"
-
- alias {
- name = "${var.console_elb_dns_name}"
- zone_id = "${var.console_elb_zone_id}"
- evaluate_target_health = true
- }
-}
-
-resource "aws_route53_record" "routes_ingress_private" {
- count = "${var.elb_alias_enabled ? local.private_endpoints_count : 0}"
- zone_id = "${var.private_zone_id}"
- name = "*.${var.cluster_name}.${var.base_domain}"
+ name = "${var.cluster_name}-api.${var.base_domain}"
  type = "A"
 
  alias {
- name = "${var.console_elb_dns_name}"
- zone_id = "${var.console_elb_zone_id}"
+ name = "${var.api_internal_lb_dns_name}"
+ zone_id = "${var.api_internal_lb_zone_id}"
  evaluate_target_health = true
  }
 }