diff --git a/modules/aws/etcd/ignition.tf b/modules/aws/etcd/ignition.tf deleted file mode 100644 index 9ad9123b92..0000000000 --- a/modules/aws/etcd/ignition.tf +++ /dev/null @@ -1,37 +0,0 @@ -data "ignition_config" "etcd" { - count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}" - - systemd = [ - "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}", - "${var.ign_etcd_dropin_id_list[count.index]}", - ] - - files = ["${compact(list( - var.ign_profile_env_id, - var.ign_systemd_default_env_id, - ))}", - "${var.ign_etcd_crt_id_list}", - ] -} - -data "ignition_systemd_unit" "locksmithd" { - count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}" - - name = "locksmithd.service" - enabled = true - - dropin = [ - { - name = "40-etcd-lock.conf" - - content = < 1') - -if [ "$API_HEALTHY" == "true" ]; then - echo "Healthy API instances found, cluster is already installed." - echo -n "false" >/run/metadata/master - exit 0 -fi - -BOOTKUBE_MASTER=$(echo "$ASG_INSTANCE_IDS" | head -n1) - -if [ "$BOOTKUBE_MASTER" != "$INSTANCE_ID" ]; then - echo "This instance is not the bootkube master, '$BOOTKUBE_MASTER' is." - echo -n "false" >/run/metadata/master - exit 0 -fi - -echo -n "true" >/run/metadata/master diff --git a/modules/aws/master-asg/resources/init-assets.sh b/modules/aws/master-asg/resources/init-assets.sh deleted file mode 100644 index bb41014e38..0000000000 --- a/modules/aws/master-asg/resources/init-assets.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -e -set -o pipefail - -detect_master() { - mkdir -p /run/metadata - # shellcheck disable=SC2086,SC2154 - /usr/bin/docker run \ - --volume /run/metadata:/run/metadata \ - --volume /opt/detect-master.sh:/detect-master.sh:ro \ - --network=host \ - --env CLUSTER_NAME=${cluster_name} \ - --entrypoint=/detect-master.sh \ - ${awscli_image} -} - -until detect_master; do - echo "failed to detect master; retrying in 5 seconds" - sleep 5 -done - -MASTER=$(cat /run/metadata/master) -if [ "$MASTER" != "true" ]; then - exit 0 -fi - -# Download the assets from S3. -# shellcheck disable=SC2154 -/usr/bin/bash /opt/s3-puller.sh "${assets_s3_location}" /var/tmp/tectonic.zip -unzip -o -d /var/tmp/tectonic/ /var/tmp/tectonic.zip -rm /var/tmp/tectonic.zip -# make files in /opt/tectonic available atomically -mv /var/tmp/tectonic /opt/tectonic - -exit 0 diff --git a/modules/aws/master-asg/variables-ignition.tf b/modules/aws/master-asg/variables-ignition.tf deleted file mode 120000 index e22a48c662..0000000000 --- a/modules/aws/master-asg/variables-ignition.tf +++ /dev/null @@ -1 +0,0 @@ -../../ignition/outputs.import \ No newline at end of file diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf index 18c6fc7f2e..3244897979 100644 --- a/modules/aws/master-asg/variables.tf +++ b/modules/aws/master-asg/variables.tf @@ -1,8 +1,3 @@ -variable "assets_s3_location" { - type = "string" - description = "Location on S3 of the Bootkube/Tectonic assets to use (bucket/key)" -} - variable "autoscaling_group_extra_tags" { description = "Extra AWS tags to be applied to created autoscaling group resources." type = "list" @@ -45,10 +40,6 @@ variable "extra_tags" { default = {} } -variable "ign_s3_puller_id" { - type = "string" -} - variable "ec2_ami" { type = "string" default = "" @@ -109,41 +100,12 @@ variable "subnet_ids" { type = "list" } -variable "ign_bootkube_service_id" { - type = "string" - description = "The ID of the bootkube systemd service unit" -} - -variable "ign_bootkube_path_unit_id" { - type = "string" -} - -variable "ign_tectonic_service_id" { - type = "string" - description = "The ID of the tectonic installer systemd service unit" -} - -variable "ign_tectonic_path_unit_id" { - type = "string" -} - -variable "ign_init_assets_service_id" { - type = "string" -} - -variable "ign_rm_assets_service_id" { - type = "string" -} - -variable "ign_rm_assets_path_unit_id" { - type = "string" -} - -variable "s3_bucket" { - type = "string" +variable "dns_server_ip" { + type = "string" + default = "" } -variable "dns_server_ip" { +variable "kubeconfig_content" { type = "string" default = "" } diff --git a/modules/aws/vpc/master-elb.tf b/modules/aws/vpc/master-elb.tf index 4f6e1a4d07..1c1be84932 100644 --- a/modules/aws/vpc/master-elb.tf +++ b/modules/aws/vpc/master-elb.tf @@ -1,3 +1,36 @@ +resource "aws_elb" "ncg" { + count = "${var.private_master_endpoints}" + name = "${var.cluster_name}-ncg" + subnets = ["${local.master_subnet_ids}"] + internal = true + security_groups = ["${aws_security_group.ncg.id}"] + + idle_timeout = 3600 + connection_draining = true + connection_draining_timeout = 300 + + listener { + instance_port = 8080 + instance_protocol = "tcp" + lb_port = 80 + lb_protocol = "tcp" + } + + health_check { + healthy_threshold = 2 + unhealthy_threshold = 2 + timeout = 3 + target = "TCP:8080" + interval = 5 + } + + tags = "${merge(map( + "Name", "${var.cluster_name}-int", + "kubernetes.io/cluster/${var.cluster_name}", "owned", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" +} + resource "aws_elb" "api_internal" { count = "${var.private_master_endpoints}" name = "${var.cluster_name}-int" diff --git a/modules/aws/vpc/outputs.tf b/modules/aws/vpc/outputs.tf index 5471abf6ad..121850c083 100644 --- a/modules/aws/vpc/outputs.tf +++ b/modules/aws/vpc/outputs.tf @@ -45,7 +45,7 @@ output "aws_elb_console_id" { } output "aws_lbs" { - value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id))}"] + value = ["${compact(concat(aws_elb.api_internal.*.id, list(aws_elb.console.id), aws_elb.api_external.*.id, aws_elb.ncg.*.id))}"] } output "aws_api_external_dns_name" { @@ -71,3 +71,11 @@ output "aws_console_dns_name" { output "aws_elb_console_zone_id" { value = "${aws_elb.console.zone_id}" } + +output "aws_elb_ncg_dns_name" { + value = "${element(concat(aws_elb.ncg.*.dns_name, list("")), 0)}" +} + +output "aws_elb_ncg_zone_id" { + value = "${aws_elb.ncg.zone_id}" +} diff --git a/modules/aws/vpc/sg-elb.tf b/modules/aws/vpc/sg-elb.tf index 22fb7a2a90..a7f15f32d2 100644 --- a/modules/aws/vpc/sg-elb.tf +++ b/modules/aws/vpc/sg-elb.tf @@ -1,3 +1,35 @@ +resource "aws_security_group" "ncg" { + vpc_id = "${data.aws_vpc.cluster_vpc.id}" + + tags = "${merge(map( + "Name", "${var.cluster_name}_console_sg", + "kubernetes.io/cluster/${var.cluster_name}", "owned", + "tectonicClusterID", "${var.cluster_id}" + ), var.extra_tags)}" + + egress { + from_port = 0 + to_port = 0 + protocol = "-1" + self = true + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 80 + to_port = 80 + } + + ingress { + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 443 + to_port = 443 + } +} + resource "aws_security_group" "api" { vpc_id = "${data.aws_vpc.cluster_vpc.id}" diff --git a/modules/aws/vpc/sg-master.tf b/modules/aws/vpc/sg-master.tf index 22430ced0f..ac08780279 100644 --- a/modules/aws/vpc/sg-master.tf +++ b/modules/aws/vpc/sg-master.tf @@ -8,6 +8,16 @@ resource "aws_security_group" "master" { ), var.extra_tags)}" } +resource "aws_security_group_rule" "master_ncg" { + type = "ingress" + security_group_id = "${aws_security_group.master.id}" + + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + from_port = 8080 + to_port = 8080 +} + resource "aws_security_group_rule" "master_egress" { type = "egress" security_group_id = "${aws_security_group.master.id}" diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf deleted file mode 100644 index 742d6e1558..0000000000 --- a/modules/aws/worker-asg/ignition.tf +++ /dev/null @@ -1,21 +0,0 @@ -data "ignition_config" "main" { - files = ["${compact(list( - var.ign_installer_kubelet_env_id, - var.ign_installer_runtime_mappings_id, - var.ign_max_user_watches_id, - var.ign_s3_puller_id, - var.ign_profile_env_id, - var.ign_systemd_default_env_id, - ))}", - "${var.ign_ca_cert_id_list}", - ] - - systemd = [ - "${var.ign_docker_dropin_id}", - "${var.ign_k8s_node_bootstrap_service_id}", - "${var.ign_kubelet_service_id}", - "${var.ign_locksmithd_service_id}", - "${var.ign_update_ca_certificates_dropin_id}", - "${var.ign_iscsi_service_id}", - ] -} diff --git a/modules/aws/worker-asg/ignition_s3.tf b/modules/aws/worker-asg/ignition_s3.tf deleted file mode 100644 index 9766c6ba5b..0000000000 --- a/modules/aws/worker-asg/ignition_s3.tf +++ /dev/null @@ -1,21 +0,0 @@ -resource "aws_s3_bucket_object" "ignition_worker" { - bucket = "${var.s3_bucket}" - key = "ignition_worker.json" - content = "${data.ignition_config.main.rendered}" - acl = "private" - - server_side_encryption = "AES256" - - tags = "${merge(map( - "Name", "${var.cluster_name}-ignition-worker", - "KubernetesCluster", "${var.cluster_name}", - "tectonicClusterID", "${var.cluster_id}" - ), var.extra_tags)}" -} - -data "ignition_config" "s3" { - replace { - source = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_worker.key)}" - verification = "sha512-${sha512(data.ignition_config.main.rendered)}" - } -} diff --git a/modules/aws/worker-asg/variables-ignition.tf b/modules/aws/worker-asg/variables-ignition.tf deleted file mode 120000 index e22a48c662..0000000000 --- a/modules/aws/worker-asg/variables-ignition.tf +++ /dev/null @@ -1 +0,0 @@ -../../ignition/outputs.import \ No newline at end of file diff --git a/modules/aws/worker-asg/variables.tf b/modules/aws/worker-asg/variables.tf index 2533d0b00e..ca2fdd5bf2 100644 --- a/modules/aws/worker-asg/variables.tf +++ b/modules/aws/worker-asg/variables.tf @@ -84,15 +84,12 @@ variable "worker_iam_role" { description = "IAM role to use for the instance profiles of worker nodes." } -variable "ign_s3_puller_id" { - type = "string" -} - -variable "s3_bucket" { - type = "string" +variable "base_domain" { + type = "string" + description = "Domain on which the ELB records will be created" } -variable "dns_server_ip" { +variable "kubeconfig_content" { type = "string" default = "" } diff --git a/modules/aws/worker-asg/worker.tf b/modules/aws/worker-asg/worker.tf index b40a38ba46..79a058e6fb 100644 --- a/modules/aws/worker-asg/worker.tf +++ b/modules/aws/worker-asg/worker.tf @@ -25,6 +25,24 @@ data "aws_ami" "coreos_ami" { } } +data "ignition_config" "ncg" { + append { + source = "http://${var.cluster_name}-ncg.${var.base_domain}/ignition?profile=worker" + } + + files = ["${data.ignition_file.kubeconfig.id}"] +} + +data "ignition_file" "kubeconfig" { + filesystem = "root" + path = "/etc/kubernetes/kubeconfig" + mode = 0644 + + content { + content = "${var.kubeconfig_content}" + } +} + resource "aws_launch_configuration" "worker_conf" { instance_type = "${var.ec2_type}" image_id = "${coalesce(var.ec2_ami, data.aws_ami.coreos_ami.image_id)}" @@ -32,7 +50,7 @@ resource "aws_launch_configuration" "worker_conf" { key_name = "${var.ssh_key}" security_groups = ["${var.sg_ids}"] iam_instance_profile = "${aws_iam_instance_profile.worker_profile.arn}" - user_data = "${data.ignition_config.s3.rendered}" + user_data = "${data.ignition_config.ncg.rendered}" lifecycle { create_before_destroy = true diff --git a/modules/dns/route53/tectonic.tf b/modules/dns/route53/tectonic.tf index 730251a061..e3e337375b 100644 --- a/modules/dns/route53/tectonic.tf +++ b/modules/dns/route53/tectonic.tf @@ -15,7 +15,7 @@ locals { } resource "aws_route53_zone" "tectonic_int" { - count = "${var.tectonic_private_endpoints ? "${var.tectonic_external_private_zone == "" ? 1 : 0 }" : 0}" + count = "0" vpc_id = "${var.tectonic_external_vpc_id}" name = "${var.base_domain}" force_destroy = true diff --git a/platforms/aws/inputs.tf b/platforms/aws/inputs.tf index 34d5ee4cce..118e039ca2 100644 --- a/platforms/aws/inputs.tf +++ b/platforms/aws/inputs.tf @@ -27,4 +27,6 @@ locals { tectonic_path_unit = "${data.terraform_remote_state.assets.tectonic_path_unit}" bootkube_service = "${data.terraform_remote_state.assets.bootkube_service}" bootkube_path_unit = "${data.terraform_remote_state.assets.bootkube_path_unit}" + kubeconfig_content = "${data.terraform_remote_state.assets.kubeconfig_content}" + s3_bucket_domain_name = "${data.terraform_remote_state.assets.s3_bucket_domain_name}" } diff --git a/platforms/aws/main.tf b/platforms/aws/main.tf index a1fa5c1fe2..c228102c58 100644 --- a/platforms/aws/main.tf +++ b/platforms/aws/main.tf @@ -75,170 +75,77 @@ module "vpc" { module "etcd" { source = "../../modules/aws/etcd" - base_domain = "${var.tectonic_base_domain}" - cluster_id = "${local.cluster_id}" - cluster_name = "${var.tectonic_cluster_name}" - container_image = "${var.tectonic_container_images["etcd"]}" - container_linux_channel = "${var.tectonic_container_linux_channel}" - container_linux_version = "${module.container_linux.version}" - ec2_type = "${var.tectonic_aws_etcd_ec2_type}" - external_endpoints = "${compact(var.tectonic_etcd_servers)}" - extra_tags = "${var.tectonic_aws_extra_tags}" - ign_etcd_crt_id_list = "${module.ignition_masters.etcd_crt_id_list}" - ign_etcd_dropin_id_list = "${module.ignition_masters.etcd_dropin_id_list}" - ign_profile_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_masters.profile_env_id : ""}" - ign_systemd_default_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_masters.systemd_default_env_id : ""}" - instance_count = "${length(data.template_file.etcd_hostname_list.*.id)}" - root_volume_iops = "${var.tectonic_aws_etcd_root_volume_iops}" - root_volume_size = "${var.tectonic_aws_etcd_root_volume_size}" - root_volume_type = "${var.tectonic_aws_etcd_root_volume_type}" - s3_bucket = "${local.s3_bucket}" - sg_ids = "${concat(var.tectonic_aws_etcd_extra_sg_ids, list(module.vpc.etcd_sg_id))}" - ssh_key = "${var.tectonic_aws_ssh_key}" - subnets = "${module.vpc.worker_subnet_ids}" - etcd_iam_role = "${var.tectonic_aws_etcd_iam_role_name}" - ec2_ami = "${var.tectonic_aws_ec2_ami_override}" -} - -module "ignition_masters" { - source = "../../modules/ignition" - - assets_location = "${local.tectonic_bucket}/${local.tectonic_key}" - base_domain = "${var.tectonic_base_domain}" - bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" - cloud_provider = "aws" - cluster_name = "${var.tectonic_cluster_name}" - container_images = "${var.tectonic_container_images}" - custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}" - etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}" - etcd_ca_cert_pem = "${local.etcd_ca_crt_pem}" - etcd_client_crt_pem = "${local.etcd_client_crt_pem}" - etcd_client_key_pem = "${local.etcd_client_key_pem}" - etcd_count = "${length(data.template_file.etcd_hostname_list.*.id)}" - etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}" - etcd_peer_crt_pem = "${local.etcd_peer_crt_pem}" - etcd_peer_key_pem = "${local.etcd_peer_key_pem}" - etcd_server_crt_pem = "${local.etcd_server_crt_pem}" - etcd_server_key_pem = "${local.etcd_server_key_pem}" - http_proxy = "${var.tectonic_http_proxy_address}" - https_proxy = "${var.tectonic_https_proxy_address}" - image_re = "${var.tectonic_image_re}" - ingress_ca_cert_pem = "${local.ingress_certs_ca_cert_pem}" - iscsi_enabled = "${var.tectonic_iscsi_enabled}" - kube_ca_cert_pem = "${local.kube_certs_ca_cert_pem}" - kube_dns_service_ip = "${local.kube_dns_service_ip}" - kubeconfig_fetch_cmd = "/opt/s3-puller.sh ${local.kubeconfig_bucket}/${local.kubeconfig_key} /etc/kubernetes/kubeconfig" - kubelet_debug_config = "${var.tectonic_kubelet_debug_config}" - kubelet_node_label = "node-role.kubernetes.io/master" - kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" - no_proxy = "${var.tectonic_no_proxy}" + base_domain = "${var.tectonic_base_domain}" + cluster_id = "${local.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_image = "${var.tectonic_container_images["etcd"]}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_type = "${var.tectonic_aws_etcd_ec2_type}" + external_endpoints = "${compact(var.tectonic_etcd_servers)}" + extra_tags = "${var.tectonic_aws_extra_tags}" + instance_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + root_volume_iops = "${var.tectonic_aws_etcd_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_etcd_root_volume_size}" + root_volume_type = "${var.tectonic_aws_etcd_root_volume_type}" + s3_bucket = "${local.s3_bucket}" + sg_ids = "${concat(var.tectonic_aws_etcd_extra_sg_ids, list(module.vpc.etcd_sg_id))}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnets = "${module.vpc.worker_subnet_ids}" + etcd_iam_role = "${var.tectonic_aws_etcd_iam_role_name}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" } module "masters" { source = "../../modules/aws/master-asg" - assets_s3_location = "${local.tectonic_bucket}/${local.tectonic_key}" - autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" - aws_lbs = "${module.vpc.aws_lbs}" - base_domain = "${var.tectonic_base_domain}" - cluster_id = "${local.cluster_id}" - cluster_name = "${var.tectonic_cluster_name}" - container_images = "${var.tectonic_container_images}" - container_linux_channel = "${var.tectonic_container_linux_channel}" - container_linux_version = "${module.container_linux.version}" - ec2_type = "${var.tectonic_aws_master_ec2_type}" - extra_tags = "${var.tectonic_aws_extra_tags}" - ign_bootkube_path_unit_id = "${data.ignition_systemd_unit.bootkube_path_unit.id}" - ign_bootkube_service_id = "${data.ignition_systemd_unit.bootkube_service.id}" - ign_ca_cert_id_list = "${module.ignition_masters.ca_cert_id_list}" - ign_docker_dropin_id = "${module.ignition_masters.docker_dropin_id}" - ign_init_assets_service_id = "${module.ignition_masters.init_assets_service_id}" - ign_installer_kubelet_env_id = "${module.ignition_masters.installer_kubelet_env_id}" - ign_installer_runtime_mappings_id = "${module.ignition_masters.installer_runtime_mappings_id}" - ign_iscsi_service_id = "${module.ignition_masters.iscsi_service_id}" - ign_k8s_node_bootstrap_service_id = "${module.ignition_masters.k8s_node_bootstrap_service_id}" - ign_kubelet_service_id = "${module.ignition_masters.kubelet_service_id}" - ign_locksmithd_service_id = "${module.ignition_masters.locksmithd_service_id}" - ign_max_user_watches_id = "${module.ignition_masters.max_user_watches_id}" - ign_profile_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_masters.profile_env_id : ""}" - ign_rm_assets_path_unit_id = "${module.ignition_masters.rm_assets_path_unit_id}" - ign_rm_assets_service_id = "${module.ignition_masters.rm_assets_service_id}" - ign_s3_puller_id = "${module.ignition_masters.s3_puller_id}" - ign_systemd_default_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_masters.systemd_default_env_id : ""}" - ign_tectonic_path_unit_id = "${data.ignition_systemd_unit.tectonic_path_unit.id}" - ign_tectonic_service_id = "${data.ignition_systemd_unit.tectonic_service.id}" - ign_update_ca_certificates_dropin_id = "${module.ignition_masters.update_ca_certificates_dropin_id}" - instance_count = "1" - master_iam_role = "${var.tectonic_aws_master_iam_role_name}" - master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}" - private_endpoints = "${var.tectonic_aws_private_endpoints}" - public_endpoints = "${var.tectonic_aws_public_endpoints}" - root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}" - root_volume_size = "${var.tectonic_aws_master_root_volume_size}" - root_volume_type = "${var.tectonic_aws_master_root_volume_type}" - s3_bucket = "${local.tectonic_bucket}" - ssh_key = "${var.tectonic_aws_ssh_key}" - subnet_ids = "${module.vpc.master_subnet_ids}" - ec2_ami = "${var.tectonic_aws_ec2_ami_override}" -} - -module "ignition_workers" { - source = "../../modules/ignition" - - bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" - cloud_provider = "aws" - container_images = "${var.tectonic_container_images}" - custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}" - etcd_ca_cert_pem = "${local.etcd_ca_crt_pem}" - http_proxy = "${var.tectonic_http_proxy_address}" - https_proxy = "${var.tectonic_https_proxy_address}" - image_re = "${var.tectonic_image_re}" - ingress_ca_cert_pem = "${local.ingress_certs_ca_cert_pem}" - iscsi_enabled = "${var.tectonic_iscsi_enabled}" - kube_ca_cert_pem = "${local.kube_certs_ca_cert_pem}" - kube_dns_service_ip = "${local.kube_dns_service_ip}" - kubeconfig_fetch_cmd = "/opt/s3-puller.sh ${local.kubeconfig_bucket}/${local.kubeconfig_key} /etc/kubernetes/kubeconfig" - kubelet_debug_config = "${var.tectonic_kubelet_debug_config}" - kubelet_node_label = "node-role.kubernetes.io/node" - kubelet_node_taints = "" - no_proxy = "${var.tectonic_no_proxy}" + autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" + aws_lbs = "${module.vpc.aws_lbs}" + base_domain = "${var.tectonic_base_domain}" + cluster_id = "${local.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_images = "${var.tectonic_container_images}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_type = "${var.tectonic_aws_master_ec2_type}" + extra_tags = "${var.tectonic_aws_extra_tags}" + instance_count = "1" + master_iam_role = "${var.tectonic_aws_master_iam_role_name}" + master_sg_ids = "${concat(var.tectonic_aws_master_extra_sg_ids, list(module.vpc.master_sg_id))}" + private_endpoints = "${var.tectonic_aws_private_endpoints}" + public_endpoints = "${var.tectonic_aws_public_endpoints}" + root_volume_iops = "${var.tectonic_aws_master_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_master_root_volume_size}" + root_volume_type = "${var.tectonic_aws_master_root_volume_type}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnet_ids = "${module.vpc.master_subnet_ids}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + kubeconfig_content = "${local.kubeconfig_content}" } module "workers" { source = "../../modules/aws/worker-asg" - autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" - cluster_id = "${local.cluster_id}" - cluster_name = "${var.tectonic_cluster_name}" - container_linux_channel = "${var.tectonic_container_linux_channel}" - container_linux_version = "${module.container_linux.version}" - ec2_type = "${var.tectonic_aws_worker_ec2_type}" - extra_tags = "${var.tectonic_aws_extra_tags}" - ign_ca_cert_id_list = "${module.ignition_masters.ca_cert_id_list}" - ign_docker_dropin_id = "${module.ignition_workers.docker_dropin_id}" - ign_installer_kubelet_env_id = "${module.ignition_workers.installer_kubelet_env_id}" - ign_installer_runtime_mappings_id = "${module.ignition_workers.installer_runtime_mappings_id}" - ign_iscsi_service_id = "${module.ignition_workers.iscsi_service_id}" - ign_k8s_node_bootstrap_service_id = "${module.ignition_workers.k8s_node_bootstrap_service_id}" - ign_kubelet_service_id = "${module.ignition_workers.kubelet_service_id}" - ign_locksmithd_service_id = "${module.ignition_workers.locksmithd_service_id}" - ign_max_user_watches_id = "${module.ignition_workers.max_user_watches_id}" - ign_profile_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_workers.profile_env_id : ""}" - ign_s3_puller_id = "${module.ignition_workers.s3_puller_id}" - ign_systemd_default_env_id = "${local.tectonic_http_proxy_enabled ? module.ignition_workers.systemd_default_env_id : ""}" - ign_update_ca_certificates_dropin_id = "${module.ignition_workers.update_ca_certificates_dropin_id}" - instance_count = "0" - load_balancers = "${var.tectonic_aws_worker_load_balancers}" - root_volume_iops = "${var.tectonic_aws_worker_root_volume_iops}" - root_volume_size = "${var.tectonic_aws_worker_root_volume_size}" - root_volume_type = "${var.tectonic_aws_worker_root_volume_type}" - s3_bucket = "${local.s3_bucket}" - sg_ids = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}" - ssh_key = "${var.tectonic_aws_ssh_key}" - subnet_ids = "${module.vpc.worker_subnet_ids}" - vpc_id = "${module.vpc.vpc_id}" - worker_iam_role = "${var.tectonic_aws_worker_iam_role_name}" - ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + autoscaling_group_extra_tags = "${var.tectonic_autoscaling_group_extra_tags}" + cluster_id = "${local.cluster_id}" + cluster_name = "${var.tectonic_cluster_name}" + container_linux_channel = "${var.tectonic_container_linux_channel}" + container_linux_version = "${module.container_linux.version}" + ec2_type = "${var.tectonic_aws_worker_ec2_type}" + extra_tags = "${var.tectonic_aws_extra_tags}" + instance_count = "0" + load_balancers = "${var.tectonic_aws_worker_load_balancers}" + root_volume_iops = "${var.tectonic_aws_worker_root_volume_iops}" + root_volume_size = "${var.tectonic_aws_worker_root_volume_size}" + root_volume_type = "${var.tectonic_aws_worker_root_volume_type}" + sg_ids = "${concat(var.tectonic_aws_worker_extra_sg_ids, list(module.vpc.worker_sg_id))}" + ssh_key = "${var.tectonic_aws_ssh_key}" + subnet_ids = "${module.vpc.worker_subnet_ids}" + vpc_id = "${module.vpc.vpc_id}" + worker_iam_role = "${var.tectonic_aws_worker_iam_role_name}" + ec2_ami = "${var.tectonic_aws_ec2_ami_override}" + base_domain = "${var.tectonic_base_domain}" + kubeconfig_content = "${local.kubeconfig_content}" } module "dns" { @@ -260,33 +167,9 @@ module "dns" { etcd_ip_addresses = "${module.etcd.ip_addresses}" external_endpoints = ["${compact(var.tectonic_etcd_servers)}"] master_count = "${var.tectonic_master_count}" - tectonic_external_private_zone = "${var.tectonic_aws_external_private_zone}" + tectonic_external_private_zone = "${aws_route53_zone.tectonic_int.id}" tectonic_external_vpc_id = "${module.vpc.vpc_id}" tectonic_extra_tags = "${var.tectonic_aws_extra_tags}" tectonic_private_endpoints = "${var.tectonic_aws_private_endpoints}" tectonic_public_endpoints = "${var.tectonic_aws_public_endpoints}" } - -data "ignition_systemd_unit" "tectonic_service" { - name = "tectonic.service" - enabled = false - content = "${local.tectonic_service}" -} - -data "ignition_systemd_unit" "tectonic_path_unit" { - name = "tectonic.path" - enabled = true - content = "${local.tectonic_path_unit}" -} - -data "ignition_systemd_unit" "bootkube_service" { - name = "bootkube.service" - enabled = false - content = "${local.bootkube_service}" -} - -data "ignition_systemd_unit" "bootkube_path_unit" { - name = "bootkube.path" - enabled = true - content = "${local.bootkube_path_unit}" -} diff --git a/platforms/aws/ncg-s3.tf b/platforms/aws/ncg-s3.tf new file mode 100644 index 0000000000..1b1d445d45 --- /dev/null +++ b/platforms/aws/ncg-s3.tf @@ -0,0 +1,21 @@ +# NCG +resource "aws_route53_zone" "tectonic_int" { + vpc_id = "${module.vpc.vpc_id}" + name = "${var.tectonic_base_domain}" + force_destroy = true + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}_tectonic_int", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${local.cluster_id}" + ), var.tectonic_aws_extra_tags)}" +} + +resource "aws_route53_record" "tectonic_ncg" { + zone_id = "${aws_route53_zone.tectonic_int.id}" + name = "${var.tectonic_cluster_name}-ncg.${var.tectonic_base_domain}" + type = "CNAME" + ttl = "1" + + records = ["${local.s3_bucket_domain_name}"] +} diff --git a/platforms/aws/outputs.tf b/platforms/aws/outputs.tf index 7bb7dc21a0..f892bbf7fd 100644 --- a/platforms/aws/outputs.tf +++ b/platforms/aws/outputs.tf @@ -31,3 +31,16 @@ output "subnet_ids_workers" { output "aws_lbs_workers" { value = "${module.workers.aws_lbs}" } + +# NCG +output "private_zone_id" { + value = "${aws_route53_zone.tectonic_int.id}" +} + +output "ncg_elb_dns_name" { + value = "${module.vpc.aws_elb_ncg_dns_name}" +} + +output "ncg_elb_zone_id" { + value = "${module.vpc.aws_elb_api_internal_zone_id}" +} diff --git a/steps/assets/ignition-bootstrap.tf b/steps/assets/ignition-bootstrap.tf new file mode 100644 index 0000000000..00206ea778 --- /dev/null +++ b/steps/assets/ignition-bootstrap.tf @@ -0,0 +1,120 @@ +resource "aws_s3_bucket_object" "ignition_bootstrap" { + bucket = "${aws_s3_bucket.tectonic.bucket}" + key = "ignition" + content = "${data.ignition_config.bootstrap.rendered}" + acl = "public-read" + + # TODO: Lock down permissions. + # At the minute this is pulic (so accessible via http) so joiners nodes can reach the NCG using the same url + server_side_encryption = "AES256" + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}-ignition-master", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${module.tectonic.cluster_id}" + ), var.tectonic_aws_extra_tags)}" +} + +module "ignition_bootstrap" { + source = "../../modules/ignition" + + assets_location = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}" + base_domain = "${var.tectonic_base_domain}" + bootstrap_upgrade_cl = "${var.tectonic_bootstrap_upgrade_cl}" + cloud_provider = "aws" + cluster_name = "${var.tectonic_cluster_name}" + container_images = "${var.tectonic_container_images}" + custom_ca_cert_pem_list = "${var.tectonic_custom_ca_pem_list}" + etcd_advertise_name_list = "${data.template_file.etcd_hostname_list.*.rendered}" + etcd_ca_cert_pem = "${module.etcd_certs.etcd_ca_crt_pem}" + etcd_client_crt_pem = "${module.etcd_certs.etcd_client_crt_pem}" + etcd_client_key_pem = "${module.etcd_certs.etcd_client_key_pem}" + etcd_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + etcd_initial_cluster_list = "${data.template_file.etcd_hostname_list.*.rendered}" + etcd_peer_crt_pem = "${module.etcd_certs.etcd_peer_crt_pem}" + etcd_peer_key_pem = "${module.etcd_certs.etcd_peer_key_pem}" + etcd_server_crt_pem = "${module.etcd_certs.etcd_server_crt_pem}" + etcd_server_key_pem = "${module.etcd_certs.etcd_server_key_pem}" + http_proxy = "${var.tectonic_http_proxy_address}" + https_proxy = "${var.tectonic_https_proxy_address}" + image_re = "${var.tectonic_image_re}" + ingress_ca_cert_pem = "${module.ingress_certs.ca_cert_pem}" + iscsi_enabled = "${var.tectonic_iscsi_enabled}" + kube_ca_cert_pem = "${module.kube_certs.ca_cert_pem}" + kube_dns_service_ip = "${module.bootkube.kube_dns_service_ip}" + kubelet_debug_config = "${var.tectonic_kubelet_debug_config}" + kubelet_node_label = "node-role.kubernetes.io/master" + kubelet_node_taints = "node-role.kubernetes.io/master=:NoSchedule" + no_proxy = "${var.tectonic_no_proxy}" +} + +data "ignition_config" "bootstrap" { + files = ["${compact(list( + data.ignition_file.init_assets.id, + data.ignition_file.rm_assets.id, + module.ignition_bootstrap.installer_kubelet_env_id, + module.ignition_bootstrap.installer_runtime_mappings_id, + module.ignition_bootstrap.max_user_watches_id, + module.ignition_bootstrap.s3_puller_id, + module.ignition_bootstrap.profile_env_id, + module.ignition_bootstrap.systemd_default_env_id, + ))}", + "${module.ignition_bootstrap.ca_cert_id_list}", + ] + + systemd = ["${compact(list( + module.ignition_bootstrap.docker_dropin_id, + module.ignition_bootstrap.locksmithd_service_id, + module.ignition_bootstrap.kubelet_service_id, + module.ignition_bootstrap.k8s_node_bootstrap_service_id, + module.ignition_bootstrap.init_assets_service_id, + module.ignition_bootstrap.rm_assets_service_id, + module.ignition_bootstrap.rm_assets_path_unit_id, + module.bootkube.systemd_service_id, + module.bootkube.systemd_path_unit_id, + module.tectonic.systemd_service_id, + module.tectonic.systemd_path_unit_id, + module.ignition_bootstrap.update_ca_certificates_dropin_id, + module.ignition_bootstrap.iscsi_service_id, + ))}"] +} + +data "template_file" "init_assets" { + template = "${file("${path.module}/resources/init-assets.sh")}" + + vars { + cluster_name = "${var.tectonic_cluster_name}" + awscli_image = "${var.tectonic_container_images["awscli"]}" + assets_s3_location = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}" + } +} + +data "ignition_file" "init_assets" { + filesystem = "root" + path = "/opt/init-assets.sh" + mode = 0755 + + content { + content = "${data.template_file.init_assets.rendered}" + } +} + +data "template_file" "rm_assets" { + template = "${file("${path.module}/resources/rm-assets.sh")}" + + vars { + cluster_name = "${var.tectonic_cluster_name}" + awscli_image = "${var.tectonic_container_images["awscli"]}" + assets_s3_location = "${aws_s3_bucket_object.tectonic_assets.bucket}/${aws_s3_bucket_object.tectonic_assets.key}" + } +} + +data "ignition_file" "rm_assets" { + filesystem = "root" + path = "/opt/rm-assets.sh" + mode = 0755 + + content { + content = "${data.template_file.rm_assets.rendered}" + } +} diff --git a/steps/assets/ignition-etcd.tf b/steps/assets/ignition-etcd.tf new file mode 100644 index 0000000000..68288a7e20 --- /dev/null +++ b/steps/assets/ignition-etcd.tf @@ -0,0 +1,58 @@ +locals { + etcd_internal_instance_count = "${length(data.template_file.etcd_hostname_list.*.id)}" + etcd_instance_count = "${length(compact(var.tectonic_etcd_servers)) == 0 ? local.etcd_internal_instance_count : 0}" +} + +resource "aws_s3_bucket_object" "ignition_etcd" { + count = "${local.etcd_instance_count}" + bucket = "${aws_s3_bucket.tectonic.bucket}" + key = "ignition_etcd_${count.index}.json" + content = "${data.ignition_config.etcd.*.rendered[count.index]}" + acl = "private" + + server_side_encryption = "AES256" + + tags = "${merge(map( + "Name", "${var.tectonic_cluster_name}-ignition-etcd-${count.index}", + "KubernetesCluster", "${var.tectonic_cluster_name}", + "tectonicClusterID", "${module.tectonic.cluster_id}" + ), var.tectonic_aws_extra_tags)}" +} + +data "ignition_config" "etcd" { + count = "${local.etcd_instance_count}" + + systemd = [ + "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}", + "${module.ignition_bootstrap.etcd_dropin_id_list[count.index]}", + ] + + files = ["${compact(list( + module.ignition_bootstrap.profile_env_id, + module.ignition_bootstrap.systemd_default_env_id, + ))}", + "${module.ignition_bootstrap.etcd_crt_id_list}", + ] +} + +data "ignition_systemd_unit" "locksmithd" { + count = "${local.etcd_instance_count}" + + name = "locksmithd.service" + enabled = true + + dropin = [ + { + name = "40-etcd-lock.conf" + + content = <