modules/*: trust CA certificates on the nodes (openshift#2362)

* modules/*: trust CA certificates on the nodes This also bumps the ignition provider to 1.0.0 to support S3 downloads via ignition. Fixes #INST-67 * modules/ignition: unify etcd certificate generation Currently etcd certificate ignition files are created in their respective platform module. This unifies it by declaring the ignition file units centrally in the ignition module. * modules/tls/etcd: remove zip generation Currently the etcd TLS module also generates a zip file which is only used on AWS to reduce the userdata size to be <20k (hard limit on AWS). Since the etcd TLS assets will be provisioned via S3 this optimization/hack is not needed any more. * */aws: use S3 for ignition provisioning We hit the limits of the AWS userdata limit (20k) constantly. This fixes it by provisioning a minimal ignition configuration only which points to a replacement ignition configuration hosted on S3. This also removes workarounds/hacks to keep the userdata size small, especially for provisioning TLS assets on etcd nodes. * Documentation/examples: regenerate * */azure: use unified etcd TLS ignition files * */gcp: use unified etcd TLS ignition files * */openstack: use unified etcd TLS ignition files * */vmware: use unified etcd TLS ignition files
yifan-gu · Nov 22, 2017 · 0c7ac90 · 0c7ac90
1 parent f82cae0
commit 0c7ac90
Show file tree

Hide file tree

Showing 55 changed files with 915 additions and 981 deletions.
diff --git a/Documentation/variables/config.md b/Documentation/variables/config.md
@@ -20,6 +20,7 @@ This document gives an overview of variables used in all platforms of the Tecton
 | tectonic_container_images | (internal) Container images to use | map | `<map>` |
 | tectonic_container_linux_channel | (optional) The Container Linux update channel.<br><br>Examples: `stable`, `beta`, `alpha` | string | `stable` |
 | tectonic_container_linux_version | The Container Linux version to use. Set to `latest` to select the latest available version for the selected update channel.<br><br>Examples: `latest`, `1465.6.0` | string | `latest` |
+| tectonic_custom_ca_pem_list | (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes. | list | `<list>` |
 | tectonic_ddns_key_algorithm | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key algorithm. | string | `` |
 | tectonic_ddns_key_name | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key name. | string | `` |
 | tectonic_ddns_key_secret | (optional) This only applies if you use the modules/dns/ddns module.<br><br>Specifies the RFC2136 Dynamic DNS server key secret. | string | `` |

diff --git a/config.tf b/config.tf
@@ -11,7 +11,7 @@ provider "external" {
 }
 
 provider "ignition" {
- version = "0.1.0"
+ version = "1.0.0"
 }
 
 provider "local" {
@@ -34,6 +34,14 @@ provider "tls" {
  version = "1.0.0"
 }
 
+locals {
+ // The total amount of public CA certificates present in Tectonic.
+ // That is all custom CAs + kube CA + etcd CA + ingress CA
+ // This is a local constant, which needs to be dependency inject because TF cannot handle length() on computed values,
+ // see https://github.com/hashicorp/terraform/issues/10857#issuecomment-268289775.
+ tectonic_ca_count = "${length(var.tectonic_custom_ca_pem_list) + 3}"
+}
+
 variable "tectonic_config_version" {
  description = <<EOF
 (internal) This declares the version of the global configuration variables.
@@ -512,3 +520,12 @@ variable "tectonic_kubelet_debug_config" {
 
  description = "(internal) debug flags for the kubelet (used in CI only)"
 }
+
+variable "tectonic_custom_ca_pem_list" {
+ type = "list"
+ default = []
+
+ description = <<EOF
+(optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+EOF
+}
diff --git a/examples/terraform.tfvars.aws b/examples/terraform.tfvars.aws
@@ -194,6 +194,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/examples/terraform.tfvars.azure b/examples/terraform.tfvars.azure
@@ -155,6 +155,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/examples/terraform.tfvars.gcp b/examples/terraform.tfvars.gcp
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/examples/terraform.tfvars.metal b/examples/terraform.tfvars.metal
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/examples/terraform.tfvars.openstack-neutron b/examples/terraform.tfvars.openstack-neutron
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/examples/terraform.tfvars.vmware b/examples/terraform.tfvars.vmware
@@ -46,6 +46,9 @@ tectonic_cluster_name = ""
 // Examples: `latest`, `1465.6.0`
 tectonic_container_linux_version = "latest"
 
+// (optional) A list of PEM encoded CA files that will be installed in /etc/ssl/certs on etcd, master, and worker nodes.
+// tectonic_custom_ca_pem_list = ""
+
 // (optional) This only applies if you use the modules/dns/ddns module.
 // 
 // Specifies the RFC2136 Dynamic DNS server key algorithm.

diff --git a/modules/aws/etcd/ignition.tf b/modules/aws/etcd/ignition.tf
@@ -4,64 +4,18 @@ data "ignition_config" "etcd" {
  systemd = [
  "${data.ignition_systemd_unit.locksmithd.*.id[count.index]}",
  "${var.ign_etcd_dropin_id_list[count.index]}",
- "${data.ignition_systemd_unit.etcd_unzip_tls.id}",
  ]
 
  files = [
- "${data.ignition_file.node_hostname.*.id[count.index]}",
- "${data.ignition_file.etcd_tls_zip.id}",
+ "${var.ign_etcd_crt_id_list}",
  ]
 }
 
-data "ignition_file" "node_hostname" {
- count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
- path = "/etc/hostname"
- mode = 0644
- filesystem = "root"
-
- content {
- content = "${var.cluster_name}-etcd-${count.index}.${var.base_domain}"
- }
-}
-
-data "ignition_file" "etcd_tls_zip" {
- path = "/etc/ssl/etcd/tls.zip"
- mode = 0400
- uid = 0
- gid = 0
- filesystem = "root"
-
- content {
- mime = "application/octet-stream"
- content = "${var.tls_zip}"
- }
-}
-
-data "ignition_systemd_unit" "etcd_unzip_tls" {
- name = "etcd-unzip-tls.service"
- enable = true
-
- content = <<EOF
-[Unit]
-ConditionPathExists=!/etc/ssl/etcd/ca.crt
-[Service]
-Type=oneshot
-WorkingDirectory=/etc/ssl/etcd
-ExecStart=/usr/bin/bash -c 'unzip /etc/ssl/etcd/tls.zip && \
-chown etcd:etcd /etc/ssl/etcd/peer.* && \
-chown etcd:etcd /etc/ssl/etcd/server.* && \
-chmod 0400 /etc/ssl/etcd/peer.* /etc/ssl/etcd/server.* /etc/ssl/etcd/client.*'
-[Install]
-WantedBy=multi-user.target
-RequiredBy=etcd-member.service locksmithd.service
-EOF
-}
-
 data "ignition_systemd_unit" "locksmithd" {
  count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
 
- name = "locksmithd.service"
- enable = true
+ name  = "locksmithd.service"
+ enabled = true
 
  dropin = [
  {

diff --git a/modules/aws/etcd/ignition_s3.tf b/modules/aws/etcd/ignition_s3.tf
@@ -0,0 +1,25 @@
+resource "aws_s3_bucket_object" "ignition_etcd" {
+ count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
+
+ bucket = "${var.s3_bucket}"
+ key = "ignition_etcd_${count.index}.json"
+ content = "${data.ignition_config.etcd.*.rendered[count.index]}"
+ acl = "private"
+
+ server_side_encryption = "AES256"
+
+ tags = "${merge(map(
+ "Name", "${var.cluster_name}-ignition-etcd-${count.index}",
+ "KubernetesCluster", "${var.cluster_name}",
+ "tectonicClusterID", "${var.cluster_id}"
+ ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+ count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
+
+ replace {
+ source = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_etcd.*.key[count.index])}"
+ verification = "sha512-${sha512(data.ignition_config.etcd.*.rendered[count.index])}"
+ }
+}
diff --git a/modules/aws/etcd/nodes.tf b/modules/aws/etcd/nodes.tf
@@ -20,14 +20,88 @@ data "aws_ami" "coreos_ami" {
  }
 }
 
+resource "aws_iam_role" "etcd" {
+ count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+ name = "${var.cluster_name}-etcd-role"
+ path = "/"
+
+ assume_role_policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Action": "sts:AssumeRole",
+ "Principal": {
+ "Service": "ec2.amazonaws.com"
+ },
+ "Effect": "Allow",
+ "Sid": ""
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_role_policy" "etcd" {
+ count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+ name = "${var.cluster_name}_etcd_policy"
+ role = "${aws_iam_role.etcd.id}"
+
+ policy = <<EOF
+{
+ "Version": "2012-10-17",
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": "ec2:Describe*",
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": "ec2:AttachVolume",
+ "Resource": "*"
+ },
+ {
+ "Effect": "Allow",
+ "Action": "ec2:DetachVolume",
+ "Resource": "*"
+ },
+ {
+ "Action": [
+ "ecr:DescribeRepositories",
+ "ecr:ListImages",
+ "ecr:BatchGetImage"
+ ],
+ "Resource": "*",
+ "Effect": "Allow"
+ },
+ {
+ "Action" : [
+ "s3:GetObject"
+ ],
+ "Resource": "arn:aws:s3:::*",
+ "Effect": "Allow"
+ }
+ ]
+}
+EOF
+}
+
+resource "aws_iam_instance_profile" "etcd" {
+ count = "${length(var.external_endpoints) == 0 ? 1 : 0}"
+ name = "${var.cluster_name}-etcd-profile"
+ role = "${aws_iam_role.etcd.name}"
+}
+
 resource "aws_instance" "etcd_node" {
  count = "${length(var.external_endpoints) == 0 ? var.instance_count : 0}"
  ami = "${data.aws_ami.coreos_ami.image_id}"
 
+ iam_instance_profile = "${aws_iam_instance_profile.etcd.name}"
  instance_type = "${var.ec2_type}"
- subnet_id = "${element(var.subnets, count.index)}"
  key_name = "${var.ssh_key}"
- user_data = "${data.ignition_config.etcd.*.rendered[count.index]}"
+ subnet_id = "${element(var.subnets, count.index)}"
+ user_data = "${data.ignition_config.s3.*.rendered[count.index]}"
  vpc_security_group_ids = ["${var.sg_ids}"]
 
  lifecycle {

diff --git a/modules/aws/etcd/variables.tf b/modules/aws/etcd/variables.tf
@@ -73,10 +73,14 @@ variable "tls_enabled" {
  default = false
 }
 
-variable "tls_zip" {
+variable "ign_etcd_dropin_id_list" {
+ type = "list"
+}
+
+variable "s3_bucket" {
  type = "string"
 }
 
-variable "ign_etcd_dropin_id_list" {
+variable "ign_etcd_crt_id_list" {
  type = "list"
 }
diff --git a/modules/aws/master-asg/ignition.tf b/modules/aws/master-asg/ignition.tf
@@ -7,6 +7,7 @@ data "ignition_config" "main" {
  "${var.ign_installer_runtime_mappings_id}",
  "${var.ign_max_user_watches_id}",
  "${var.ign_s3_puller_id}",
+ "${var.ign_ca_cert_id_list}",
  ]
 
  systemd = ["${compact(list(
@@ -21,6 +22,7 @@ data "ignition_config" "main" {
  var.ign_bootkube_path_unit_id,
  var.ign_tectonic_path_unit_id,
  var.ign_rm_assets_path_unit_id,
+ var.ign_update_ca_certificates_dropin_id,
  ))}"]
 }
 

diff --git a/modules/aws/master-asg/ignition_s3.tf b/modules/aws/master-asg/ignition_s3.tf
@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_object" "ignition_master" {
+ bucket = "${var.s3_bucket}"
+ key = "ignition_master.json"
+ content = "${data.ignition_config.main.rendered}"
+ acl = "private"
+
+ server_side_encryption = "AES256"
+
+ tags = "${merge(map(
+ "Name", "${var.cluster_name}-ignition-master",
+ "KubernetesCluster", "${var.cluster_name}",
+ "tectonicClusterID", "${var.cluster_id}"
+ ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+ replace {
+ source = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_master.key)}"
+ verification = "sha512-${sha512(data.ignition_config.main.rendered)}"
+ }
+}
diff --git a/modules/aws/master-asg/master.tf b/modules/aws/master-asg/master.tf
@@ -62,7 +62,7 @@ resource "aws_launch_configuration" "master_conf" {
  security_groups = ["${var.master_sg_ids}"]
  iam_instance_profile = "${aws_iam_instance_profile.master_profile.arn}"
  associate_public_ip_address = "${var.public_endpoints}"
- user_data = "${data.ignition_config.main.rendered}"
+ user_data = "${data.ignition_config.s3.rendered}"
 
  lifecycle {
  create_before_destroy = true

diff --git a/modules/aws/master-asg/variables.tf b/modules/aws/master-asg/variables.tf
@@ -138,3 +138,7 @@ variable "ign_rm_assets_service_id" {
 variable "ign_rm_assets_path_unit_id" {
  type = "string"
 }
+
+variable "s3_bucket" {
+ type = "string"
+}
diff --git a/modules/aws/worker-asg/ignition.tf b/modules/aws/worker-asg/ignition.tf
@@ -4,12 +4,14 @@ data "ignition_config" "main" {
  "${var.ign_installer_runtime_mappings_id}",
  "${var.ign_max_user_watches_id}",
  "${var.ign_s3_puller_id}",
+ "${var.ign_ca_cert_id_list}",
  ]
 
  systemd = [
  "${var.ign_docker_dropin_id}",
  "${var.ign_k8s_node_bootstrap_service_id}",
  "${var.ign_kubelet_service_id}",
  "${var.ign_locksmithd_service_id}",
+ "${var.ign_update_ca_certificates_dropin_id}",
  ]
 }
diff --git a/modules/aws/worker-asg/ignition_s3.tf b/modules/aws/worker-asg/ignition_s3.tf
@@ -0,0 +1,21 @@
+resource "aws_s3_bucket_object" "ignition_worker" {
+ bucket = "${var.s3_bucket}"
+ key = "ignition_worker.json"
+ content = "${data.ignition_config.main.rendered}"
+ acl = "private"
+
+ server_side_encryption = "AES256"
+
+ tags = "${merge(map(
+ "Name", "${var.cluster_name}-ignition-worker",
+ "KubernetesCluster", "${var.cluster_name}",
+ "tectonicClusterID", "${var.cluster_id}"
+ ), var.extra_tags)}"
+}
+
+data "ignition_config" "s3" {
+ replace {
+ source = "${format("s3://%s/%s", var.s3_bucket, aws_s3_bucket_object.ignition_worker.key)}"
+ verification = "sha512-${sha512(data.ignition_config.main.rendered)}"
+ }
+}