openstack: Remove the Service VM

The experimental OpenStack backend used to create an extra server running DNS and load balancer services that the cluster needed. OpenStack does not always come with DNSaaS or LBaaS so we had to provide the functionality the OpenShift cluster depends on (e.g. the etcd SRV records, the api-int records & load balancing, etc.). This approach is undesirable for two reasons: first, it adds an extra node that the other IPI platforms do not need. Second, this node is a single point of failure. The Baremetal platform has faced the same issues and they have solved them with a few virtual IP addresses managed by keepalived in combination with coredns static pod running on every node using the mDNS protocol to update records as new nodes are added or removed and a similar static pod haproxy to load balance the control plane internally. The VIPs are defined here in the installer and they use the PlatformStatus field to be passed to the necessary machine-config-operator fields: openshift/api#374 The Bare Metal IPI Networking Infrastructure document is applicable here as well: https://github.com/openshift/installer/blob/master/docs/design/baremetal/networking-infrastructure.md There is also a great opportunity to share some of the configuration files and scripts here. This change needs several other pull requests: Keepalived plus the coredns & haproxy static pods in the MCO: openshift/machine-config-operator#740 Passing the API and DNS VIPs through the installer: #1998 Co-authored-by: Emilio Garcia <egarcia@redhat.com> Co-authored-by: John Trowbridge <trown@redhat.com> Co-authored-by: Martin Andre <m.andre@redhat.com> Co-authored-by: Tomas Sedovic <tsedovic@redhat.com> Massive thanks to the Bare Metal and oVirt people!
openshift · Jul 26, 2019 · 81a74dc · 81a74dc
1 parent 9680e78
commit 81a74dc
Show file tree

Hide file tree

Showing 23 changed files with 190 additions and 648 deletions.
diff --git a/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template b/data/data/bootstrap/files/usr/local/bin/bootkube.sh.template
@@ -33,6 +33,12 @@ OPENSHIFT_HYPERSHIFT_IMAGE=$(podman run --quiet --rm ${release} image hypershift
 OPENSHIFT_HYPERKUBE_IMAGE=$(podman run --quiet --rm ${release} image hyperkube)
 
 CLUSTER_BOOTSTRAP_IMAGE=$(podman run --quiet --rm ${release} image cluster-bootstrap)
+KEEPALIVED_IMAGE=$(podman run --quiet --rm ${release} image keepalived-ipfailover)
+COREDNS_IMAGE=$(podman run --quiet --rm ${release} image coredns)
+MDNS_PUBLISHER_IMAGE=$(podman run --quiet --rm ${release} image mdns-publisher)
+HAPROXY_IMAGE=$(podman run --quiet --rm ${release} image haproxy-router)
+RUNTIMECFG_IMAGE=$(podman run --quiet --rm ${release} image baremetal-runtimecfg)
+
 
 # Now, as early as possible we replace the pause image and reload crio to use it, to ensure
 # that we're using the pause image from our payload just like the primary cluster.
@@ -194,17 +200,30 @@ then
 			--machine-config-operator-image=${MACHINE_CONFIG_OPERATOR_IMAGE} \
 			--machine-config-oscontent-image=${MACHINE_CONFIG_OSCONTENT} \
 			--infra-image=${MACHINE_CONFIG_INFRA_IMAGE} \
+			--keepalived-image=${KEEPALIVED_IMAGE} \
+			--coredns-image=${COREDNS_IMAGE} \
+			--mdns-publisher-image=${MDNS_PUBLISHER_IMAGE} \
+			--haproxy-image=${HAPROXY_IMAGE} \
+			--baremetal-runtimecfg-image=${RUNTIMECFG_IMAGE} \
 			--cloud-config-file=/assets/manifests/cloud-provider-config.yaml
 
 	# Bootstrap MachineConfigController uses /etc/mcc/bootstrap/manifests/ dir to
 	# 1. read the controller config rendered by MachineConfigOperator
 	# 2. read the default MachineConfigPools rendered by MachineConfigOperator
 	# 3. read any additional MachineConfigs that are needed for the default MachineConfigPools.
-	mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests
+	mkdir --parents /etc/mcc/bootstrap /etc/mcs/bootstrap /etc/kubernetes/manifests /etc/kubernetes/static-pod-resources
 	cp mco-bootstrap/bootstrap/manifests/* /etc/mcc/bootstrap/
 	cp openshift/* /etc/mcc/bootstrap/
 	cp auth/kubeconfig-kubelet /etc/mcs/kubeconfig
 	cp mco-bootstrap/bootstrap/machineconfigoperator-bootstrap-pod.yaml /etc/kubernetes/manifests/
+	if [ -d mco-bootstrap/baremetal/manifests ]; then
+	    cp mco-bootstrap/baremetal/manifests/* /etc/kubernetes/manifests/
+	    cp -r mco-bootstrap/baremetal/static-pod-resources/* /etc/kubernetes/static-pod-resources/
+	fi
+	if [ -d mco-bootstrap/openstack/manifests ]; then
+	    cp mco-bootstrap/openstack/manifests/* /etc/kubernetes/manifests/
+	    cp -r mco-bootstrap/openstack/static-pod-resources/* /etc/kubernetes/static-pod-resources/
+	fi
 	cp mco-bootstrap/manifests/* manifests/
 
 	# /etc/ssl/mcs/tls.{crt, key} are locations for MachineConfigServer's tls assets.

diff --git a/data/data/openstack/bootstrap/main.tf b/data/data/openstack/bootstrap/main.tf
@@ -18,40 +18,46 @@ data "ignition_config" "redirect" {
 
   files = [
     data.ignition_file.hostname.id,
-    data.ignition_file.bootstrap_ifcfg.id,
+    data.ignition_file.dns_conf.id,
+    data.ignition_file.dhcp_conf.id,
   ]
 }
 
-data "ignition_file" "bootstrap_ifcfg" {
+data "ignition_file" "dhcp_conf" {
   filesystem = "root"
-  mode       = "420" // 0644
-  path       = "/etc/sysconfig/network-scripts/ifcfg-eth0"
+  mode       = "420"
+  path       = "/etc/NetworkManager/conf.d/dhcp-client.conf"
 
   content {
     content = <<EOF
-DEVICE="eth0"
-BOOTPROTO="dhcp"
-ONBOOT="yes"
-TYPE="Ethernet"
-PERSISTENT_DHCLIENT="yes"
-DNS1="${var.service_vm_fixed_ip}"
-PEERDNS="no"
-NM_CONTROLLED="yes"
+[main]
+dhcp=dhclient
 EOF
+  }
+}
 
+data "ignition_file" "dns_conf" {
+  filesystem = "root"
+  mode = "420"
+  path = "/etc/dhcp/dhclient.conf"
+
+  content {
+    content = <<EOF
+send dhcp-client-identifier = hardware;
+prepend domain-name-servers 127.0.0.1;
+EOF
   }
 }
 
 data "ignition_file" "hostname" {
   filesystem = "root"
-  mode = "420" // 0644
-  path = "/etc/hostname"
+  mode       = "420" // 0644
+  path       = "/etc/hostname"
 
   content {
     content = <<EOF
 ${var.cluster_id}-bootstrap
 EOF
-
   }
 }
 
@@ -81,4 +87,3 @@ resource "openstack_compute_instance_v2" "bootstrap" {
     openshiftClusterID = var.cluster_id
   }
 }
-
diff --git a/data/data/openstack/bootstrap/variables.tf b/data/data/openstack/bootstrap/variables.tf
@@ -13,11 +13,6 @@ variable "cluster_id" {
   description = "The identifier for the cluster."
 }
 
-variable "cluster_domain" {
-  type        = string
-  description = "The domain name of the cluster. All DNS records must be under this domain."
-}
-
 variable "ignition" {
   type        = string
   description = "The content of the bootstrap ignition file."
@@ -32,8 +27,3 @@ variable "bootstrap_port_id" {
   type        = string
   description = "The subnet ID for the bootstrap node."
 }
-
-variable "service_vm_fixed_ip" {
-  type = string
-}
-
diff --git a/data/data/openstack/main.tf b/data/data/openstack/main.tf
@@ -22,72 +22,45 @@ provider "openstack" {
   user_name           = var.openstack_credentials_user_name
 }
 
-module "service" {
-  source = "./service"
+module "bootstrap" {
+  source = "./bootstrap"
 
   swift_container   = openstack_objectstorage_container_v1.container.name
   cluster_id        = var.cluster_id
-  cluster_domain    = var.cluster_domain
   image_name        = var.openstack_base_image
   flavor_name       = var.openstack_master_flavor_name
   ignition          = var.ignition_bootstrap
-  lb_floating_ip    = var.openstack_lb_floating_ip
-  service_port_id   = module.topology.service_port_id
-  service_port_ip   = module.topology.service_port_ip
-  master_ips        = module.topology.master_ips
-  master_port_names = module.topology.master_port_names
-  bootstrap_ip      = module.topology.bootstrap_port_ip
-}
-
-module "bootstrap" {
-  source = "./bootstrap"
-
-  swift_container     = openstack_objectstorage_container_v1.container.name
-  cluster_id          = var.cluster_id
-  cluster_domain      = var.cluster_domain
-  image_name          = var.openstack_base_image
-  flavor_name         = var.openstack_master_flavor_name
-  ignition            = var.ignition_bootstrap
-  bootstrap_port_id   = module.topology.bootstrap_port_id
-  service_vm_fixed_ip = module.topology.service_vm_fixed_ip
+  bootstrap_port_id = module.topology.bootstrap_port_id
 }
 
 module "masters" {
   source = "./masters"
 
-  base_image          = var.openstack_base_image
-  bootstrap_ip        = module.topology.bootstrap_port_ip
-  cluster_id          = var.cluster_id
-  cluster_domain      = var.cluster_domain
-  flavor_name         = var.openstack_master_flavor_name
-  instance_count      = var.master_count
-  lb_floating_ip      = var.openstack_lb_floating_ip
-  master_ips          = module.topology.master_ips
-  master_port_ids     = module.topology.master_port_ids
-  master_port_names   = module.topology.master_port_names
-  user_data_ign       = var.ignition_master
-  service_vm_fixed_ip = module.topology.service_vm_fixed_ip
-  api_int_ip          = var.openstack_api_int_ip
-  node_dns_ip         = var.openstack_node_dns_ip
+  base_image      = var.openstack_base_image
+  cluster_id      = var.cluster_id
+  flavor_name     = var.openstack_master_flavor_name
+  instance_count  = var.master_count
+  master_port_ids = module.topology.master_port_ids
+  user_data_ign   = var.ignition_master
   master_sg_ids = concat(
     var.openstack_master_extra_sg_ids,
     [module.topology.master_sg_id],
   )
 }
 
-# TODO(shadower) add a dns module here
-
 module "topology" {
   source = "./topology"
 
   cidr_block          = var.machine_cidr
   cluster_id          = var.cluster_id
+  cluster_domain      = var.cluster_domain
   external_network    = var.openstack_external_network
   external_network_id = var.openstack_external_network_id
   masters_count       = var.master_count
   lb_floating_ip      = var.openstack_lb_floating_ip
   api_int_ip          = var.openstack_api_int_ip
   node_dns_ip         = var.openstack_node_dns_ip
+  ingress_ip          = var.openstack_ingress_ip
   trunk_support       = var.openstack_trunk_support
   octavia_support     = var.openstack_octavia_support
 }
@@ -98,9 +71,11 @@ resource "openstack_objectstorage_container_v1" "container" {
   # "kubernetes.io/cluster/${var.cluster_id}" = "owned"
   metadata = merge(
     {
-      "Name"               = "${var.cluster_id}-ignition-master"
+      "Name"               = "${var.cluster_id}-ignition"
       "openshiftClusterID" = var.cluster_id
     },
+    # FIXME(mandre) the openstack_extra_tags should be applied to all resources
+    # created
     var.openstack_extra_tags,
   )
 }

diff --git a/data/data/openstack/masters/main.tf b/data/data/openstack/masters/main.tf
@@ -16,23 +16,6 @@ data "ignition_file" "hostname" {
   content {
     content = <<EOF
 ${var.cluster_id}-master-${count.index}
-EOF
-
-  }
-}
-
-data "ignition_file" "clustervars" {
-  filesystem = "root"
-  mode = "420" // 0644
-  path = "/etc/kubernetes/static-pod-resources/clustervars"
-
-  content {
-    content = <<EOF
-export API_VIP=${var.api_int_ip}
-export DNS_VIP=${var.node_dns_ip}
-export FLOATING_IP=${var.lb_floating_ip}
-export BOOTSTRAP_IP=${var.bootstrap_ip}
-${replace(join("\n", formatlist("export MASTER_FIXED_IPS_%s=%s", var.master_port_names, var.master_ips)), "${var.cluster_id}-master-port-", "")}
 EOF
   }
 }
@@ -45,8 +28,7 @@ data "ignition_config" "master_ignition_config" {
   }
 
   files = [
-    element(data.ignition_file.hostname.*.id, count.index),
-    data.ignition_file.clustervars.id,
+    element(data.ignition_file.hostname.*.id, count.index)
   ]
 }
 
@@ -67,6 +49,7 @@ resource "openstack_compute_instance_v2" "master_conf" {
   }
 
   metadata = {
+    # FIXME(mandre) shouldn't it be "${var.cluster_id}-master-${count.index}" ?
     Name = "${var.cluster_id}-master"
     # "kubernetes.io/cluster/${var.cluster_id}" = "owned"
     openshiftClusterID = var.cluster_id

diff --git a/data/data/openstack/masters/variables.tf b/data/data/openstack/masters/variables.tf
@@ -2,20 +2,11 @@ variable "base_image" {
   type = string
 }
 
-variable "bootstrap_ip" {
-  type = string
-}
-
 variable "cluster_id" {
   type        = string
   description = "The identifier for the cluster."
 }
 
-variable "cluster_domain" {
-  type        = string
-  description = "The domain name of the cluster. All DNS records must be under this domain."
-}
-
 variable "flavor_name" {
   type = string
 }
@@ -24,14 +15,6 @@ variable "instance_count" {
   type = string
 }
 
-variable "lb_floating_ip" {
-  type = string
-}
-
-variable "master_ips" {
-  type = list(string)
-}
-
 variable "master_sg_ids" {
   type        = list(string)
   default     = ["default"]
@@ -43,22 +26,6 @@ variable "master_port_ids" {
   description = "List of port ids for the master nodes"
 }
 
-variable "master_port_names" {
-  type = list(string)
-}
-
 variable "user_data_ign" {
   type = string
 }
-
-variable "api_int_ip" {
-  type = string
-}
-
-variable "node_dns_ip" {
-  type = string
-}
-
-variable "service_vm_fixed_ip" {
-  type = string
-}