UpCloud server plan, firewall, load balancer integration (kubernetes-…

…sigs#8758)

* [upcloud] add option to use preconfigured cpu/mem plan

* [upcloud] add option to use firewall rules for API server/SSH access

* [upcloud] add option to use managed load balancer

contrib/terraform/upcloud/README.md

@@ -104,9 +104,22 @@ terraform destroy --var-file cluster-settings.tfvars \
 * `zone`: The zone where to run the cluster
 * `machines`: Machines to provision. Key of this object will be used as the name of the machine
   * `node_type`: The role of this node *(master|worker)*
+  * `plan`: Preconfigured cpu/mem plan to use (disables `cpu` and `mem` attributes below)
   * `cpu`: number of cpu cores
   * `mem`: memory size in MB
   * `disk_size`: The size of the storage in GB
   * `additional_disks`: Additional disks to attach to the node.
     * `size`: The size of the additional disk in GB
     * `tier`: The tier of disk to use (`maxiops` is the only one you can choose atm)
+* `firewall_enabled`: Enable firewall rules
+* `master_allowed_remote_ips`: List of IP ranges that should be allowed to access API of masters
+  * `start_address`: Start of address range to allow
+  * `end_address`: End of address range to allow
+* `k8s_allowed_remote_ips`: List of IP ranges that should be allowed SSH access to all nodes
+  * `start_address`: Start of address range to allow
+  * `end_address`: End of address range to allow
+* `loadbalancer_enabled`: Enable managed load balancer
+* `loadbalancer_plan`: Plan to use for load balancer *(development|production-small)*
+* `loadbalancers`: Ports to load balance and which machines to forward to. Key of this object will be used as the name of the load balancer frontends/backends
+  * `port`: Port to load balance.
+  * `backend_servers`: List of servers that traffic to the port should be forwarded to.

contrib/terraform/upcloud/cluster-settings.tfvars

@@ -20,6 +20,8 @@ ssh_public_keys = [
 machines = {
   "master-0" : {
     "node_type" : "master",
+    # plan to use instead of custom cpu/mem
+    "plan" : null,
     #number of cpu cores
     "cpu" : "2",
     #memory size in MB
@@ -30,6 +32,8 @@ machines = {
   },
   "worker-0" : {
     "node_type" : "worker",
+    # plan to use instead of custom cpu/mem
+    "plan" : null,
     #number of cpu cores
     "cpu" : "2",
     #memory size in MB
@@ -49,6 +53,8 @@ machines = {
   },
   "worker-1" : {
     "node_type" : "worker",
+    # plan to use instead of custom cpu/mem
+    "plan" : null,
     #number of cpu cores
     "cpu" : "2",
     #memory size in MB
@@ -68,6 +74,8 @@ machines = {
   },
   "worker-2" : {
     "node_type" : "worker",
+    # plan to use instead of custom cpu/mem
+    "plan" : null,
     #number of cpu cores
     "cpu" : "2",
     #memory size in MB
@@ -86,3 +94,32 @@ machines = {
     }
   }
 }
+
+firewall_enabled = false
+
+master_allowed_remote_ips = [
+  {
+    "start_address" : "0.0.0.0"
+    "end_address" : "255.255.255.255"
+  }
+]
+
+k8s_allowed_remote_ips = [
+  {
+    "start_address" : "0.0.0.0"
+    "end_address" : "255.255.255.255"
+  }
+]
+
+loadbalancer_enabled = false
+loadbalancer_plan    = "development"
+loadbalancers = {
+  # "http" : {
+  #   "port" : 80,
+  #   "backend_servers" : [
+  #     "worker-0",
+  #     "worker-1",
+  #     "worker-2"
+  #   ]
+  # }
+}

contrib/terraform/upcloud/main.tf

@@ -22,6 +22,14 @@ module "kubernetes" {
   machines = var.machines
 
   ssh_public_keys = var.ssh_public_keys
+
+  firewall_enabled          = var.firewall_enabled
+  master_allowed_remote_ips = var.master_allowed_remote_ips
+  k8s_allowed_remote_ips    = var.k8s_allowed_remote_ips
+
+  loadbalancer_enabled = var.loadbalancer_enabled
+  loadbalancer_plan    = var.loadbalancer_plan
+  loadbalancers        = var.loadbalancers
 }
 
 #

contrib/terraform/upcloud/modules/kubernetes-cluster/main.tf

@@ -10,6 +10,16 @@ locals {
     ]
   ])
 
+  lb_backend_servers = flatten([
+    for lb_name, loadbalancer in var.loadbalancers : [
+      for backend_server in loadbalancer.backend_servers : {
+        port = loadbalancer.port
+        lb_name = lb_name
+        server_name = backend_server
+      }
+    ]
+  ])
+
   # If prefix is set, all resources will be prefixed with "${var.prefix}-"
   # Else don't prefix with anything
   resource-prefix = "%{ if var.prefix != ""}${var.prefix}-%{ endif }"
@@ -45,8 +55,9 @@ resource "upcloud_server" "master" {
   }
 
   hostname = "${local.resource-prefix}${each.key}"
-  cpu      = each.value.cpu
-  mem      = each.value.mem
+  plan     = each.value.plan
+  cpu      = each.value.plan == null ? each.value.cpu : null
+  mem      = each.value.plan == null ? each.value.mem : null
   zone     = var.zone
 
   template {
@@ -69,6 +80,8 @@ resource "upcloud_server" "master" {
   lifecycle {
     ignore_changes = [storage_devices]
   }
+
+  firewall  = var.firewall_enabled
 
   dynamic "storage_devices" {
     for_each = {
@@ -99,8 +112,9 @@ resource "upcloud_server" "worker" {
   }
 
   hostname = "${local.resource-prefix}${each.key}"
-  cpu      = each.value.cpu
-  mem      = each.value.mem
+  plan     = each.value.plan
+  cpu      = each.value.plan == null ? each.value.cpu : null
+  mem      = each.value.plan == null ? each.value.mem : null
   zone     = var.zone
 
   template {
@@ -124,6 +138,8 @@ resource "upcloud_server" "worker" {
     ignore_changes = [storage_devices]
   }
 
+  firewall  = var.firewall_enabled
+
   dynamic "storage_devices" {
     for_each = {
       for disk_key_name, disk in upcloud_storage.additional_disks :
@@ -144,3 +160,151 @@ resource "upcloud_server" "worker" {
     create_password = false
   }
 }
+
+resource "upcloud_firewall_rules" "master" {
+  for_each = upcloud_server.master
+  server_id = each.value.id
+
+  dynamic firewall_rule {
+    for_each = var.master_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow master API access from this network"
+      destination_port_end   = "6443"
+      destination_port_start = "6443"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = length(var.master_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Deny master API access from other networks"
+      destination_port_end   = "6443"
+      destination_port_start = "6443"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = var.k8s_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow SSH from this network"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Deny SSH from other networks"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+}
+
+resource "upcloud_firewall_rules" "k8s" {
+  for_each = upcloud_server.worker
+  server_id = each.value.id
+
+  dynamic firewall_rule {
+    for_each = var.k8s_allowed_remote_ips
+
+    content {
+      action                 = "accept"
+      comment                = "Allow SSH from this network"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = firewall_rule.value.end_address
+      source_address_start   = firewall_rule.value.start_address
+    }
+  }
+
+  dynamic firewall_rule {
+    for_each = length(var.k8s_allowed_remote_ips) > 0 ? [1] : []
+
+    content {
+      action                 = "drop"
+      comment                = "Deny SSH from other networks"
+      destination_port_end   = "22"
+      destination_port_start = "22"
+      direction              = "in"
+      family                 = "IPv4"
+      protocol               = "tcp"
+      source_address_end     = "255.255.255.255"
+      source_address_start   = "0.0.0.0"
+    }
+  }
+}
+
+resource "upcloud_loadbalancer" "lb" {
+  count             = var.loadbalancer_enabled ? 1 : 0
+  configured_status = "started"
+  name              = "${local.resource-prefix}lb"
+  plan              = var.loadbalancer_plan
+  zone              = var.zone
+  network           = upcloud_network.private.id
+}
+
+resource "upcloud_loadbalancer_backend" "lb_backend" {
+  for_each = var.loadbalancer_enabled ? var.loadbalancers : {}
+
+  loadbalancer = upcloud_loadbalancer.lb[0].id
+  name         = "lb-backend-${each.key}"
+}
+
+resource "upcloud_loadbalancer_frontend" "lb_frontend" {
+  for_each = var.loadbalancer_enabled ? var.loadbalancers : {}
+
+  loadbalancer         = upcloud_loadbalancer.lb[0].id
+  name                 = "lb-frontend-${each.key}"
+  mode                 = "tcp"
+  port                 = each.value.port
+  default_backend_name = upcloud_loadbalancer_backend.lb_backend[each.key].name
+}
+
+resource "upcloud_loadbalancer_static_backend_member" "lb_backend_member" {
+  for_each = {
+    for be_server in local.lb_backend_servers: 
+      "${be_server.server_name}-lb-backend-${be_server.lb_name}" => be_server
+      if var.loadbalancer_enabled
+  }
+
+  backend      = upcloud_loadbalancer_backend.lb_backend[each.value.lb_name].id
+  name         = "${local.resource-prefix}${each.key}"
+  ip           = merge(upcloud_server.master, upcloud_server.worker)[each.value.server_name].network_interface[1].ip_address
+  port         = each.value.port
+  weight       = 100
+  max_sessions = var.loadbalancer_plan == "production-small" ? 50000 : 1000
+  enabled      = true
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/output.tf

@@ -18,3 +18,7 @@ output "worker_ip" {
     }
   }
 }
+
+output "loadbalancer_domain" {
+  value = var.loadbalancer_enabled ? upcloud_loadbalancer.lb[0].dns_name : null
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/variables.tf

@@ -16,6 +16,7 @@ variable "machines" {
   description = "Cluster machines"
   type = map(object({
     node_type       = string
+    plan            = string
     cpu             = string
     mem             = string
     disk_size       =  number
@@ -29,3 +30,38 @@ variable "machines" {
 variable "ssh_public_keys" {
   type = list(string)
 }
+
+variable "firewall_enabled" {
+  type = bool
+}
+
+variable "master_allowed_remote_ips" {
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+}
+
+variable "k8s_allowed_remote_ips" {
+  type = list(object({
+    start_address = string
+    end_address   = string
+  }))
+}
+
+variable "loadbalancer_enabled" {
+  type = bool
+}
+
+variable "loadbalancer_plan" {
+  type = string
+}
+
+variable "loadbalancers" {
+  description = "Load balancers"
+
+  type = map(object({
+    port            = number
+    backend_servers = list(string)
+  }))
+}

contrib/terraform/upcloud/modules/kubernetes-cluster/versions.tf

@@ -3,7 +3,7 @@ terraform {
   required_providers {
     upcloud = {
       source = "UpCloudLtd/upcloud"
-      version = "~>2.0.0"
+      version = "~>2.4.0"
     }
   }
   required_version = ">= 0.13"

contrib/terraform/upcloud/output.tf

@@ -6,3 +6,7 @@ output "master_ip" {
 output "worker_ip" {
   value = module.kubernetes.worker_ip
 }
+
+output "loadbalancer_domain" {
+  value = module.kubernetes.loadbalancer_domain
+}