From 19484e1770a4adae98d48acf0a6b592f2289d24f Mon Sep 17 00:00:00 2001 From: M4t7e <7535523+M4t7e@users.noreply.github.com> Date: Thu, 20 Jul 2023 15:59:14 +0200 Subject: [PATCH 1/4] Cleanup of firewall rules --- README.md | 27 ++++++++++++++++++++++++- data.tf | 11 ++++++++++ kube.tf.example | 12 +++++++++++ locals.tf | 53 +++++-------------------------------------------- variables.tf | 6 ++++++ versions.tf | 4 ++++ 6 files changed, 64 insertions(+), 49 deletions(-) diff --git a/README.md b/README.md index 6161ff25..17db698e 100644 --- a/README.md +++ b/README.md @@ -143,6 +143,32 @@ _Once you start with Terraform, it's best not to change the state of the project When your brand-new cluster is up and running, the sky is your limit! 🎉 +You can view all kinds of details about the cluster by running `terraform output kubeconfig` or `terraform output -json kubeconfig | jq`. + +### Connect via SSH + +Connect to one of the control plane nodes via SSH with `ssh root@`. Now you can use kubectl to manage your workloads right away. By default, the firewall only allows SSH connections from your public IPv4 address unless `firewall_ssh_source` is set to a different value in your kube.tf. + +To update your current public IPv4 address to the firewall, you can run: +```bash +terraform apply -target=module.kube-hetzner.hcloud_firewall.k3s +``` + +### Connect via Kube API + +Add an additional firewall rule to your kube.tf to access the Kube API. Be careful when exposing the Kube API. It is recommended not to expose it to the public world. If possible, only allow connections from trusted source IPs. Example configuration: +```hcl +extra_firewall_rules = [ + { + description = "Allow Incoming Requests to Kube API Server" + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = ["1.2.3.4/32"] + } +] +``` + You can immediately kubectl into it (using the `clustername_kubeconfig.yaml` saved to the project's directory after the installation). By doing `kubectl --kubeconfig clustername_kubeconfig.yaml`, but for more convenience, either create a symlink from `~/.kube/config` to `clustername_kubeconfig.yaml` or add an export statement to your `~/.bashrc` or `~/.zshrc` file, as follows (you can get the path of `clustername_kubeconfig.yaml` by running `pwd`): ```sh @@ -153,7 +179,6 @@ If chose to turn `create_kubeconfig` to false in your kube.tf (good practice), y You can also use it in an automated flow, in which case `create_kubeconfig` should be set to false, and you can use the `kubeconfig` output variable to get the kubeconfig file in a structured data format. -_You can view all kinds of details about the cluster by running `terraform output kubeconfig` or `terraform output -json kubeconfig | jq`._ ## CNI diff --git a/data.tf b/data.tf index 53c2f896..b9afd821 100644 --- a/data.tf +++ b/data.tf @@ -39,3 +39,14 @@ data "hcloud_ssh_keys" "keys_by_selector" { count = length(var.ssh_hcloud_key_label) > 0 ? 1 : 0 with_selector = var.ssh_hcloud_key_label } + +data "http" "client_public_ipv4" { + url = "https://ipv4.icanhazip.com" + + lifecycle { + postcondition { + condition = self.status_code == 200 + error_message = "Status code invalid" + } + } +} diff --git a/kube.tf.example b/kube.tf.example index 47b944c7..0f1c074b 100644 --- a/kube.tf.example +++ b/kube.tf.example @@ -470,10 +470,22 @@ module "kube-hetzner" { # If you want to allow all outbound traffic you can set this to "false". Default is "true". # restrict_outbound_traffic = false + # Allow SSH access from the specified networks. By default the public IPv4 address is automatically determined and applied to the firewall. + # If you want to update the firewall with your current public IP, you can execute `terraform apply -target=module.kube-hetzner.hcloud_firewall.k3s`. + # Allowed values: null (disable ssh rule entirely), [] (automatic IPv4 detection) or a list of allowed networks with CIDR notation (detection disabled) + # firewall_ssh_source = ["0.0.0.0/0", "::/0"] + # Adding extra firewall rules, like opening a port # More info on the format here https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall # extra_firewall_rules = [ # { + # description = "Allow Incoming Requests to Kube API Server" + # direction = "in" + # protocol = "tcp" + # port = "6443" + # source_ips = ["1.2.3.4/32"] + # }, + # { # description = "For Postgres" # direction = "in" # protocol = "tcp" diff --git a/locals.tf b/locals.tf index c983fdc0..68c899d7 100644 --- a/locals.tf +++ b/locals.tf @@ -119,51 +119,15 @@ locals { default_control_plane_taints = concat([], local.allow_scheduling_on_control_plane ? [] : ["node-role.kubernetes.io/control-plane:NoSchedule"]) default_agent_taints = concat([], var.cni_plugin == "cilium" ? ["node.cilium.io/agent-not-ready:NoExecute"] : []) - # The following IPs are important to be whitelisted because they communicate with Hetzner services and enable the CCM and CSI to work properly. - # Source https://github.com/hetznercloud/csi-driver/issues/204#issuecomment-848625566 - hetzner_metadata_service_ipv4 = "169.254.169.254/32" - hetzner_cloud_api_ipv4 = "213.239.246.21/32" - - whitelisted_ips = [ - var.network_ipv4_cidr, - local.hetzner_metadata_service_ipv4, - local.hetzner_cloud_api_ipv4, - "127.0.0.1/32", - ] - - base_firewall_rules = concat([ - # Allowing internal cluster traffic and Hetzner metadata service and cloud API IPs - { - description = "Allow Internal Cluster TCP Traffic" - direction = "in" - protocol = "tcp" - port = "any" - source_ips = local.whitelisted_ips - }, - { - description = "Allow Internal Cluster UDP Traffic" - direction = "in" - protocol = "udp" - port = "any" - source_ips = local.whitelisted_ips - }, - - # Allow all traffic to the kube api server - { - description = "Allow Incoming Requests to Kube API Server" - direction = "in" - protocol = "tcp" - port = "6443" - source_ips = ["0.0.0.0/0", "::/0"] - }, - + base_firewall_rules = concat( + var.firewall_ssh_source == null ? [] : [ # Allow all traffic to the ssh port { description = "Allow Incoming SSH Traffic" direction = "in" protocol = "tcp" port = var.ssh_port - source_ips = ["0.0.0.0/0", "::/0"] + source_ips = coalescelist(var.firewall_ssh_source, ["${chomp(data.http.client_public_ipv4.response_body)}/32"]) }, ], !var.restrict_outbound_traffic ? [] : [ # Allow basic out traffic @@ -241,15 +205,8 @@ locals { port = "" source_ips = ["0.0.0.0/0", "::/0"] } - ], var.cni_plugin != "cilium" ? [] : [ - { - description = "Allow Incoming Requests to Hubble Server & Hubble Relay (Cilium)" - direction = "in" - protocol = "tcp" - port = "4244-4245" - source_ips = ["0.0.0.0/0", "::/0"] - } - ]) + ] + ) # create a new firewall list based on base_firewall_rules but with direction-protocol-port as key # this is needed to avoid duplicate rules diff --git a/variables.tf b/variables.tf index 83bc0a70..0378ab93 100644 --- a/variables.tf +++ b/variables.tf @@ -356,6 +356,12 @@ variable "extra_firewall_rules" { description = "Additional firewall rules to apply to the cluster." } +variable "firewall_ssh_source" { + type = list(string) + default = [] + description = "Source networks that have SSH access to the servers." +} + variable "use_cluster_name_in_node_name" { type = bool default = true diff --git a/versions.tf b/versions.tf index 2f4397c8..5f30a67a 100644 --- a/versions.tf +++ b/versions.tf @@ -9,6 +9,10 @@ terraform { source = "hetznercloud/hcloud" version = ">= 1.41.0" } + http = { + source = "hashicorp/http" + version = ">= 3.0" + } local = { source = "hashicorp/local" version = ">= 2.0.0" From caa26f60f44b05fe65a2469ecc4d961f2f954fda Mon Sep 17 00:00:00 2001 From: M4t7e <7535523+M4t7e@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:05:46 +0200 Subject: [PATCH 2/4] Fixed space indentation --- data.tf | 2 +- locals.tf | 168 +++++++++++++++++++++++++++--------------------------- 2 files changed, 85 insertions(+), 85 deletions(-) diff --git a/data.tf b/data.tf index b9afd821..b721e058 100644 --- a/data.tf +++ b/data.tf @@ -42,7 +42,7 @@ data "hcloud_ssh_keys" "keys_by_selector" { data "http" "client_public_ipv4" { url = "https://ipv4.icanhazip.com" - + lifecycle { postcondition { condition = self.status_code == 200 diff --git a/locals.tf b/locals.tf index 68c899d7..187b6973 100644 --- a/locals.tf +++ b/locals.tf @@ -121,90 +121,90 @@ locals { base_firewall_rules = concat( var.firewall_ssh_source == null ? [] : [ - # Allow all traffic to the ssh port - { - description = "Allow Incoming SSH Traffic" - direction = "in" - protocol = "tcp" - port = var.ssh_port - source_ips = coalescelist(var.firewall_ssh_source, ["${chomp(data.http.client_public_ipv4.response_body)}/32"]) - }, - ], !var.restrict_outbound_traffic ? [] : [ - # Allow basic out traffic - # ICMP to ping outside services - { - description = "Allow Outbound ICMP Ping Requests" - direction = "out" - protocol = "icmp" - port = "" - destination_ips = ["0.0.0.0/0", "::/0"] - }, - - # DNS - { - description = "Allow Outbound TCP DNS Requests" - direction = "out" - protocol = "tcp" - port = "53" - destination_ips = ["0.0.0.0/0", "::/0"] - }, - { - description = "Allow Outbound UDP DNS Requests" - direction = "out" - protocol = "udp" - port = "53" - destination_ips = ["0.0.0.0/0", "::/0"] - }, - - # HTTP(s) - { - description = "Allow Outbound HTTP Requests" - direction = "out" - protocol = "tcp" - port = "80" - destination_ips = ["0.0.0.0/0", "::/0"] - }, - { - description = "Allow Outbound HTTPS Requests" - direction = "out" - protocol = "tcp" - port = "443" - destination_ips = ["0.0.0.0/0", "::/0"] - }, - - #NTP - { - description = "Allow Outbound UDP NTP Requests" - direction = "out" - protocol = "udp" - port = "123" - destination_ips = ["0.0.0.0/0", "::/0"] - } - ], !local.using_klipper_lb ? [] : [ - # Allow incoming web traffic for single node clusters, because we are using k3s servicelb there, - # not an external load-balancer. - { - description = "Allow Incoming HTTP Connections" - direction = "in" - protocol = "tcp" - port = "80" - source_ips = ["0.0.0.0/0", "::/0"] - }, - { - description = "Allow Incoming HTTPS Connections" - direction = "in" - protocol = "tcp" - port = "443" - source_ips = ["0.0.0.0/0", "::/0"] - } - ], var.block_icmp_ping_in ? [] : [ - { - description = "Allow Incoming ICMP Ping Requests" - direction = "in" - protocol = "icmp" - port = "" - source_ips = ["0.0.0.0/0", "::/0"] - } + # Allow all traffic to the ssh port + { + description = "Allow Incoming SSH Traffic" + direction = "in" + protocol = "tcp" + port = var.ssh_port + source_ips = coalescelist(var.firewall_ssh_source, ["${chomp(data.http.client_public_ipv4.response_body)}/32"]) + }, + ], !var.restrict_outbound_traffic ? [] : [ + # Allow basic out traffic + # ICMP to ping outside services + { + description = "Allow Outbound ICMP Ping Requests" + direction = "out" + protocol = "icmp" + port = "" + destination_ips = ["0.0.0.0/0", "::/0"] + }, + + # DNS + { + description = "Allow Outbound TCP DNS Requests" + direction = "out" + protocol = "tcp" + port = "53" + destination_ips = ["0.0.0.0/0", "::/0"] + }, + { + description = "Allow Outbound UDP DNS Requests" + direction = "out" + protocol = "udp" + port = "53" + destination_ips = ["0.0.0.0/0", "::/0"] + }, + + # HTTP(s) + { + description = "Allow Outbound HTTP Requests" + direction = "out" + protocol = "tcp" + port = "80" + destination_ips = ["0.0.0.0/0", "::/0"] + }, + { + description = "Allow Outbound HTTPS Requests" + direction = "out" + protocol = "tcp" + port = "443" + destination_ips = ["0.0.0.0/0", "::/0"] + }, + + #NTP + { + description = "Allow Outbound UDP NTP Requests" + direction = "out" + protocol = "udp" + port = "123" + destination_ips = ["0.0.0.0/0", "::/0"] + } + ], !local.using_klipper_lb ? [] : [ + # Allow incoming web traffic for single node clusters, because we are using k3s servicelb there, + # not an external load-balancer. + { + description = "Allow Incoming HTTP Connections" + direction = "in" + protocol = "tcp" + port = "80" + source_ips = ["0.0.0.0/0", "::/0"] + }, + { + description = "Allow Incoming HTTPS Connections" + direction = "in" + protocol = "tcp" + port = "443" + source_ips = ["0.0.0.0/0", "::/0"] + } + ], var.block_icmp_ping_in ? [] : [ + { + description = "Allow Incoming ICMP Ping Requests" + direction = "in" + protocol = "icmp" + port = "" + source_ips = ["0.0.0.0/0", "::/0"] + } ] ) From 9ef39931c94dfa77a83afe6d7f97c7b00898bc2d Mon Sep 17 00:00:00 2001 From: M4t7e <7535523+M4t7e@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:12:04 +0200 Subject: [PATCH 3/4] Fixed weird autoformatting from `terraform fmt` --- locals.tf | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/locals.tf b/locals.tf index 187b6973..e1ce3918 100644 --- a/locals.tf +++ b/locals.tf @@ -129,7 +129,8 @@ locals { port = var.ssh_port source_ips = coalescelist(var.firewall_ssh_source, ["${chomp(data.http.client_public_ipv4.response_body)}/32"]) }, - ], !var.restrict_outbound_traffic ? [] : [ + ], + !var.restrict_outbound_traffic ? [] : [ # Allow basic out traffic # ICMP to ping outside services { @@ -180,7 +181,8 @@ locals { port = "123" destination_ips = ["0.0.0.0/0", "::/0"] } - ], !local.using_klipper_lb ? [] : [ + ], + !local.using_klipper_lb ? [] : [ # Allow incoming web traffic for single node clusters, because we are using k3s servicelb there, # not an external load-balancer. { @@ -197,7 +199,8 @@ locals { port = "443" source_ips = ["0.0.0.0/0", "::/0"] } - ], var.block_icmp_ping_in ? [] : [ + ], + var.block_icmp_ping_in ? [] : [ { description = "Allow Incoming ICMP Ping Requests" direction = "in" From 7399d1733acdd50c5cf207e892e07ba212abced9 Mon Sep 17 00:00:00 2001 From: M4t7e <7535523+M4t7e@users.noreply.github.com> Date: Mon, 24 Jul 2023 21:04:01 +0200 Subject: [PATCH 4/4] Removed client public IP auto detection --- README.md | 28 +++++++++------------------- data.tf | 11 ----------- kube.tf.example | 18 +++++++----------- locals.tf | 11 ++++++++++- variables.tf | 8 +++++++- versions.tf | 4 ---- 6 files changed, 33 insertions(+), 47 deletions(-) diff --git a/README.md b/README.md index dab6b577..50088079 100644 --- a/README.md +++ b/README.md @@ -145,31 +145,21 @@ When your brand-new cluster is up and running, the sky is your limit! 🎉 You can view all kinds of details about the cluster by running `terraform output kubeconfig` or `terraform output -json kubeconfig | jq`. -### Connect via SSH +To manage your cluster with `kubectl`, you can either use SSH to connect to a control plane node or connect to the Kube API directly. -Connect to one of the control plane nodes via SSH with `ssh root@`. Now you can use kubectl to manage your workloads right away. By default, the firewall only allows SSH connections from your public IPv4 address unless `firewall_ssh_source` is set to a different value in your kube.tf. +### Connect via SSH -To update your current public IPv4 address to the firewall, you can run: -```bash -terraform apply -target=module.kube-hetzner.hcloud_firewall.k3s -``` +You can connect to one of the control plane nodes via SSH with `ssh root@`. Now you are able to use `kubectl` to manage your workloads right away. By default, the firewall allows SSH connections from everywhere. You can change that by configuring the `firewall_ssh_source` in your kube.tf file. ### Connect via Kube API -Add an additional firewall rule to your kube.tf to access the Kube API. Be careful when exposing the Kube API. It is recommended not to expose it to the public world. If possible, only allow connections from trusted source IPs. Example configuration: +Make sure you can connect to the Kube API from a trusted network by configuring `firewall_kube_api_source` in your kube.tf file like that: ```hcl -extra_firewall_rules = [ - { - description = "Allow Incoming Requests to Kube API Server" - direction = "in" - protocol = "tcp" - port = "6443" - source_ips = ["1.2.3.4/32"] - } -] -``` - -You can immediately kubectl into it (using the `clustername_kubeconfig.yaml` saved to the project's directory after the installation). By doing `kubectl --kubeconfig clustername_kubeconfig.yaml`, but for more convenience, either create a symlink from `~/.kube/config` to `clustername_kubeconfig.yaml` or add an export statement to your `~/.bashrc` or `~/.zshrc` file, as follows (you can get the path of `clustername_kubeconfig.yaml` by running `pwd`): +firewall_kube_api_source = ["1.2.3.4/32"] +``` +**Info:** Opening the Kube API to the public (`["0.0.0.0/0", "::/0"]`) is not recommended! + +If you have access to the Kube API, you can immediately kubectl into it (using the `clustername_kubeconfig.yaml` saved to the project's directory after the installation). By doing `kubectl --kubeconfig clustername_kubeconfig.yaml`, but for more convenience, either create a symlink from `~/.kube/config` to `clustername_kubeconfig.yaml` or add an export statement to your `~/.bashrc` or `~/.zshrc` file, as follows (you can get the path of `clustername_kubeconfig.yaml` by running `pwd`): ```sh export KUBECONFIG=//clustername_kubeconfig.yaml diff --git a/data.tf b/data.tf index b721e058..53c2f896 100644 --- a/data.tf +++ b/data.tf @@ -39,14 +39,3 @@ data "hcloud_ssh_keys" "keys_by_selector" { count = length(var.ssh_hcloud_key_label) > 0 ? 1 : 0 with_selector = var.ssh_hcloud_key_label } - -data "http" "client_public_ipv4" { - url = "https://ipv4.icanhazip.com" - - lifecycle { - postcondition { - condition = self.status_code == 200 - error_message = "Status code invalid" - } - } -} diff --git a/kube.tf.example b/kube.tf.example index f743dad8..372988cc 100644 --- a/kube.tf.example +++ b/kube.tf.example @@ -482,22 +482,18 @@ module "kube-hetzner" { # If you want to allow all outbound traffic you can set this to "false". Default is "true". # restrict_outbound_traffic = false - # Allow SSH access from the specified networks. By default the public IPv4 address is automatically determined and applied to the firewall. - # If you want to update the firewall with your current public IP, you can execute `terraform apply -target=module.kube-hetzner.hcloud_firewall.k3s`. - # Allowed values: null (disable ssh rule entirely), [] (automatic IPv4 detection) or a list of allowed networks with CIDR notation (detection disabled) - # firewall_ssh_source = ["0.0.0.0/0", "::/0"] + # Allow access to the Kube API from the specified networks. Default: ["0.0.0.0/0", "::/0"] + # Allowed values: null (disable Kube API rule entirely) or a list of allowed networks with CIDR notation + firewall_kube_api_source = null + + # Allow SSH access from the specified networks. Default: ["0.0.0.0/0", "::/0"] + # Allowed values: null (disable SSH rule entirely) or a list of allowed networks with CIDR notation + # firewall_ssh_source = ["1.2.3.4/32", "1234::1/128"] # Adding extra firewall rules, like opening a port # More info on the format here https://registry.terraform.io/providers/hetznercloud/hcloud/latest/docs/resources/firewall # extra_firewall_rules = [ # { - # description = "Allow Incoming Requests to Kube API Server" - # direction = "in" - # protocol = "tcp" - # port = "6443" - # source_ips = ["1.2.3.4/32"] - # }, - # { # description = "For Postgres" # direction = "in" # protocol = "tcp" diff --git a/locals.tf b/locals.tf index 334d4edf..77b44517 100644 --- a/locals.tf +++ b/locals.tf @@ -144,9 +144,18 @@ locals { direction = "in" protocol = "tcp" port = var.ssh_port - source_ips = coalescelist(var.firewall_ssh_source, ["${chomp(data.http.client_public_ipv4.response_body)}/32"]) + source_ips = var.firewall_ssh_source }, ], + var.firewall_kube_api_source == null ? [] : [ + { + description = "Allow Incoming Requests to Kube API Server" + direction = "in" + protocol = "tcp" + port = "6443" + source_ips = var.firewall_kube_api_source + } + ], !var.restrict_outbound_traffic ? [] : [ # Allow basic out traffic # ICMP to ping outside services diff --git a/variables.tf b/variables.tf index 0b227b35..44f6b955 100644 --- a/variables.tf +++ b/variables.tf @@ -379,9 +379,15 @@ variable "extra_firewall_rules" { description = "Additional firewall rules to apply to the cluster." } +variable "firewall_kube_api_source" { + type = list(string) + default = ["0.0.0.0/0", "::/0"] + description = "Source networks that have Kube API access to the servers." +} + variable "firewall_ssh_source" { type = list(string) - default = [] + default = ["0.0.0.0/0", "::/0"] description = "Source networks that have SSH access to the servers." } diff --git a/versions.tf b/versions.tf index 5f30a67a..2f4397c8 100644 --- a/versions.tf +++ b/versions.tf @@ -9,10 +9,6 @@ terraform { source = "hetznercloud/hcloud" version = ">= 1.41.0" } - http = { - source = "hashicorp/http" - version = ">= 3.0" - } local = { source = "hashicorp/local" version = ">= 2.0.0"