diff --git a/README.md b/README.md index 1d9986e2..f41c06f6 100644 --- a/README.md +++ b/README.md @@ -70,7 +70,7 @@ Follow those simple steps, and your world's cheapest Kubernetes cluster will be First and foremost, you need to have a Hetzner Cloud account. You can sign up for free [here](https://hetzner.com/cloud/). -Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli), [packer](https://developer.hashicorp.com/packer/tutorials/docker-get-started/get-started-install-cli#installing-packer) (for the initial snapshot creation only, no longer needed once that's done), [kubectl](https://kubernetes.io/docs/tasks/tools/) cli and [hcloud]() the Hetzner cli for convenience. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem). +Then you'll need to have [terraform](https://learn.hashicorp.com/tutorials/terraform/install-cli), [packer](https://developer.hashicorp.com/packer/tutorials/docker-get-started/get-started-install-cli#installing-packer) (for the initial snapshot creation only, no longer needed once that's done), [kubectl](https://kubernetes.io/docs/tasks/tools/) cli and [hcloud](https://github.com/hetznercloud/cli) the Hetzner cli for convenience. The easiest way is to use the [homebrew](https://brew.sh/) package manager to install them (available on Linux, Mac, and Windows Linux Subsystem). ```sh brew install terraform @@ -85,44 +85,44 @@ brew install hcloud 2. Generate a passphrase-less ed25519 SSH key pair for your cluster; take note of the respective paths of your private and public keys. Or, see our detailed [SSH options](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/blob/master/docs/ssh.md). ✅ 3. Now navigate to where you want to have your project live and execute the following command, which will help you get started with a **new folder** along with the required files, and will propose you to create a needed MicroOS snapshot. ✅ - ```sh - tmp_script=$(mktemp) && curl -sSL -o "${tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh && chmod +x "${tmp_script}" && "${tmp_script}" && rm "${tmp_script}" - ``` + ```sh + tmp_script=$(mktemp) && curl -sSL -o "${tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh && chmod +x "${tmp_script}" && "${tmp_script}" && rm "${tmp_script}" + ``` - Or for fish shell: + Or for fish shell: - ```fish - set tmp_script (mktemp); curl -sSL -o "{tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh; chmod +x "{tmp_script}"; bash "{tmp_script}"; rm "{tmp_script}" - ``` + ```fish + set tmp_script (mktemp); curl -sSL -o "{tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh; chmod +x "{tmp_script}"; bash "{tmp_script}"; rm "{tmp_script}" + ``` - _Optionally, for future usage, save that command as an alias in your shell preferences, like so:_ + _Optionally, for future usage, save that command as an alias in your shell preferences, like so:_ - ```sh - alias createkh='tmp_script=$(mktemp) && curl -sSL -o "${tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh && chmod +x "${tmp_script}" && "${tmp_script}" && rm "${tmp_script}"' - ``` + ```sh + alias createkh='tmp_script=$(mktemp) && curl -sSL -o "${tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh && chmod +x "${tmp_script}" && "${tmp_script}" && rm "${tmp_script}"' + ``` - Or for fish shell: + Or for fish shell: - ```fish - alias createkh='set tmp_script (mktemp); curl -sSL -o "{tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh; chmod +x "{tmp_script}"; bash "{tmp_script}"; rm "{tmp_script}"' - ``` + ```fish + alias createkh='set tmp_script (mktemp); curl -sSL -o "{tmp_script}" https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/scripts/create.sh; chmod +x "{tmp_script}"; bash "{tmp_script}"; rm "{tmp_script}"' + ``` - _For the curious, here is what the script does:_ + _For the curious, here is what the script does:_ - ```sh - mkdir /path/to/your/new/folder - cd /path/to/your/new/folder - curl -sL https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/kube.tf.example -o kube.tf - curl -sL https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/packer-template/hcloud-microos-snapshots.pkr.hcl -o hcloud-microos-snapshots.pkr.hcl - export HCLOUD_TOKEN="your_hcloud_token" - packer init hcloud-microos-snapshots.pkr.hcl - packer build hcloud-microos-snapshots.pkr.hcl - hcloud context create - ``` + ```sh + mkdir /path/to/your/new/folder + cd /path/to/your/new/folder + curl -sL https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/kube.tf.example -o kube.tf + curl -sL https://raw.githubusercontent.com/kube-hetzner/terraform-hcloud-kube-hetzner/master/packer-template/hcloud-microos-snapshots.pkr.hcl -o hcloud-microos-snapshots.pkr.hcl + export HCLOUD_TOKEN="your_hcloud_token" + packer init hcloud-microos-snapshots.pkr.hcl + packer build hcloud-microos-snapshots.pkr.hcl + hcloud context create + ``` -1. In that new project folder that gets created, you will find your `kube.tf` and it must be customized to suit your needs. ✅ +4. In that new project folder that gets created, you will find your `kube.tf` and it must be customized to suit your needs. ✅ - _A complete reference of all inputs, outputs, modules etc. can be found in the [terraform.md](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/blob/master/docs/terraform.md) file._ + _A complete reference of all inputs, outputs, modules etc. can be found in the [terraform.md](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/blob/master/docs/terraform.md) file._ ### 🎯 Installation @@ -403,16 +403,16 @@ metadata: name: egress-sample spec: selectors: - - podSelector: - matchLabels: - org: empire - class: mediabot - io.kubernetes.pod.namespace: default + - podSelector: + matchLabels: + org: empire + class: mediabot + io.kubernetes.pod.namespace: default destinationCIDRs: - - "0.0.0.0/0" + - "0.0.0.0/0" excludedCIDRs: - - "10.0.0.0/8" + - "10.0.0.0/8" egressGateway: nodeSelector: @@ -421,7 +421,7 @@ spec: # Specify the IP address used to SNAT traffic matched by the policy. # It must exist as an IP associated with a network interface on the instance. - egressIP: {FLOATING_IP} + egressIP: { FLOATING_IP } ``` @@ -449,20 +449,20 @@ metadata: cert-manager.io/cluster-issuer: letsencrypt spec: tls: - - hosts: - - '*.example.com' - secretName: example-com-letsencrypt-tls + - hosts: + - "*.example.com" + secretName: example-com-letsencrypt-tls rules: - - host: '*.example.com' - http: - paths: - - path: / - pathType: Prefix - backend: - service: - name: my-service - port: - number: 80 + - host: "*.example.com" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: my-service + port: + number: 80 ``` _⚠️ In case of using Ingress-Nginx as an ingress controller if you choose to use the HTTP challenge method you need to do an additional step of adding variable `lb_hostname = "cluster.example.org"` to your kube.tf. You must set it to an FQDN that points to your LB address._ @@ -562,17 +562,17 @@ stringData: And to create a new storage class: ```yaml - apiVersion: storage.k8s.io/v1 - kind: StorageClass - metadata: - name: hcloud-volumes-encrypted - provisioner: csi.hetzner.cloud - reclaimPolicy: Delete - volumeBindingMode: WaitForFirstConsumer - allowVolumeExpansion: true - parameters: - csi.storage.k8s.io/node-publish-secret-name: encryption-secret - csi.storage.k8s.io/node-publish-secret-namespace: kube-system +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: hcloud-volumes-encrypted +provisioner: csi.hetzner.cloud +reclaimPolicy: Delete +volumeBindingMode: WaitForFirstConsumer +allowVolumeExpansion: true +parameters: + csi.storage.k8s.io/node-publish-secret-name: encryption-secret + csi.storage.k8s.io/node-publish-secret-namespace: kube-system ``` @@ -630,6 +630,7 @@ For more details, see [Longhorn's documentation](https://longhorn.io/docs/1.4.0/ Assign all pods in a namespace to either arm64 or amd64 nodes with admission controllers To enable the [PodNodeSelector and optionally the PodTolerationRestriction](https://kubernetes.io/docs/reference/access-authn-authz/admission-controllers/#podnodeselector) api modules, set the following value: + ```terraform k3s_exec_server_args = "--kube-apiserver-arg enable-admission-plugins=PodTolerationRestriction,PodNodeSelector" ``` @@ -637,6 +638,7 @@ k3s_exec_server_args = "--kube-apiserver-arg enable-admission-plugins=PodTolerat Next, you can set default nodeSelector values per namespace. This lets you assign namespaces to specific nodes. Note though, that this is the default as well as the whitelist, so if a pod sets its own nodeSelector value that must be a subset of the default. Otherwise, the pod will not be scheduled. Then set the according annotations on your namespaces: + ```yaml apiVersion: v1 kind: Namespace @@ -645,25 +647,23 @@ metadata: scheduler.alpha.kubernetes.io/node-selector: kubernetes.io/arch=amd64 name: this-runs-on-amd64 ``` + or with taints and tolerations: + ```yaml apiVersion: v1 kind: Namespace metadata: annotations: scheduler.alpha.kubernetes.io/node-selector: kubernetes.io/arch=arm64 - scheduler.alpha.kubernetes.io/defaultTolerations: "[{ \"operator\" : \"Equal\", \"effect\" : \"NoSchedule\", \"key\" : \"workload-type\", \"value\" : \"machine-learning\" }]" + scheduler.alpha.kubernetes.io/defaultTolerations: '[{ "operator" : "Equal", "effect" : "NoSchedule", "key" : "workload-type", "value" : "machine-learning" }]' name: this-runs-on-arm64 ``` This can be helpful when you set up a mixed-architecture cluster, and there are many other use cases. - - - -
Backup and restore a cluster @@ -671,17 +671,21 @@ This can be helpful when you set up a mixed-architecture cluster, and there are K3s allows for automated etcd backups to S3. Etcd is the default storage backend on kube-hetzner, even for a single control plane cluster, hence this should work for all cluster deployments. **For backup do:** + 1. Fill the kube.tf config `etcd_s3_backup`, it will trigger a regular automated backup to S3. 2. Add the k3s_token as an output to your kube.tf + ```tf output "k3s_token" { value = module.kube-hetzner.k3s_token sensitive = true } ``` + 3. Make sure you can access the k3s_token via `terraform output k3s_token`. **For restoration do:** + 1. Before cluster creation, add the following to your kube.tf. Replace the local variables to match your values. ```tf @@ -774,15 +778,69 @@ module "kube-hetzner" { ``` 2. Set the following sensible environment variables - - `export TF_VAR_k3s_token="..."` (Be careful, this token is like an admin password to the entire cluster. You need to use the same k3s_token which you saved when creating the backup.) - - `export etcd_s3_secret_key="..."` + + - `export TF_VAR_k3s_token="..."` (Be careful, this token is like an admin password to the entire cluster. You need to use the same k3s_token which you saved when creating the backup.) + - `export etcd_s3_secret_key="..."` 3. Create the cluster as usual. You can also change the cluster-name and deploy it next to the original backed up cluster. Awesome! You restored a whole cluster from a backup.
+
+Deploy in a pre-constructed private network (for proxies etc) +If you want to deploy other machines on the private network before deploying the k3s cluster, +you can. One use-case is if you want to setup a proxy or a NAT router on the private network, +which is needed by the k3s cluster already at the time of construction. + +It is important to get all the address ranges right in this case, although the +number of changes needed is minimal. If your network is created with 10.0.0.0/8, +and you use subnet 10.128.0.0/9 for your non-k3s business, then adapting +`network_ipv4_cidr = "10.0.0.0/9"` should be all you need. + +For example +``` +resource "hcloud_network" "k3s_proxied" { + name = "k3s-proxied" + ip_range = "10.0.0.0/8" +} + +resource "hcloud_network_subnet" "k3s_proxy" { + network_id = hcloud_network.k3s_proxied.id + type = "cloud" + network_zone = "eu-central" + ip_range = "10.128.0.0/9" +} +resource "hcloud_server" "your_proxy_server" { + ... +} +resource "hcloud_server_network" "your_proxy_server" { + depends_on = [ + hcloud_server.your_proxy_server + ] + server_id = hcloud_server.your_proxy_server.id + network_id = hcloud_network.k3s_proxied.id + ip = "10.128.0.1" +} +module "kube-hetzner" { + ... + existing_network_id = [hcloud_network.k3s_proxied.id] + network_ipv4_cidr = "10.0.0.0/9" + additional_k3s_environment = { + "http_proxy" : "http://10.128.0.1:3128", + "HTTP_PROXY" : "http://10.128.0.1:3128", + "HTTPS_PROXY" : "http://10.128.0.1:3128", + "CONTAINERD_HTTP_PROXY" : "http://10.128.0.1:3128", + "CONTAINERD_HTTPS_PROXY" : "http://10.128.0.1:3128", + "NO_PROXY" : "127.0.0.0/8,10.0.0.0/8,", + } +} +``` + +NOTE: square brackets in existing_network_id! This must be a list of length 1. + +
## Debugging @@ -838,6 +896,7 @@ When moving from 1.x to 2.x: - Then run `terraform init -upgrade && terraform apply`. + ## History This project has tried two other OS flavors before settling on MicroOS. Fedora Server, and k3OS. The latter, k3OS, is now defunct! However, our code base for it lives on in the [k3os branch](https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/tree/k3os). Do not hesitate to check it out, it should still work. @@ -854,17 +913,17 @@ Code contributions are very much **welcome**. 1. Create your Branch (`git checkout -b AmazingFeature`) 1. Develop your feature - In your kube.tf, point the `source` of module to your local clone of the repo. + In your kube.tf, point the `source` of module to your local clone of the repo. - Useful commands: + Useful commands: - ```sh - # To cleanup a Hetzner project - ../kube-hetzner/scripts/cleanup.sh + ```sh + # To cleanup a Hetzner project + ../kube-hetzner/scripts/cleanup.sh - # To build the Packer image - packer build ../kube-hetzner/packer-template/hcloud-microos-snapshots.pkr.hcl - ``` + # To build the Packer image + packer build ../kube-hetzner/packer-template/hcloud-microos-snapshots.pkr.hcl + ``` 1. Update examples in `kube.tf.example` if required. 1. Commit your Changes (`git commit -m 'Add some AmazingFeature') @@ -872,6 +931,7 @@ Code contributions are very much **welcome**. 1. Open a Pull Request targeting the `staging` branch. + ## Acknowledgements - [k-andy](https://github.com/StarpTech/k-andy) was the starting point for this project. It wouldn't have been possible without it. @@ -882,4 +942,5 @@ Code contributions are very much **welcome**. - [openSUSE](https://www.opensuse.org) for MicroOS, which is just next-level Container OS technology. + [product-screenshot]: https://github.com/kube-hetzner/terraform-hcloud-kube-hetzner/raw/master/.images/kubectl-pod-all-17022022.png diff --git a/autoscaler-agents.tf b/autoscaler-agents.tf index 17b41e86..bbc0f4d0 100644 --- a/autoscaler-agents.tf +++ b/autoscaler-agents.tf @@ -14,7 +14,7 @@ locals { cluster_autoscaler_log_to_stderr = var.cluster_autoscaler_log_to_stderr cluster_autoscaler_stderr_threshold = var.cluster_autoscaler_stderr_threshold ssh_key = local.hcloud_ssh_key_id - ipv4_subnet_id = hcloud_network.k3s.id + ipv4_subnet_id = data.hcloud_network.k3s.id snapshot_id = local.first_nodepool_snapshot_id firewall_id = hcloud_firewall.k3s.id cluster_name = local.cluster_prefix diff --git a/init.tf b/init.tf index 1f3edc2d..f71f8c49 100644 --- a/init.tf +++ b/init.tf @@ -250,7 +250,7 @@ resource "null_resource" "kustomization" { provisioner "remote-exec" { inline = [ "set -ex", - "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${hcloud_network.k3s.name} --dry-run=client -o yaml | kubectl apply -f -", + "kubectl -n kube-system create secret generic hcloud --from-literal=token=${var.hcloud_token} --from-literal=network=${data.hcloud_network.k3s.name} --dry-run=client -o yaml | kubectl apply -f -", "kubectl -n kube-system create secret generic hcloud-csi --from-literal=token=${var.hcloud_token} --dry-run=client -o yaml | kubectl apply -f -", local.csi_version != null ? "curl https://raw.githubusercontent.com/hetznercloud/csi-driver/${coalesce(local.csi_version, "v2.4.0")}/deploy/kubernetes/hcloud-csi.yml -o /var/post_install/hcloud-csi.yml" : "echo 'Skipping hetzner csi.'" ] diff --git a/kube.tf.example b/kube.tf.example index ceb2dca1..76a4bd1b 100644 --- a/kube.tf.example +++ b/kube.tf.example @@ -57,6 +57,18 @@ module "kube-hetzner" { # * For Hetzner locations see https://docs.hetzner.com/general/others/data-centers-and-connection/ network_region = "eu-central" # change to `us-east` if location is ash + # If you want to create the private network before calling this module, + # you can do so and pass its id here. For example if you want to use a proxy + # which only listens on your private network. Advanced use case. + # + # NOTE1: make sure to adapt network_ipv4_cidr, cluster_ipv4_cidr, and service_ipv4_cidr accordingly. + # If your network is created with 10.0.0.0/8, and you use subnet 10.128.0.0/9 for your + # non-k3s business, then adapting `network_ipv4_cidr = "10.0.0.0/9"` should be all you need. + # + # NOTE2: square brackets! This must be a list of length 1. + # + # existing_network_id = [hcloud_network.your_network.id] + # If you must change the network CIDR you can do so below, but it is highly advised against. # network_ipv4_cidr = "10.0.0.0/8" diff --git a/locals.tf b/locals.tf index 394c64cc..0b253864 100644 --- a/locals.tf +++ b/locals.tf @@ -141,6 +141,8 @@ locals { } ]...) + use_existing_network = length(var.existing_network_id) > 0 + # The first two subnets are respectively the default subnet 10.0.0.0/16 use for potientially anything and 10.1.0.0/16 used for control plane nodes. # the rest of the subnets are for agent nodes in each nodepools. network_ipv4_subnets = [for index in range(256) : cidrsubnet(var.network_ipv4_cidr, 8, index)] diff --git a/main.tf b/main.tf index fead3288..3e1cbb13 100644 --- a/main.tf +++ b/main.tf @@ -23,16 +23,21 @@ resource "hcloud_ssh_key" "k3s" { } resource "hcloud_network" "k3s" { + count = local.use_existing_network ? 0 : 1 name = var.cluster_name ip_range = var.network_ipv4_cidr labels = local.labels } +data "hcloud_network" "k3s" { + id = local.use_existing_network ? var.existing_network_id[0] : hcloud_network.k3s[0].id +} + # We start from the end of the subnets cidr array, # as we would have fewer control plane nodepools, than agent ones. resource "hcloud_network_subnet" "control_plane" { count = length(var.control_plane_nodepools) - network_id = hcloud_network.k3s.id + network_id = data.hcloud_network.k3s.id type = "cloud" network_zone = var.network_region ip_range = local.network_ipv4_subnets[255 - count.index] @@ -41,7 +46,7 @@ resource "hcloud_network_subnet" "control_plane" { # Here we start at the beginning of the subnets cidr array resource "hcloud_network_subnet" "agent" { count = length(var.agent_nodepools) - network_id = hcloud_network.k3s.id + network_id = data.hcloud_network.k3s.id type = "cloud" network_zone = var.network_region ip_range = local.network_ipv4_subnets[count.index] diff --git a/output.tf b/output.tf index 0faf83de..dda5d9c8 100644 --- a/output.tf +++ b/output.tf @@ -4,7 +4,7 @@ output "cluster_name" { } output "network_id" { - value = hcloud_network.k3s.id + value = data.hcloud_network.k3s.id description = "The ID of the HCloud network." } diff --git a/variables.tf b/variables.tf index b04d0afe..3faffaab 100644 --- a/variables.tf +++ b/variables.tf @@ -74,7 +74,22 @@ variable "network_region" { type = string default = "eu-central" } - +variable "existing_network_id" { + # Unfortunately, we need this to be a list or null. If we only use a plain + # string here, and check that existing_network_id is null, terraform will + # complain that it cannot set `count` variables based on existing_network_id + # != null, because that id is an output value from + # hcloud_network.your_network.id, which terraform will only know after its + # construction. + description = "If you want to create the private network before calling this module, you can do so and pass its id here. NOTE: make sure to adapt network_ipv4_cidr accordingly to a range which does not collide with your other nodes." + type = list(string) + default = [] + nullable = false + validation { + condition = var.existing_network_id == null || (can(var.existing_network_id[0]) && length(var.existing_network_id) == 1) + error_message = "If you pass an existing_network_id, it must be enclosed in square brackets: [id]. This is necessary to be able to unambiguously distinguish between an empty network id (default) and a user-supplied network id." + } +} variable "network_ipv4_cidr" { description = "The main network cidr that all subnets will be created upon." type = string