From 0b0ac794b2ba9e580cdaef2fa6d21040b6ea9e42 Mon Sep 17 00:00:00 2001 From: e-minguez Date: Thu, 3 Oct 2024 13:56:58 +0200 Subject: [PATCH] October 2024 revamp Replaced Bird by native MetalLB Setup Ingress IP properly Added Rancher deployment K3s or RKE2 available Also, fixed pre-commit action --- .github/workflows/pre-commit.yaml | 2 + README.md | 289 +++++-- examples/demo_cluster/README.md | 12 +- examples/demo_cluster/outputs.tf | 2 +- .../demo_cluster/terraform.tfvars.example | 23 +- examples/demo_cluster/variables.tf | 17 +- main.tf | 15 +- modules/k3s_cluster/outputs.tf | 4 - modules/k3s_cluster/templates/user-data.tftpl | 388 ---------- .../{k3s_cluster => kube_cluster}/README.md | 46 +- modules/{k3s_cluster => kube_cluster}/main.tf | 74 +- modules/kube_cluster/outputs.tf | 34 + .../kube_cluster/templates/user-data.tftpl | 707 ++++++++++++++++++ .../variables.tf | 44 +- .../{k3s_cluster => kube_cluster}/versions.tf | 0 outputs.tf | 22 +- variables.tf | 17 +- 17 files changed, 1144 insertions(+), 552 deletions(-) delete mode 100644 modules/k3s_cluster/outputs.tf delete mode 100644 modules/k3s_cluster/templates/user-data.tftpl rename modules/{k3s_cluster => kube_cluster}/README.md (62%) rename modules/{k3s_cluster => kube_cluster}/main.tf (68%) create mode 100644 modules/kube_cluster/outputs.tf create mode 100644 modules/kube_cluster/templates/user-data.tftpl rename modules/{k3s_cluster => kube_cluster}/variables.tf (57%) rename modules/{k3s_cluster => kube_cluster}/versions.tf (100%) diff --git a/.github/workflows/pre-commit.yaml b/.github/workflows/pre-commit.yaml index 669555c..c2c54d9 100644 --- a/.github/workflows/pre-commit.yaml +++ b/.github/workflows/pre-commit.yaml @@ -31,6 +31,8 @@ jobs: - name: Install Python3 uses: actions/setup-python@v5 + with: + python-version: '3.x' - name: Install tflint uses: terraform-linters/setup-tflint@v4 diff --git a/README.md b/README.md index 47d5455..fcc8251 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# K3s on Equinix Metal +# K3s/RKE2 on Equinix Metal [![GitHub release](https://img.shields.io/github/release/equinix-labs/terraform-equinix-metal-k3s/all.svg?style=flat-square)](https://github.com/equinix-labs/terraform-equinix-metal-k3s/releases) ![](https://img.shields.io/badge/Stability-Experimental-red.svg) @@ -33,18 +33,19 @@ ## Introduction -This is a [Terraform](hhttps://registry.terraform.io/providers/equinix/metal/latest/docs) project for deploying [K3s](https://k3s.io) on [Equinix Metal](https://metal.equinix.com) intended to allow you to quickly spin-up and down K3s clusters. +This is a [Terraform](hhttps://registry.terraform.io/providers/equinix/metal/latest/docs) project for deploying [K3s](https://k3s.io) (or [RKE2](https://docs.rke2.io/)) on [Equinix Metal](https://metal.equinix.com) intended to allow you to quickly spin-up and down K3s/RKE2 clusters. -[K3s](https://docs.k3s.io/) is a fully compliant and lightweight Kubernetes distribution focused on Edge, IoT, ARM or just for situations where a PhD in K8s clusterology is infeasible. +[K3s](https://docs.k3s.io/) is a fully compliant and lightweight Kubernetes distribution focused on Edge, IoT, ARM or just for situations where a PhD in K8s clusterology is infeasible. [RKE2](https://docs.rke2.io/) is Rancher’s next-generation Kubernetes distribution, it combines the best-of-both-worlds from the 1.x version of RKE (hereafter referred to as RKE1) anxd K3s. From K3s, it inherits the usability, ease-of-operations, and deployment model. From RKE1, it inherits close alignment with upstream Kubernetes. In places K3s has diverged from upstream Kubernetes in order to optimize for edge deployments, but RKE1 and RKE2 can stay closely aligned with upstream. > :warning: This repository is [Experimental](https://github.com/packethost/standards/blob/master/experimental-statement.md) meaning that it's based on untested ideas or techniques and not yet established or finalized or involves a radically new and innovative style! This means that support is best effort (at best!) and we strongly encourage you to NOT use this in production. This terraform project supports a wide variety of scenarios and mostly focused on Edge, such as: -* Single node K3s cluster on a single Equinix Metal Metro. -* HA K3s cluster (3 control plane nodes) using BGP to provide an HA K3s API entrypoint. +* Single node K3s/RKE2 cluster on a single Equinix Metal Metro. +* HA K3s/RKE2 cluster (3 control plane nodes) using [MetalLB](https://metallb.universe.tf/) + BGP to provide an HA K3s/RKE2 API entrypoint. * Any number of worker nodes (both for single node or HA scenarios). -* Any number of public IPv4s to be used to expose services to the outside using `LoadBalancer` services via [MetalLB](https://metallb.universe.tf/) (deployed automatically). +* Any number of public IPv4s to be used to expose services to the outside using `LoadBalancer` services via MetalLB. +* Optionally it can deploy Rancher Manager on top of the cluster. * All those previous scenarios but deploying multiple clusters on multiple Equinix Metal metros. * A Global IPv4 that is shared in all cluster among all Equnix Metal Metros and can be used to expose an example application to demonstrate load balancing between different Equinix Metal Metros. @@ -89,13 +90,13 @@ There is a lot of flexibility in the module to allow customization of the differ |------|-------------|------|---------|:--------:| | [metal\_auth\_token](#input\_metal\_auth\_token) | Your Equinix Metal API key | `string` | n/a | yes | | [metal\_project\_id](#input\_metal\_project\_id) | Your Equinix Metal Project ID | `string` | n/a | yes | -| [clusters](#input\_clusters) | K3s cluster definition | `list of K3s cluster objects` | n/a | yes | +| [clusters](#input\_clusters) | Kubernetes cluster definition | `list of kubernetes cluster objects` | n/a | yes | > :note: The Equinix Metal Auth Token should be defined in a `provider` block in your own Terraform config. In this project, that is done in `examples/demo_cluster/`, not in the root. This pattern facilitates [Implicit Provider Inheritance](https://developer.hashicorp.com/terraform/language/modules/develop/providers#implicit-provider-inheritance) and better reuse of Terraform modules. For more details on the variables, see the [Terraform module documentation](#terraform-module-documentation) section. -The default variables are set to deploy a single node K3s cluster in the FR Metro, using a Equinix Metal's c3.small.x86. You just need to add the cluster name as: +The default variables are set to deploy a single node K3s (latest K3s version available) cluster in the FR Metro, using a Equinix Metal's c3.small.x86. You just need to add the cluster name as: ```bash metal_auth_token = "redacted" @@ -107,7 +108,7 @@ clusters = [ ] ``` -Change each default variable at your own risk, see [Example scenarios](#example-scenarios) and the [K3s module README.md file](modules/k3s_cluster/README.md) for more details. +Change each default variable at your own risk, see [Example scenarios](#example-scenarios) and the [kube_cluster module README.md file](modules/kube_cluster/README.md) for more details. > :warning: The hostnames are created based on the Cluster Name and the `control_plane_hostnames` & `node_hostnames` variables (normalized), beware the lenght of those variables. @@ -117,7 +118,7 @@ You can create a [terraform.tfvars](https://developer.hashicorp.com/terraform/la ## Demo application -If enabled (`deploy_demo = true`), a demo application ([hello-kubernetes](https://github.com/paulbouwer/hello-kubernetes)) will be deployed on all the clusters. The Global IPv4 will be used by the [K3s Traefik Ingress Controller](https://docs.k3s.io/networking#traefik-ingress-controller) to expose that application and the load will be spreaded among all the clusters. This means that different requests will be routed to different clusters. See [the MetalLB documentation](https://metallb.universe.tf/concepts/bgp/#load-balancing-behavior) for more information about how BGP load balancing works. +If enabled (`deploy_demo = true`), a demo application ([hello-kubernetes](https://github.com/paulbouwer/hello-kubernetes)) will be deployed on all the clusters. An extra [Ingress-NGINX Controller](https://github.com/kubernetes/ingress-nginx) is deployed on each cluster to expose that application and the load will be spreaded among all the clusters. This means that different requests will be routed to different clusters. See [the MetalLB documentation](https://metallb.universe.tf/concepts/bgp/#load-balancing-behavior) for more information about how BGP load balancing works. ## Example scenarios @@ -138,8 +139,18 @@ This will produce something similar to: ```bash Outputs: -k3s_api = { - "FR DEV Cluster" = "145.40.94.83" +clusters_output = { + "cluster_details" = { + "FR DEV Cluster" = { + "api" = "147.28.184.239" + "nodes" = { + "fr-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.25.49.1" + "node_public_ipv4" = "147.28.184.239" + } + } + } + } } ``` @@ -164,36 +175,59 @@ This will produce something similar to: ```bash Outputs: -k3s_api = { - "FR DEV Cluster" = "145.40.94.83", - "SV DEV Cluster" = "86.109.11.205" +clusters_output = { + "cluster_details" = { + "FR DEV Cluster" = { + "api" = "147.28.184.239" + "nodes" = { + "fr-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.25.49.1" + "node_public_ipv4" = "147.28.184.239" + } + } + } + "SV DEV Cluster" = { + "api" = "139.178.70.53" + "nodes" = { + "sv-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.67.31.129" + "node_public_ipv4" = "139.178.70.53" + } + } + } + } } ``` -### 1 x HA cluster with 3 nodes & 4 public IPs + 2 x Single Node cluster (same Metro), a Global IPV4 and the demo app deployed +### 1 x All-in-one cluster with Rancher (stable), a custom K3s version & 1 public IP (+1 for Ingress) + 1 x All-in-one with 1 extra node & a custom RKE2 version + 1 x HA cluster with 3 nodes & 4 public IPs. Global IPV4 and demo app deployed ```bash metal_auth_token = "redacted" metal_project_id = "redacted" -clusters = [{ - name = "SV Production" - ip_pool_count = 4 - k3s_ha = true - metro = "SV" - node_count = 3 -}, -{ - name = "FR Dev 1" - metro = "FR" -}, -{ - name = "FR Dev 2" - metro = "FR" -} +clusters = [ + { + name = "FR DEV Cluster" + rancher_flavor = "stable" + ip_pool_count = 1 + kube_version = "v1.29.9+k3s1" + }, + { + name = "SV DEV Cluster" + metro = "SV" + node_count = 1 + kube_version = "v1.30.3+rke2r1" + }, + { + name = "SV Production" + ip_pool_count = 4 + ha = true + metro = "SV" + node_count = 3 + } ] -global_ip = true -deploy_demo = true +global_ip = true +deploy_demo = true ``` This will produce something similar to: @@ -201,12 +235,72 @@ This will produce something similar to: ```bash Outputs: -anycast_ip = "147.75.40.52" -demo_url = "http://hellok3s.147.75.40.52.sslip.io" -k3s_api = { - "FR Dev 1" = "145.40.94.83", - "FR Dev 2" = "147.75.192.250", - "SV Production" = "86.109.11.205" +clusters_output = { + "anycast_ip" = "147.75.40.34" + "cluster_details" = { + "FR DEV Cluster" = { + "api" = "147.28.184.239" + "ingress" = "147.28.184.119" + "ip_pool_cidr" = "147.28.184.118/32" + "nodes" = { + "fr-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.25.49.1" + "node_public_ipv4" = "147.28.184.239" + } + } + } + "SV DEV Cluster" = { + "api" = "139.178.70.53" + "nodes" = { + "sv-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.67.31.129" + "node_public_ipv4" = "139.178.70.53" + } + "sv-dev-cluster-node-00" = { + "node_private_ipv4" = "10.67.31.131" + "node_public_ipv4" = "86.109.11.115" + } + } + } + "SV Production" = { + "api" = "86.109.11.239" + "ingress" = "86.109.11.53" + "ip_pool_cidr" = "139.178.70.68/30" + "nodes" = { + "sv-production-cp-0" = { + "node_private_ipv4" = "10.67.31.133" + "node_public_ipv4" = "139.178.70.141" + } + "sv-production-cp-1" = { + "node_private_ipv4" = "10.67.31.137" + "node_public_ipv4" = "136.144.54.109" + } + "sv-production-cp-2" = { + "node_private_ipv4" = "10.67.31.143" + "node_public_ipv4" = "139.178.94.11" + } + "sv-production-node-00" = { + "node_private_ipv4" = "10.67.31.141" + "node_public_ipv4" = "136.144.54.113" + } + "sv-production-node-01" = { + "node_private_ipv4" = "10.67.31.135" + "node_public_ipv4" = "139.178.70.233" + } + "sv-production-node-02" = { + "node_private_ipv4" = "10.67.31.139" + "node_public_ipv4" = "136.144.54.111" + } + } + } + } + "demo_url" = "http://hellok3s.147.75.40.34.sslip.io" + "rancher_urls" = { + "FR DEV Cluster" = { + "rancher_initial_password_base64" = "Zm9vdmFsdWU=" + "rancher_url" = "https://rancher.147.28.184.119.sslip.io" + } + } } ``` @@ -251,8 +345,18 @@ Apply complete! Resources: 3 added, 0 changed, 0 destroyed. Outputs: -k3s_api = { - "FR example" = "145.40.94.83" +clusters_output = { + "cluster_details" = { + "FR DEV Cluster" = { + "api" = "147.28.184.239" + "nodes" = { + "fr-dev-cluster-cp-aio" = { + "node_private_ipv4" = "10.25.49.1" + "node_public_ipv4" = "147.28.184.239" + } + } + } + } } ``` @@ -262,66 +366,102 @@ As the SSH key for the project has been injected, the clusters can be accessed a ```bash ( -MODULENAME="demo_cluster" IFS=$'\n' -for cluster in $(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api | keys[]"); do - IP=$(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api[\"${cluster}\"]") - ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${IP} kubectl get nodes +for cluster in $(terraform output -json | jq -r ".clusters_output.value.cluster_details | keys[]"); do + FIRSTHOST=$(terraform output -json | jq -r "first(.clusters_output.value.cluster_details[\"${cluster}\"].nodes[].node_public_ipv4)") + echo "=== ${cluster} ===" + ssh -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${FIRSTHOST} -tt 'bash -l -c "kubectl get nodes -o wide"' done ) -NAME STATUS ROLES AGE VERSION -ny-k3s-aio Ready control-plane,master 9m35s v1.26.5+k3s1 -NAME STATUS ROLES AGE VERSION -sv-k3s-aio Ready control-plane,master 10m v1.26.5+k3s +=== FR DEV Cluster === +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +fr-dev-cluster-cp-aio Ready control-plane,master 4m31s v1.29.9+k3s1 10.25.49.1 147.28.184.239 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +=== SV DEV Cluster === +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +sv-dev-cluster-cp-aio Ready control-plane,etcd,master 4m3s v1.30.3+rke2r1 10.67.31.129 139.178.70.53 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.17-k3s1 +sv-dev-cluster-node-00 Ready 2m29s v1.30.3+rke2r1 10.67.31.133 139.178.70.233 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.17-k3s1 +=== SV Production === +NAME STATUS ROLES AGE VERSION INTERNAL-IP EXTERNAL-IP OS-IMAGE KERNEL-VERSION CONTAINER-RUNTIME +sv-production-cp-0 Ready control-plane,etcd,master 2m46s v1.30.5+k3s1 10.67.31.131 139.178.70.141 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +sv-production-cp-1 Ready control-plane,etcd,master 42s v1.30.5+k3s1 10.67.31.137 136.144.54.111 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +sv-production-cp-2 Ready control-plane,etcd,master 26s v1.30.5+k3s1 10.67.31.139 136.144.54.113 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +sv-production-node-00 Ready 63s v1.30.5+k3s1 10.67.31.135 136.144.54.109 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +sv-production-node-01 Ready 59s v1.30.5+k3s1 10.67.31.141 139.178.94.11 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 +sv-production-node-02 Ready 57s v1.30.5+k3s1 10.67.31.143 139.178.94.19 Debian GNU/Linux 11 (bullseye) 5.10.0-32-amd64 containerd://1.7.21-k3s2 ``` -To access from outside, the K3s kubeconfig file can be copied to any host and replace the `server` field with the IP of the K3s API: +To access from outside, the kubeconfig file can be copied to any host and replace the `server` field with the IP of the kubernetes API: ```bash ( -MODULENAME="demo_cluster" IFS=$'\n' -for cluster in $(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api | keys[]"); do - IP=$(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api[\"${cluster}\"]") +for cluster in $(terraform output -json | jq -r ".clusters_output.value.cluster_details | keys[]"); do + FIRSTHOST=$(terraform output -json | jq -r "first(.clusters_output.value.cluster_details[\"${cluster}\"].nodes[].node_public_ipv4)") + API=$(terraform output -json | jq -r "first(.clusters_output.value.cluster_details[\"${cluster}\"].api") export KUBECONFIG="./$(echo ${cluster}| tr -c -s '[:alnum:]' '-')-kubeconfig" - scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${IP}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} - sed -i "s/127.0.0.1/${IP}/g" ${KUBECONFIG} + scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${FIRSTHOST}:/root/.kube/config ${KUBECONFIG} + sed -i "s/127.0.0.1/${API}/g" ${KUBECONFIG} chmod 600 ${KUBECONFIG} + echo "=== ${cluster} ===" kubectl get nodes done ) -NAME STATUS ROLES AGE VERSION -ny-k3s-aio Ready control-plane,master 8m41s v1.26.5+k3s1 -NAME STATUS ROLES AGE VERSION -sv-k3s-aio Ready control-plane,master 9m20s v1.26.5+k3s1 +=== FR DEV Cluster === +NAME STATUS ROLES AGE VERSION +fr-dev-cluster-cp-aio Ready control-plane,master 10m v1.29.9+k3s1 +=== SV DEV Cluster === +NAME STATUS ROLES AGE VERSION +sv-dev-cluster-cp-aio Ready control-plane,etcd,master 10m v1.30.3+rke2r1 +sv-dev-cluster-node-00 Ready 8m43s v1.30.3+rke2r1 +=== SV Production === +NAME STATUS ROLES AGE VERSION +sv-production-cp-0 Ready control-plane,etcd,master 9m v1.30.5+k3s1 +sv-production-cp-1 Ready control-plane,etcd,master 6m56s v1.30.5+k3s1 +sv-production-cp-2 Ready control-plane,etcd,master 6m40s v1.30.5+k3s1 +sv-production-node-00 Ready 7m17s v1.30.5+k3s1 +sv-production-node-01 Ready 7m13s v1.30.5+k3s1 +sv-production-node-02 Ready 7m11s v1.30.5+k3s1 ``` -> :warning: OSX sed is different, it needs to be used as `sed -i "" "s/127.0.0.1/${IP}/g" ${KUBECONFIG}` instead. +> :warning: OSX sed is different, it needs to be used as `sed -i "" "s/127.0.0.1/${API}/g" ${KUBECONFIG}` instead. ```bash ( -MODULENAME="demo_cluster" IFS=$'\n' -for cluster in $(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api | keys[]"); do - IP=$(terraform output -json | jq -r ".${MODULENAME}.value.k3s_api[\"${cluster}\"]") +for cluster in $(terraform output -json | jq -r ".clusters_output.value.cluster_details | keys[]"); do + FIRSTHOST=$(terraform output -json | jq -r "first(.clusters_output.value.cluster_details[\"${cluster}\"].nodes[].node_public_ipv4)") + API=$(terraform output -json | jq -r ".clusters_output.value.cluster_details[\"${cluster}\"].api") export KUBECONFIG="./$(echo ${cluster}| tr -c -s '[:alnum:]' '-')-kubeconfig" - scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${IP}:/etc/rancher/k3s/k3s.yaml ${KUBECONFIG} - sed -i "" "s/127.0.0.1/${IP}/g" ${KUBECONFIG} + scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no root@${FIRSTHOST}:/root/.kube/config ${KUBECONFIG} + sed -i "" "s/127.0.0.1/${API}/g" ${KUBECONFIG} chmod 600 ${KUBECONFIG} + echo "=== ${cluster} ===" kubectl get nodes done ) -NAME STATUS ROLES AGE VERSION -ny-k3s-aio Ready control-plane,master 8m41s v1.26.5+k3s1 -NAME STATUS ROLES AGE VERSION -sv-k3s-aio Ready control-plane,master 9m20s v1.26.5+k3s1 +=== FR DEV Cluster === +NAME STATUS ROLES AGE VERSION +fr-dev-cluster-cp-aio Ready control-plane,master 10m v1.29.9+k3s1 +=== SV DEV Cluster === +NAME STATUS ROLES AGE VERSION +sv-dev-cluster-cp-aio Ready control-plane,etcd,master 10m v1.30.3+rke2r1 +sv-dev-cluster-node-00 Ready 8m43s v1.30.3+rke2r1 +=== SV Production === +NAME STATUS ROLES AGE VERSION +sv-production-cp-0 Ready control-plane,etcd,master 9m v1.30.5+k3s1 +sv-production-cp-1 Ready control-plane,etcd,master 6m56s v1.30.5+k3s1 +sv-production-cp-2 Ready control-plane,etcd,master 6m40s v1.30.5+k3s1 +sv-production-node-00 Ready 7m17s v1.30.5+k3s1 +sv-production-node-01 Ready 7m13s v1.30.5+k3s1 +sv-production-node-02 Ready 7m11s v1.30.5+k3s1 ``` ## Terraform module documentation + ### Requirements @@ -335,13 +475,13 @@ sv-k3s-aio Ready control-plane,master 9m20s v1.26.5+k3s1 | Name | Version | |------|---------| -| [equinix](#provider\_equinix) | >= 1.14.2 | +| [equinix](#provider\_equinix) | 1.14.3 | ### Modules | Name | Source | Version | |------|--------|---------| -| [k3s\_cluster](#module\_k3s\_cluster) | ./modules/k3s_cluster | n/a | +| [kube\_cluster](#module\_kube\_cluster) | ./modules/kube_cluster | n/a | ### Resources @@ -353,18 +493,19 @@ sv-k3s-aio Ready control-plane,master 9m20s v1.26.5+k3s1 | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [metal\_project\_id](#input\_metal\_project\_id) | Equinix Metal Project ID | `string` | n/a | yes | -| [clusters](#input\_clusters) | K3s cluster definition | 
list(object({
    name                    = optional(string, "K3s demo cluster")
    metro                   = optional(string, "FR")
    plan_control_plane      = optional(string, "c3.small.x86")
    plan_node               = optional(string, "c3.small.x86")
    node_count              = optional(number, 0)
    k3s_ha                  = optional(bool, false)
    os                      = optional(string, "debian_11")
    control_plane_hostnames = optional(string, "k3s-cp")
    node_hostnames          = optional(string, "k3s-node")
    custom_k3s_token        = optional(string, "")
    ip_pool_count           = optional(number, 0)
    k3s_version             = optional(string, "")
    metallb_version         = optional(string, "")
  }))
| 
[
  {}
]
| no | +| [clusters](#input\_clusters) | Cluster definition | 
list(object({
    name                    = optional(string, "Demo cluster")
    metro                   = optional(string, "FR")
    plan_control_plane      = optional(string, "c3.small.x86")
    plan_node               = optional(string, "c3.small.x86")
    node_count              = optional(number, 0)
    ha                      = optional(bool, false)
    os                      = optional(string, "debian_11")
    control_plane_hostnames = optional(string, "cp")
    node_hostnames          = optional(string, "node")
    custom_token            = optional(string, "")
    ip_pool_count           = optional(number, 0)
    kube_version            = optional(string, "")
    metallb_version         = optional(string, "")
    rancher_flavor          = optional(string, "")
    rancher_version         = optional(string, "")
    custom_rancher_password = optional(string, "")
  }))
| 
[
  {}
]
| no | | [deploy\_demo](#input\_deploy\_demo) | Deploys a simple demo using a global IP as ingress and a hello-kubernetes pods | `bool` | `false` | no | | [global\_ip](#input\_global\_ip) | Enables a global anycast IPv4 that will be shared for all clusters in all metros | `bool` | `false` | no | +| [metal\_project\_id](#input\_metal\_project\_id) | Equinix Metal Project ID | `string` | n/a | yes | ### Outputs | Name | Description | |------|-------------| | [anycast\_ip](#output\_anycast\_ip) | Global IP shared across Metros | +| [cluster\_details](#output\_cluster\_details) | List of Clusters => K8s details | | [demo\_url](#output\_demo\_url) | URL of the demo application to demonstrate a global IP shared across Metros | -| [k3s\_api](#output\_k3s\_api) | List of Clusters => K3s APIs | +| [rancher\_urls](#output\_rancher\_urls) | List of Clusters => Rancher details | ## Contributing diff --git a/examples/demo_cluster/README.md b/examples/demo_cluster/README.md index 6b5f6ec..3040d1d 100644 --- a/examples/demo_cluster/README.md +++ b/examples/demo_cluster/README.md @@ -1,6 +1,6 @@ -# SiDemo Cluster Example +# Demo Cluster Examples -This example demonstrates usage of the Equinix Metal K3s module. A Demo application is installed. +This example demonstrates usage of the Equinix Metal K3s/RKE2 module. A Demo application is installed. ## Usage @@ -36,15 +36,15 @@ No resources. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [metal\_auth\_token](#input\_metal\_auth\_token) | Your Equinix Metal API key | `string` | n/a | yes | -| [metal\_project\_id](#input\_metal\_project\_id) | Your Equinix Metal Project ID | `string` | n/a | yes | -| [clusters](#input\_clusters) | K3s cluster definition | 
list(object({
    name                    = optional(string, "K3s demo cluster")
    metro                   = optional(string, "FR")
    plan_control_plane      = optional(string, "c3.small.x86")
    plan_node               = optional(string, "c3.small.x86")
    node_count              = optional(number, 0)
    k3s_ha                  = optional(bool, false)
    os                      = optional(string, "debian_11")
    control_plane_hostnames = optional(string, "k3s-cp")
    node_hostnames          = optional(string, "k3s-node")
    custom_k3s_token        = optional(string, "")
    ip_pool_count           = optional(number, 0)
    k3s_version             = optional(string, "")
    metallb_version         = optional(string, "")
  }))
| 
[
  {}
]
| no | +| [clusters](#input\_clusters) | Cluster definition | 
list(object({
    name                    = optional(string, "Demo cluster")
    metro                   = optional(string, "FR")
    plan_control_plane      = optional(string, "c3.small.x86")
    plan_node               = optional(string, "c3.small.x86")
    node_count              = optional(number, 0)
    ha                      = optional(bool, false)
    os                      = optional(string, "debian_11")
    control_plane_hostnames = optional(string, "cp")
    node_hostnames          = optional(string, "node")
    custom_token            = optional(string, "")
    ip_pool_count           = optional(number, 0)
    kube_version            = optional(string, "")
    metallb_version         = optional(string, "")
    rancher_version         = optional(string, "")
    rancher_flavor          = optional(string, "")
    custom_rancher_password = optional(string, "")
  }))
| 
[
  {}
]
| no | | [deploy\_demo](#input\_deploy\_demo) | Deploys a simple demo using a global IP as ingress and a hello-kubernetes pods | `bool` | `false` | no | | [global\_ip](#input\_global\_ip) | Enables a global anycast IPv4 that will be shared for all clusters in all metros | `bool` | `false` | no | +| [metal\_auth\_token](#input\_metal\_auth\_token) | Your Equinix Metal API key | `string` | n/a | yes | +| [metal\_project\_id](#input\_metal\_project\_id) | Your Equinix Metal Project ID | `string` | n/a | yes | ### Outputs | Name | Description | |------|-------------| -| [demo\_cluster](#output\_demo\_cluster) | Passthrough of the root module output | +| [clusters\_output](#output\_clusters\_output) | Passthrough of the root module output | diff --git a/examples/demo_cluster/outputs.tf b/examples/demo_cluster/outputs.tf index bcf7ada..8432eb1 100644 --- a/examples/demo_cluster/outputs.tf +++ b/examples/demo_cluster/outputs.tf @@ -1,4 +1,4 @@ -output "demo_cluster" { +output "clusters_output" { description = "Passthrough of the root module output" value = module.demo } diff --git a/examples/demo_cluster/terraform.tfvars.example b/examples/demo_cluster/terraform.tfvars.example index 4c7f268..a374c00 100644 --- a/examples/demo_cluster/terraform.tfvars.example +++ b/examples/demo_cluster/terraform.tfvars.example @@ -1,11 +1,26 @@ metal_auth_token="your_token_here" #This must be a user API token metal_project_id="your_project_id" -clusters = [ +clusters = [ { - name = "Your cluster name" + name = "FR DEV Cluster" + rancher_flavor = "stable" + ip_pool_count = 1 + kube_version = "v1.29.9+k3s1" }, { - name = "Your cluster name" - metro = "SV" + name = "SV DEV Cluster" + metro = "SV" + node_count = 1 + kube_version = "v1.30.3+rke2r1" + }, + { + name = "SV Production" + ip_pool_count = 4 + ha = true + metro = "SV" + node_count = 3 } ] + +global_ip = true +deploy_demo = true diff --git a/examples/demo_cluster/variables.tf b/examples/demo_cluster/variables.tf index 527256d..4bbddb0 100644 --- a/examples/demo_cluster/variables.tf +++ b/examples/demo_cluster/variables.tf @@ -22,21 +22,24 @@ variable "deploy_demo" { } variable "clusters" { - description = "K3s cluster definition" + description = "Cluster definition" type = list(object({ - name = optional(string, "K3s demo cluster") + name = optional(string, "Demo cluster") metro = optional(string, "FR") plan_control_plane = optional(string, "c3.small.x86") plan_node = optional(string, "c3.small.x86") node_count = optional(number, 0) - k3s_ha = optional(bool, false) + ha = optional(bool, false) os = optional(string, "debian_11") - control_plane_hostnames = optional(string, "k3s-cp") - node_hostnames = optional(string, "k3s-node") - custom_k3s_token = optional(string, "") + control_plane_hostnames = optional(string, "cp") + node_hostnames = optional(string, "node") + custom_token = optional(string, "") ip_pool_count = optional(number, 0) - k3s_version = optional(string, "") + kube_version = optional(string, "") metallb_version = optional(string, "") + rancher_version = optional(string, "") + rancher_flavor = optional(string, "") + custom_rancher_password = optional(string, "") })) default = [{}] } diff --git a/main.tf b/main.tf index d64c0e7..a068362 100644 --- a/main.tf +++ b/main.tf @@ -5,11 +5,11 @@ locals { } ################################################################################ -# K3S Cluster In-line Module +# K8s Cluster In-line Module ################################################################################ -module "k3s_cluster" { - source = "./modules/k3s_cluster" +module "kube_cluster" { + source = "./modules/kube_cluster" for_each = { for cluster in var.clusters : cluster.name => cluster } @@ -18,14 +18,17 @@ module "k3s_cluster" { plan_control_plane = each.value.plan_control_plane plan_node = each.value.plan_node node_count = each.value.node_count - k3s_ha = each.value.k3s_ha + ha = each.value.ha os = each.value.os control_plane_hostnames = each.value.control_plane_hostnames node_hostnames = each.value.node_hostnames - custom_k3s_token = each.value.custom_k3s_token - k3s_version = each.value.k3s_version + custom_token = each.value.custom_token + kube_version = each.value.kube_version metallb_version = each.value.metallb_version ip_pool_count = each.value.ip_pool_count + rancher_flavor = each.value.rancher_flavor + rancher_version = each.value.rancher_version + custom_rancher_password = each.value.custom_rancher_password metal_project_id = var.metal_project_id deploy_demo = var.deploy_demo global_ip_cidr = local.global_ip_cidr diff --git a/modules/k3s_cluster/outputs.tf b/modules/k3s_cluster/outputs.tf deleted file mode 100644 index 7e6cb62..0000000 --- a/modules/k3s_cluster/outputs.tf +++ /dev/null @@ -1,4 +0,0 @@ -output "k3s_api_ip" { - value = try(equinix_metal_reserved_ip_block.api_vip_addr[0].address, equinix_metal_device.all_in_one[0].network[0].address) - description = "K3s API IPs" -} diff --git a/modules/k3s_cluster/templates/user-data.tftpl b/modules/k3s_cluster/templates/user-data.tftpl deleted file mode 100644 index 0fb1ff4..0000000 --- a/modules/k3s_cluster/templates/user-data.tftpl +++ /dev/null @@ -1,388 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -wait_for_k3s_api(){ - # Wait for the node to be available, meaning the K8s API is available - while ! kubectl wait --for condition=ready node $(cat /etc/hostname | tr '[:upper:]' '[:lower:]') --timeout=60s; do sleep 2 ; done -} - -install_bird(){ - # Install bird - apt update && apt install bird jq -y - - # In order to configure bird, the metadata information is required. - # BGP info can take a few seconds to be populated, retry if that's the case - INTERNAL_IP="null" - while [ $${INTERNAL_IP} == "null" ]; do - echo "BGP data still not available..." - sleep 5 - METADATA=$(curl -s https://metadata.platformequinix.com/metadata) - INTERNAL_IP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_ip') - done - PEER_IP_1=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[0]') - PEER_IP_2=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[1]') - ASN=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_as') - ASN_AS=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_as') - MULTIHOP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].multihop') - GATEWAY=$(echo $${METADATA} | jq -r '.network.addresses[] | select(.public == true and .address_family == 4) | .gateway') - - # Generate the bird configuration based on the metadata values - # https://deploy.equinix.com/developers/guides/configuring-bgp-with-bird/ - cat <<-EOF >/etc/bird/bird.conf - router id $${INTERNAL_IP}; - - protocol direct { - interface "lo"; - } - - protocol kernel { - persist; - scan time 60; - import all; - export all; - } - - protocol device { - scan time 60; - } - - protocol static { - route $${PEER_IP_1}/32 via $${GATEWAY}; - route $${PEER_IP_2}/32 via $${GATEWAY}; - } - - filter metal_bgp { - accept; - } - - protocol bgp neighbor_v4_1 { - export filter metal_bgp; - local as $${ASN}; - multihop; - neighbor $${PEER_IP_1} as $${ASN_AS}; - } - - protocol bgp neighbor_v4_2 { - export filter metal_bgp; - local as $${ASN}; - multihop; - neighbor $${PEER_IP_2} as $${ASN_AS}; - } - EOF - - # Wait for K3s to be up, otherwise the second and third control plane nodes will try to join localhost - wait_for_k3s_api - - # Configure the BGP interface - # https://deploy.equinix.com/developers/guides/configuring-bgp-with-bird/ - if ! grep -q 'lo:0' /etc/network/interfaces; then - cat <<-EOF >>/etc/network/interfaces - - auto lo:0 - iface lo:0 inet static - address ${API_IP} - netmask 255.255.255.255 - EOF - ifup lo:0 - fi - - # Enable IP forward for bird - # TODO: Check if this is done automatically with K3s, it doesn't hurt however - echo "net.ipv4.ip_forward=1" | tee /etc/sysctl.d/99-ip-forward.conf - sysctl --load /etc/sysctl.d/99-ip-forward.conf - - # Debian usually starts the service after being installed, but just in case - systemctl enable bird - systemctl restart bird -} - -install_metallb(){ - apt update && apt install -y curl jq - -%{ if metallb_version != "" ~} - export METALLB_VERSION=${metallb_version} -%{ else ~} - export METALLB_VERSION=$(curl --silent "https://api.github.com/repos/metallb/metallb/releases/latest" | jq -r .tag_name) -%{ endif ~} - - # Wait for K3s to be up. It should be up already but just in case. - wait_for_k3s_api - - # Apply the MetalLB manifest - kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/$${METALLB_VERSION}/config/manifests/metallb-native.yaml - - # Wait for MetalLB to be up - while ! kubectl wait --for condition=ready -n metallb-system $(kubectl get pods -n metallb-system -l component=controller -o name) --timeout=10s; do sleep 2 ; done - - # In order to configure MetalLB, the metadata information is required. - # BGP info can take a few seconds to be populated, retry if that's the case - INTERNAL_IP="null" - while [ $${INTERNAL_IP} == "null" ]; do - echo "BGP data still not available..." - sleep 5 - METADATA=$(curl -s https://metadata.platformequinix.com/metadata) - INTERNAL_IP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_ip') - done - PEER_IP_1=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[0]') - PEER_IP_2=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[1]') - ASN=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_as') - ASN_AS=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_as') - -%{ if global_ip_cidr != "" ~} - # Configure the IPAddressPool for the Global IP if present - cat <<- EOF | kubectl apply -f - - apiVersion: metallb.io/v1beta1 - kind: IPAddressPool - metadata: - name: anycast-ip - namespace: metallb-system - spec: - addresses: - - ${global_ip_cidr} - autoAssign: false - EOF -%{ endif ~} - -%{ if ip_pool != "" ~} - # Configure the IPAddressPool for the IP pool if present - cat <<- EOF | kubectl apply -f - - apiVersion: metallb.io/v1beta1 - kind: IPAddressPool - metadata: - name: ippool - namespace: metallb-system - spec: - addresses: - - ${ip_pool} - autoAssign: false - EOF -%{ endif ~} - - # Configure the BGPPeer for each peer IP - cat <<- EOF | kubectl apply -f - - apiVersion: metallb.io/v1beta2 - kind: BGPPeer - metadata: - name: equinix-metal-peer-1 - namespace: metallb-system - spec: - peerASN: $${ASN_AS} - myASN: $${ASN} - peerAddress: $${PEER_IP_1} - sourceAddress: $${INTERNAL_IP} - EOF - - cat <<- EOF | kubectl apply -f - - apiVersion: metallb.io/v1beta2 - kind: BGPPeer - metadata: - name: equinix-metal-peer-1 - namespace: metallb-system - spec: - peerASN: $${ASN_AS} - myASN: $${ASN} - peerAddress: $${PEER_IP_2} - sourceAddress: $${INTERNAL_IP} - EOF - - # Enable the BGPAdvertisement, only to be executed in the control-plane nodes - cat <<- EOF | kubectl apply -f - - apiVersion: metallb.io/v1beta1 - kind: BGPAdvertisement - metadata: - name: bgp-peers - namespace: metallb-system - spec: - nodeSelectors: - - matchLabels: - node-role.kubernetes.io/control-plane: "true" - EOF -} - -install_k3s(){ - # Curl is needed to download the k3s binary - # Jq is needed to parse the Equinix Metal metadata (json format) - apt update && apt install curl jq -y - - # Download the K3s installer script - curl -L --output k3s_installer.sh https://get.k3s.io && install -m755 k3s_installer.sh /usr/local/bin/ - -%{ if node_type == "control-plane" ~} - # If the node to be installed is the second or third control plane or extra nodes, wait for the API to be up - # Wait for the first control plane node to be up - while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done -%{ endif ~} -%{ if node_type == "node" ~} - # Wait for the first control plane node to be up - while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done -%{ endif ~} - - export INSTALL_K3S_SKIP_START=false - export K3S_TOKEN="${k3s_token}" - export NODE_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) |.address') - export NODE_EXTERNAL_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == true and .address_family == 4) |.address') -%{ if node_type == "all-in-one" ~} -%{ if global_ip_cidr != "" ~} - export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ else ~} -%{ if ip_pool != "" ~} - export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ else ~} - export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ endif ~} -%{ endif ~} -%{ endif ~} -%{ if node_type == "control-plane-master" ~} - export INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644 --tls-san=${API_IP} --tls-san=${API_IP}.sslip.io --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ endif ~} -%{ if node_type == "control-plane" ~} - export INSTALL_K3S_EXEC="server --server https://${API_IP}:6443 --write-kubeconfig-mode=644 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ endif ~} -%{ if node_type == "node" ~} - export INSTALL_K3S_EXEC="agent --server https://${API_IP}:6443 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" -%{ endif ~} -%{ if k3s_version != "" ~} - export INSTALL_K3S_VERSION=${k3s_version} -%{ endif ~} - /usr/local/bin/k3s_installer.sh - - systemctl enable --now k3s -} - -deploy_demo(){ - kubectl annotate svc -n kube-system traefik "metallb.universe.tf/address-pool=anycast-ip" - - # I cannot make split work in Terraform templates - IP=$(echo ${global_ip_cidr} | cut -d/ -f1) - cat <<- EOF | kubectl apply -f - - --- - apiVersion: v1 - kind: Namespace - metadata: - name: hello-kubernetes - --- - apiVersion: v1 - kind: ServiceAccount - metadata: - name: hello-kubernetes - namespace: hello-kubernetes - labels: - app.kubernetes.io/name: hello-kubernetes - --- - apiVersion: v1 - kind: Service - metadata: - name: hello-kubernetes - namespace: hello-kubernetes - labels: - app.kubernetes.io/name: hello-kubernetes - spec: - type: ClusterIP - ports: - - port: 80 - targetPort: http - protocol: TCP - name: http - selector: - app.kubernetes.io/name: hello-kubernetes - --- - apiVersion: apps/v1 - kind: Deployment - metadata: - name: hello-kubernetes - namespace: hello-kubernetes - labels: - app.kubernetes.io/name: hello-kubernetes - spec: - replicas: 2 - selector: - matchLabels: - app.kubernetes.io/name: hello-kubernetes - template: - metadata: - labels: - app.kubernetes.io/name: hello-kubernetes - spec: - serviceAccountName: hello-kubernetes - containers: - - name: hello-kubernetes - image: "paulbouwer/hello-kubernetes:1.10" - imagePullPolicy: IfNotPresent - ports: - - name: http - containerPort: 8080 - protocol: TCP - livenessProbe: - httpGet: - path: / - port: http - readinessProbe: - httpGet: - path: / - port: http - env: - - name: HANDLER_PATH_PREFIX - value: "" - - name: RENDER_PATH_PREFIX - value: "" - - name: KUBERNETES_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - - name: KUBERNETES_POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: KUBERNETES_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: CONTAINER_IMAGE - value: "paulbouwer/hello-kubernetes:1.10" - --- - apiVersion: networking.k8s.io/v1 - kind: Ingress - metadata: - name: hello-kubernetes-ingress - namespace: hello-kubernetes - spec: - rules: - - host: hellok3s.$${IP}.sslip.io - http: - paths: - - path: "/" - pathType: Prefix - backend: - service: - name: hello-kubernetes - port: - name: http - EOF -} - -install_k3s - -%{ if node_type == "control-plane-master" ~} -install_bird -install_metallb -%{ endif ~} -%{ if node_type == "control-plane" ~} -install_bird -install_metallb -%{ endif ~} - -%{ if node_type == "all-in-one" ~} -%{ if global_ip_cidr != "" ~} -INSTALL_METALLB=true -%{ else } -%{ if ip_pool != "" ~} -INSTALL_METALLB=true -%{ else } -INSTALL_METALLB=false -%{ endif ~} -%{ endif ~} -[ $${INSTALL_METALLB} == true ] && install_metallb || true -%{ endif ~} -%{ if deploy_demo != "" ~} -deploy_demo -%{ endif ~} diff --git a/modules/k3s_cluster/README.md b/modules/kube_cluster/README.md similarity index 62% rename from modules/k3s_cluster/README.md rename to modules/kube_cluster/README.md index b40cf58..444cb79 100644 --- a/modules/k3s_cluster/README.md +++ b/modules/kube_cluster/README.md @@ -1,6 +1,6 @@ -# K3S Cluster In-line Module +# K3s/RKE2 Cluster In-line Module -This in-line module deploys the K3S cluster. +This in-line module deploys the K3s/RKE2 cluster. ## Notes @@ -10,10 +10,6 @@ This in-line module deploys the K3S cluster. See [this](https://discuss.hashicorp.com/t/invalid-value-for-vars-parameter-vars-map-does-not-contain-key-issue/12074/4) and [this](https://github.com/hashicorp/terraform/issues/23384) for more information. -* The loopback interface for API LB cannot be up until K3s is fully installed in the extra control plane nodes - - Otherwise they will try to join themselves... that's why there is a curl to the K3s API that waits for the first master to be up before trying to install K3s and also why the bird configuration happens after K3s is up and running in the other nodes. - * ServiceLB disabled `--disable servicelb` is required for metallb to work @@ -30,8 +26,8 @@ This in-line module deploys the K3S cluster. | Name | Version | |------|---------| -| [equinix](#provider\_equinix) | >= 1.14.2 | -| [random](#provider\_random) | >= 3.5.1 | +| [equinix](#provider\_equinix) | 2.5.0 | +| [random](#provider\_random) | 3.6.3 | ### Modules @@ -50,33 +46,43 @@ No modules. | [equinix_metal_device.control_plane_others](https://registry.terraform.io/providers/equinix/equinix/latest/docs/resources/metal_device) | resource | | [equinix_metal_device.nodes](https://registry.terraform.io/providers/equinix/equinix/latest/docs/resources/metal_device) | resource | | [equinix_metal_reserved_ip_block.api_vip_addr](https://registry.terraform.io/providers/equinix/equinix/latest/docs/resources/metal_reserved_ip_block) | resource | +| [equinix_metal_reserved_ip_block.ingress_addr](https://registry.terraform.io/providers/equinix/equinix/latest/docs/resources/metal_reserved_ip_block) | resource | | [equinix_metal_reserved_ip_block.ip_pool](https://registry.terraform.io/providers/equinix/equinix/latest/docs/resources/metal_reserved_ip_block) | resource | -| [random_string.random_k3s_token](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [random_string.random_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | +| [random_string.random_token](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource | ### Inputs | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [metal\_metro](#input\_metal\_metro) | Equinix Metal Metro | `string` | n/a | yes | -| [metal\_project\_id](#input\_metal\_project\_id) | Equinix Metal Project ID | `string` | n/a | yes | -| [cluster\_name](#input\_cluster\_name) | Cluster name | `string` | `"K3s cluster"` | no | +| [cluster\_name](#input\_cluster\_name) | Cluster name | `string` | `"Cluster"` | no | | [control\_plane\_hostnames](#input\_control\_plane\_hostnames) | Control plane hostname prefix | `string` | `"cp"` | no | -| [custom\_k3s\_token](#input\_custom\_k3s\_token) | K3s token used for nodes to join the cluster (autogenerated otherwise) | `string` | `null` | no | +| [custom\_rancher\_password](#input\_custom\_rancher\_password) | Rancher initial password (autogenerated if not provided) | `string` | `null` | no | +| [custom\_token](#input\_custom\_token) | Token used for nodes to join the cluster (autogenerated otherwise) | `string` | `null` | no | | [deploy\_demo](#input\_deploy\_demo) | Deploys a simple demo using a global IP as ingress and a hello-kubernetes pods | `bool` | `false` | no | | [global\_ip\_cidr](#input\_global\_ip\_cidr) | Global Anycast IP that will be mapped on all metros via BGP | `string` | `null` | no | -| [ip\_pool\_count](#input\_ip\_pool\_count) | Number of public IPv4 per metro to be used as LoadBalancers with MetalLB | `number` | `0` | no | -| [k3s\_ha](#input\_k3s\_ha) | K3s HA (aka 3 control plane nodes) | `bool` | `false` | no | -| [k3s\_version](#input\_k3s\_version) | K3s version to be installed. Empty for latest | `string` | `""` | no | +| [ha](#input\_ha) | HA (aka 3 control plane nodes) | `bool` | `false` | no | +| [ip\_pool\_count](#input\_ip\_pool\_count) | Number of public IPv4 per metro to be used as LoadBalancers with MetalLB (it needs to be power of 2 between 0 and 256 as required by Equinix Metal) | `number` | `0` | no | +| [kube\_version](#input\_kube\_version) | K3s/RKE2 version to be installed. Empty for latest K3s | `string` | `""` | no | +| [metal\_metro](#input\_metal\_metro) | Equinix Metal Metro | `string` | n/a | yes | +| [metal\_project\_id](#input\_metal\_project\_id) | Equinix Metal Project ID | `string` | n/a | yes | | [metallb\_version](#input\_metallb\_version) | MetalLB version to be installed. Empty for latest | `string` | `""` | no | -| [node\_count](#input\_node\_count) | Number of K3s nodes | `number` | `"0"` | no | +| [node\_count](#input\_node\_count) | Number of nodes | `number` | `"0"` | no | | [node\_hostnames](#input\_node\_hostnames) | Node hostname prefix | `string` | `"node"` | no | | [os](#input\_os) | Operating system | `string` | `"debian_11"` | no | -| [plan\_control\_plane](#input\_plan\_control\_plane) | K3s control plane type/size | `string` | `"c3.small.x86"` | no | -| [plan\_node](#input\_plan\_node) | K3s node type/size | `string` | `"c3.small.x86"` | no | +| [plan\_control\_plane](#input\_plan\_control\_plane) | Control plane type/size | `string` | `"c3.small.x86"` | no | +| [plan\_node](#input\_plan\_node) | Node type/size | `string` | `"c3.small.x86"` | no | +| [rancher\_flavor](#input\_rancher\_flavor) | Rancher flavor to be installed (prime, latest, stable or alpha). Empty to not install it | `string` | `""` | no | +| [rancher\_version](#input\_rancher\_version) | Rancher version to be installed (vX.Y.Z). Empty for latest | `string` | `""` | no | ### Outputs | Name | Description | |------|-------------| -| [k3s\_api\_ip](#output\_k3s\_api\_ip) | K3s API IPs | +| [ingress\_ip](#output\_ingress\_ip) | Ingress IP | +| [ip\_pool\_cidr](#output\_ip\_pool\_cidr) | IP Pool for LoadBalancer SVCs | +| [kube\_api\_ip](#output\_kube\_api\_ip) | K8s API IPs | +| [nodes\_details](#output\_nodes\_details) | Nodes external and internal IPs | +| [rancher\_address](#output\_rancher\_address) | Rancher URL | +| [rancher\_password](#output\_rancher\_password) | Rancher initial password | diff --git a/modules/k3s_cluster/main.tf b/modules/kube_cluster/main.tf similarity index 68% rename from modules/k3s_cluster/main.tf rename to modules/kube_cluster/main.tf index b9906fa..15119b8 100644 --- a/modules/k3s_cluster/main.tf +++ b/modules/kube_cluster/main.tf @@ -1,10 +1,17 @@ locals { - k3s_token = coalesce(var.custom_k3s_token, random_string.random_k3s_token.result) - api_vip = var.k3s_ha ? equinix_metal_reserved_ip_block.api_vip_addr[0].address : equinix_metal_device.all_in_one[0].network[0].address + token = coalesce(var.custom_token, random_string.random_token.result) + rancher_pass = var.custom_rancher_password != null ? coalesce(var.custom_rancher_password, random_string.random_password.result) : null + api_vip = var.ha ? equinix_metal_reserved_ip_block.api_vip_addr[0].address : equinix_metal_device.all_in_one[0].network[0].address + ingress_ip = var.ip_pool_count > 0 ? equinix_metal_reserved_ip_block.ingress_addr[0].address : "" ip_pool_cidr = var.ip_pool_count > 0 ? equinix_metal_reserved_ip_block.ip_pool[0].cidr_notation : "" } -resource "random_string" "random_k3s_token" { +resource "random_string" "random_token" { + length = 16 + special = false +} + +resource "random_string" "random_password" { length = 16 special = false } @@ -20,32 +27,45 @@ resource "equinix_metal_device" "control_plane_master" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 description = var.cluster_name user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, + ingress_ip = local.ingress_ip, global_ip_cidr = var.global_ip_cidr, ip_pool = local.ip_pool_cidr, - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = var.deploy_demo, + rancher_flavor = var.rancher_flavor, + rancher_version = var.rancher_version, + rancher_pass = local.rancher_pass, node_type = "control-plane-master" }) } resource "equinix_metal_bgp_session" "control_plane_master" { device_id = equinix_metal_device.control_plane_master[0].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } resource "equinix_metal_reserved_ip_block" "api_vip_addr" { - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 + project_id = var.metal_project_id + metro = var.metal_metro + type = "public_ipv4" + quantity = 1 + description = "Kubernetes API IP for the ${var.cluster_name} cluster" +} + +resource "equinix_metal_reserved_ip_block" "ingress_addr" { + count = var.ip_pool_count > 0 ? 1 : 0 project_id = var.metal_project_id metro = var.metal_metro type = "public_ipv4" quantity = 1 - description = "K3s API IP" + description = "Ingress IP for the ${var.cluster_name} cluster" } resource "equinix_metal_device" "control_plane_others" { @@ -55,16 +75,20 @@ resource "equinix_metal_device" "control_plane_others" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 2 : 0 + count = var.ha ? 2 : 0 description = var.cluster_name depends_on = [equinix_metal_device.control_plane_master] user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, + ingress_ip = local.ingress_ip, global_ip_cidr = "", ip_pool = "", - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, + rancher_flavor = var.rancher_flavor, + rancher_version = var.rancher_version, + rancher_pass = local.rancher_pass, deploy_demo = false, node_type = "control-plane" }) } @@ -72,13 +96,13 @@ resource "equinix_metal_device" "control_plane_others" { resource "equinix_metal_bgp_session" "control_plane_second" { device_id = equinix_metal_device.control_plane_others[0].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } resource "equinix_metal_bgp_session" "control_plane_third" { device_id = equinix_metal_device.control_plane_others[1].id address_family = "ipv4" - count = var.k3s_ha ? 1 : 0 + count = var.ha ? 1 : 0 } ################################################################################ @@ -91,7 +115,7 @@ resource "equinix_metal_reserved_ip_block" "ip_pool" { quantity = var.ip_pool_count metro = var.metal_metro count = var.ip_pool_count > 0 ? 1 : 0 - description = "IP Pool to be used for LoadBalancers via MetalLB" + description = "IP Pool to be used for LoadBalancers via MetalLB on the ${var.cluster_name} cluster" } ################################################################################ @@ -109,12 +133,16 @@ resource "equinix_metal_device" "nodes" { description = var.cluster_name depends_on = [equinix_metal_device.control_plane_master] user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, API_IP = local.api_vip, + ingress_ip = local.ingress_ip, global_ip_cidr = "", ip_pool = "", - k3s_version = var.k3s_version, + kube_version = var.kube_version, metallb_version = var.metallb_version, + rancher_flavor = var.rancher_flavor, + rancher_version = var.rancher_version, + rancher_pass = local.rancher_pass, deploy_demo = false, node_type = "node" }) } @@ -130,16 +158,20 @@ resource "equinix_metal_device" "all_in_one" { operating_system = var.os billing_cycle = "hourly" project_id = var.metal_project_id - count = var.k3s_ha ? 0 : 1 + count = var.ha ? 0 : 1 description = var.cluster_name user_data = templatefile("${path.module}/templates/user-data.tftpl", { - k3s_token = local.k3s_token, + token = local.token, global_ip_cidr = var.global_ip_cidr, ip_pool = local.ip_pool_cidr, API_IP = "", - k3s_version = var.k3s_version, + ingress_ip = local.ingress_ip, + kube_version = var.kube_version, metallb_version = var.metallb_version, deploy_demo = var.deploy_demo, + rancher_flavor = var.rancher_flavor, + rancher_version = var.rancher_version, + rancher_pass = local.rancher_pass, node_type = "all-in-one" }) } @@ -147,5 +179,5 @@ resource "equinix_metal_device" "all_in_one" { resource "equinix_metal_bgp_session" "all_in_one" { device_id = equinix_metal_device.all_in_one[0].id address_family = "ipv4" - count = var.k3s_ha ? 0 : 1 + count = var.ha ? 0 : 1 } diff --git a/modules/kube_cluster/outputs.tf b/modules/kube_cluster/outputs.tf new file mode 100644 index 0000000..11050d5 --- /dev/null +++ b/modules/kube_cluster/outputs.tf @@ -0,0 +1,34 @@ +output "kube_api_ip" { + value = local.api_vip + description = "K8s API IPs" +} + +output "rancher_address" { + value = var.rancher_flavor != "" ? "https://rancher.${local.ingress_ip}.sslip.io" : null + description = "Rancher URL" +} + +output "rancher_password" { + value = var.rancher_flavor != "" ? local.rancher_pass : null + description = "Rancher initial password" +} + +output "ingress_ip" { + value = var.ip_pool_count > 0 ? local.ingress_ip : null + description = "Ingress IP" +} + +output "ip_pool_cidr" { + value = var.ip_pool_count > 0 ? local.ip_pool_cidr : null + description = "IP Pool for LoadBalancer SVCs" +} + +output "nodes_details" { + value = { + for node in flatten([equinix_metal_device.control_plane_master, equinix_metal_device.control_plane_others, equinix_metal_device.nodes, equinix_metal_device.all_in_one]) : node.hostname => { + node_private_ipv4 = node.access_private_ipv4 + node_public_ipv4 = node.access_public_ipv4 + } + } + description = "Nodes external and internal IPs" +} diff --git a/modules/kube_cluster/templates/user-data.tftpl b/modules/kube_cluster/templates/user-data.tftpl new file mode 100644 index 0000000..95c0fb3 --- /dev/null +++ b/modules/kube_cluster/templates/user-data.tftpl @@ -0,0 +1,707 @@ +#!/usr/bin/env bash +set -euo pipefail + +die(){ + echo $${1} >&2 + exit $${2} +} + +prechecks(){ + # Set OS + source /etc/os-release + case $${ID} in + "debian") + export PKGMANAGER="apt" + ;; + "sles") + export PKGMANAGER="zypper" + ;; + "sle-micro") + export PKGMANAGER="transactional-update" + ;; + *) + die "Unsupported OS $${ID}" 1 + ;; + esac + # Set ARCH + ARCH=$(uname -m) + case $${ARCH} in + "amd64") + export ARCH=amd64 + export SUFFIX= + ;; + "x86_64") + export ARCH=amd64 + export SUFFIX= + ;; + "arm64") + export ARCH=arm64 + export SUFFIX=-$${ARCH} + ;; + "s390x") + export ARCH=s390x + export SUFFIX=-$${ARCH} + ;; + "aarch64") + export ARCH=arm64 + export SUFFIX=-$${ARCH} + ;; + "arm*") + export ARCH=arm + export SUFFIX=-$${ARCH}hf + ;; + *) + die "Unsupported architecture $${ARCH}" 1 + ;; + esac +} + +prereqs(){ + # Required packages + case $${PKGMANAGER} in + "apt") + apt update + apt install -y jq curl + ;; + "zypper") + zypper refresh + zypper install -y jq curl + ;; + esac +} + +wait_for_kube_api(){ + # Wait for the node to be available, meaning the K8s API is available + while ! kubectl wait --for condition=ready node $(cat /etc/hostname | tr '[:upper:]' '[:lower:]') --timeout=60s; do sleep 2 ; done +} + +install_eco(){ + # Wait for K3s to be up. It should be up already but just in case. + wait_for_kube_api + + # Download helm as required to install endpoint-copier-operator + command -v helm || curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 |bash + + # Add the SUSE Edge charts and deploy ECO + helm repo add suse-edge https://suse-edge.github.io/charts + helm repo update + helm install --create-namespace -n endpoint-copier-operator endpoint-copier-operator suse-edge/endpoint-copier-operator + + # Configure the MetalLB IP Address pool for the VIP + cat <<-EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: kubernetes-vip-ip-pool + namespace: metallb-system + spec: + addresses: + - ${API_IP}/32 + serviceAllocation: + priority: 100 + namespaces: + - default + EOF + + # Create the kubernetes-vip service that will be updated by e-c-o with the control plane hosts + if [[ $${KUBETYPE} == "k3s" ]]; then + cat <<-EOF | kubectl apply -f - + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-vip + namespace: default + spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: k8s-api + port: 6443 + protocol: TCP + targetPort: 6443 + type: LoadBalancer + EOF + fi + if [[ $${KUBETYPE} == "rke2" ]]; then + cat <<-EOF | kubectl apply -f - + apiVersion: v1 + kind: Service + metadata: + name: kubernetes-vip + namespace: default + spec: + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - name: k8s-api + port: 6443 + protocol: TCP + targetPort: 6443 + - name: rke2-api + port: 9345 + protocol: TCP + targetPort: 9345 + type: LoadBalancer + EOF + fi +} + +install_metallb(){ +%{ if metallb_version != "" ~} + export METALLB_VERSION=${metallb_version} +%{ else ~} + export METALLB_VERSION=$(curl --silent "https://api.github.com/repos/metallb/metallb/releases/latest" | jq -r .tag_name) +%{ endif ~} + + # Wait for K3s to be up. It should be up already but just in case. + wait_for_kube_api + + # Apply the MetalLB manifest + kubectl apply -f https://raw.githubusercontent.com/metallb/metallb/$${METALLB_VERSION}/config/manifests/metallb-native.yaml + + # Wait for MetalLB to be up + while ! kubectl wait --for condition=ready -n metallb-system $(kubectl get pods -n metallb-system -l component=controller -o name) --timeout=10s; do sleep 2 ; done + + # In order to configure MetalLB, the metadata information is required. + # BGP info can take a few seconds to be populated, retry if that's the case + INTERNAL_IP="null" + while [ $${INTERNAL_IP} == "null" ]; do + echo "BGP data still not available..." + sleep 5 + METADATA=$(curl -s https://metadata.platformequinix.com/metadata) + INTERNAL_IP=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_ip') + done + PEER_IP_1=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[0]') + PEER_IP_2=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_ips[1]') + ASN=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].customer_as') + ASN_AS=$(echo $${METADATA} | jq -r '.bgp_neighbors[0].peer_as') + +%{ if global_ip_cidr != "" ~} + # Configure the IPAddressPool for the Global IP if present + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: anycast-ip + namespace: metallb-system + spec: + addresses: + - ${global_ip_cidr} + autoAssign: true + avoidBuggyIPs: false + serviceAllocation: + namespaces: + - ingress-nginx-global + priority: 100 + serviceSelectors: + - matchExpressions: + - key: ingress-type + operator: In + values: + - ingress-nginx-global + EOF +%{ endif ~} + +%{ if ingress_ip != "" ~} + if [ "$${KUBETYPE}" == "k3s" ]; then + # Configure an IPAddressPool for Ingress only + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: ingress + namespace: metallb-system + spec: + addresses: + - ${ingress_ip}/32 + serviceAllocation: + priority: 100 + serviceSelectors: + - matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [traefik]} + EOF + fi + if [ "$${KUBETYPE}" == "rke2" ]; then + # Configure an IPAddressPool for Ingress only + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: ingress + namespace: metallb-system + spec: + addresses: + - ${ingress_ip}/32 + serviceAllocation: + priority: 100 + serviceSelectors: + - matchExpressions: + - {key: app.kubernetes.io/name, operator: In, values: [rke2-ingress-nginx]} + EOF + fi +%{ endif ~} + +%{ if ip_pool != "" ~} + # Configure the IPAddressPool for the IP pool if present + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: IPAddressPool + metadata: + name: ippool + namespace: metallb-system + spec: + addresses: + - ${ip_pool} + autoAssign: false + EOF +%{ endif ~} + + # Configure the BGPPeer for each peer IP + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta2 + kind: BGPPeer + metadata: + name: equinix-metal-peer-1 + namespace: metallb-system + spec: + peerASN: $${ASN_AS} + myASN: $${ASN} + peerAddress: $${PEER_IP_1} + sourceAddress: $${INTERNAL_IP} + EOF + + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta2 + kind: BGPPeer + metadata: + name: equinix-metal-peer-1 + namespace: metallb-system + spec: + peerASN: $${ASN_AS} + myASN: $${ASN} + peerAddress: $${PEER_IP_2} + sourceAddress: $${INTERNAL_IP} + EOF + + # Enable the BGPAdvertisement, only to be executed in the control-plane nodes + cat <<- EOF | kubectl apply -f - + apiVersion: metallb.io/v1beta1 + kind: BGPAdvertisement + metadata: + name: bgp-peers + namespace: metallb-system + spec: + nodeSelectors: + - matchLabels: + node-role.kubernetes.io/control-plane: "true" + EOF +} + +install_k3s(){ + # Download the K3s installer script + curl -L --output k3s_installer.sh https://get.k3s.io && install -m755 k3s_installer.sh /usr/local/bin/ + +%{ if node_type == "control-plane" ~} + # If the node to be installed is the second or third control plane or extra nodes, wait for the API to be up + # Wait for the first control plane node to be up + while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done +%{ endif ~} +%{ if node_type == "node" ~} + # Wait for the first control plane node to be up + while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done +%{ endif ~} + + export INSTALL_K3S_SKIP_ENABLE=false + export INSTALL_K3S_SKIP_START=false + export K3S_TOKEN="${token}" + export NODE_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) |.address') + export NODE_EXTERNAL_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == true and .address_family == 4) |.address') +%{ if node_type == "all-in-one" ~} +%{ if global_ip_cidr != "" ~} + export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ else ~} +%{ if ip_pool != "" ~} + export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ else ~} + export INSTALL_K3S_EXEC="server --write-kubeconfig-mode=644 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ endif ~} +%{ endif ~} +%{ endif ~} +%{ if node_type == "control-plane-master" ~} + export INSTALL_K3S_EXEC="server --cluster-init --write-kubeconfig-mode=644 --tls-san=${API_IP} --tls-san=${API_IP}.sslip.io --disable=servicelb --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ endif ~} +%{ if node_type == "control-plane" ~} + export INSTALL_K3S_EXEC="server --server https://${API_IP}:6443 --write-kubeconfig-mode=644 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ endif ~} +%{ if node_type == "node" ~} + export INSTALL_K3S_EXEC="agent --server https://${API_IP}:6443 --node-ip $${NODE_IP} --node-external-ip $${NODE_EXTERNAL_IP}" +%{ endif ~} +%{ if kube_version != "" ~} + export INSTALL_K3S_VERSION="${kube_version}" +%{ endif ~} + /usr/local/bin/k3s_installer.sh +} + +install_rke2(){ + # Download the RKE2 installer script + curl -L --output rke2_installer.sh https://get.rke2.io && install -m755 rke2_installer.sh /usr/local/bin/ + + # RKE2 configuration is set via config.yaml file + mkdir -p /etc/rancher/rke2/ + +%{ if node_type == "control-plane" ~} + # If the node to be installed is the second or third control plane or extra nodes, wait for the API to be up + # Wait for the first control plane node to be up + while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done +%{ endif ~} +%{ if node_type == "node" ~} + # Wait for the first control plane node to be up + while ! curl -m 10 -s -k -o /dev/null https://${API_IP}:6443 ; do echo "API still not reachable"; sleep 2 ; done +%{ endif ~} + + export RKE2_TOKEN="${token}" + export NODE_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == false and .address_family == 4) |.address') + export NODE_EXTERNAL_IP=$(curl -s https://metadata.platformequinix.com/metadata | jq -r '.network.addresses[] | select(.public == true and .address_family == 4) |.address') +%{ if node_type == "all-in-one" ~} + export INSTALL_RKE2_TYPE="server" + cat <<- EOF >> /etc/rancher/rke2/config.yaml + token: $${RKE2_TOKEN} + write-kubeconfig-mode: "0644" + node-ip: $${NODE_IP} + node-external-ip: $${NODE_EXTERNAL_IP} + EOF +%{ endif ~} +%{ if node_type == "control-plane-master" ~} + export INSTALL_RKE2_TYPE="server" + cat <<- EOF >> /etc/rancher/rke2/config.yaml + token: $${RKE2_TOKEN} + write-kubeconfig-mode: "0644" + node-ip: $${NODE_IP} + node-external-ip: $${NODE_EXTERNAL_IP} + tls-san: + - "${API_IP}" + - "${API_IP}.sslip.io" + EOF +%{ endif ~} +%{ if node_type == "control-plane" ~} + export INSTALL_RKE2_TYPE="server" + cat <<- EOF >> /etc/rancher/rke2/config.yaml + server: https://${API_IP}:9345 + token: $${RKE2_TOKEN} + write-kubeconfig-mode: "0644" + node-ip: $${NODE_IP} + node-external-ip: $${NODE_EXTERNAL_IP} + EOF +%{ endif ~} +%{ if node_type == "node" ~} + export INSTALL_RKE2_TYPE="agent" + cat <<- EOF >> /etc/rancher/rke2/config.yaml + server: https://${API_IP}:9345 + token: $${RKE2_TOKEN} + write-kubeconfig-mode: "0644" + node-ip: $${NODE_IP} + node-external-ip: $${NODE_EXTERNAL_IP} + EOF +%{ endif ~} +%{ if ingress_ip != "" ~} + mkdir -p /var/lib/rancher/rke2/server/manifests/ + cat <<- EOF >> /var/lib/rancher/rke2/server/manifests/rke2-ingress-config.yaml + apiVersion: helm.cattle.io/v1 + kind: HelmChartConfig + metadata: + name: rke2-ingress-nginx + namespace: kube-system + spec: + valuesContent: |- + controller: + config: + use-forwarded-headers: "true" + enable-real-ip: "true" + publishService: + enabled: true + service: + enabled: true + type: LoadBalancer + externalTrafficPolicy: Local + EOF +%{ endif ~} +%{ if kube_version != "" ~} + export INSTALL_RKE2_VERSION="${kube_version}" +%{ endif ~} + /usr/local/bin/rke2_installer.sh + systemctl enable --now rke2-$${INSTALL_RKE2_TYPE} +} + +deploy_demo(){ + # Check if the demo is already deployed + if kubectl get deployment -n hello-kubernetes hello-kubernetes -o name > /dev/null 2>&1; then exit 0; fi + + if [ "$${KUBETYPE}" == "rke2" ]; then + # Wait for the rke2-ingress-nginx-controller DS to be available if using RKE2 + while ! kubectl rollout status daemonset -n kube-system rke2-ingress-nginx-controller --timeout=60s; do sleep 2 ; done + fi + # I cannot make split work in Terraform templates + IP=$(echo ${global_ip_cidr} | cut -d/ -f1) + cat <<- EOF | kubectl apply -f - + --- + apiVersion: v1 + kind: Namespace + metadata: + name: hello-kubernetes + --- + apiVersion: v1 + kind: ServiceAccount + metadata: + name: hello-kubernetes + namespace: hello-kubernetes + labels: + app.kubernetes.io/name: hello-kubernetes + --- + apiVersion: v1 + kind: Service + metadata: + name: hello-kubernetes + namespace: hello-kubernetes + labels: + app.kubernetes.io/name: hello-kubernetes + spec: + type: ClusterIP + ports: + - port: 80 + targetPort: http + protocol: TCP + name: http + selector: + app.kubernetes.io/name: hello-kubernetes + --- + apiVersion: apps/v1 + kind: Deployment + metadata: + name: hello-kubernetes + namespace: hello-kubernetes + labels: + app.kubernetes.io/name: hello-kubernetes + spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: hello-kubernetes + template: + metadata: + labels: + app.kubernetes.io/name: hello-kubernetes + spec: + serviceAccountName: hello-kubernetes + containers: + - name: hello-kubernetes + image: "paulbouwer/hello-kubernetes:1.10" + imagePullPolicy: IfNotPresent + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + env: + - name: HANDLER_PATH_PREFIX + value: "" + - name: RENDER_PATH_PREFIX + value: "" + - name: KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBERNETES_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: KUBERNETES_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CONTAINER_IMAGE + value: "paulbouwer/hello-kubernetes:1.10" + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: hello-kubernetes-ingress + namespace: hello-kubernetes + spec: + ingressClassName: ingress-nginx-global + rules: + - host: hellok3s.$${IP}.sslip.io + http: + paths: + - path: "/" + pathType: Prefix + backend: + service: + name: hello-kubernetes + port: + name: http + EOF +} + +install_rancher(){ + # Wait for Kube API to be up. It should be up already but just in case. + wait_for_kube_api + + # Download helm as required to install Rancher + command -v helm || curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 |bash + + # Get latest Cert-manager version + CMVERSION=$(curl -s "https://api.github.com/repos/cert-manager/cert-manager/releases/latest" | jq -r '.tag_name') + + RANCHERFLAVOR=${rancher_flavor} + # https://ranchermanager.docs.rancher.com/pages-for-subheaders/install-upgrade-on-a-kubernetes-cluster + case $${RANCHERFLAVOR} in + "latest" | "stable" | "alpha") + helm repo add rancher https://releases.rancher.com/server-charts/$${RANCHERFLAVOR} + ;; + "prime") + helm repo add rancher https://charts.rancher.com/server-charts/prime + ;; + *) + echo "Rancher flavor not detected, using latest" + helm repo add rancher https://releases.rancher.com/server-charts/latest + ;; + esac + + helm repo add jetstack https://charts.jetstack.io + helm repo update + + # Install the cert-manager Helm chart + helm install cert-manager jetstack/cert-manager \ + --namespace cert-manager \ + --create-namespace \ + --set crds.enabled=true \ + --version $${CMVERSION} + + IP="" + # https://github.com/rancher/rke2/issues/3958 + if [ "$${KUBETYPE}" == "rke2" ]; then + # Wait for the rke2-ingress-nginx-controller DS to be available if using RKE2 + while ! kubectl rollout status daemonset -n kube-system rke2-ingress-nginx-controller --timeout=60s; do sleep 2 ; done + IP=$(kubectl get svc -n kube-system rke2-ingress-nginx-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + fi + + # Get the IP of the ingress object if provided + if [ "$${KUBETYPE}" == "k3s" ]; then + IP=$(kubectl get svc -n kube-system traefik -o jsonpath='{.status.loadBalancer.ingress[0].ip}') + fi + + if [[ $${IP} == "" ]]; then + # Just use internal IPs + IP=$(hostname -I | awk '{print $1}') + fi + + # Install rancher using sslip.io as hostname and with just a single replica + helm install rancher rancher/rancher \ + --namespace cattle-system \ + --create-namespace \ + --set hostname=rancher.$${IP}.sslip.io \ + --set bootstrapPassword="${rancher_pass}" \ + --set replicas=1 \ + --set global.cattle.psp.enabled=false %{ if rancher_version != "" ~}--version "${rancher_version}"%{ endif ~} + + while ! kubectl wait --for condition=ready -n cattle-system $(kubectl get pods -n cattle-system -l app=rancher -o name) --timeout=10s; do sleep 2 ; done +} + +install_global_ingress(){ + command -v helm || curl -fsSL https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 |bash + + cat <<- EOF > ingress-nginx-global.yaml + controller: + ingressClassResource: + name: ingress-nginx-global + controllerValue: k8s.io/ingress-nginx-global + service: + labels: + ingress-type: ingress-nginx-global + admissionWebhooks: + enabled: false + EOF + + helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx + helm repo update + helm install -f ingress-nginx-global.yaml ingress-nginx-global --namespace ingress-nginx-global --create-namespace ingress-nginx/ingress-nginx +} + +prechecks +prereqs + +if [[ "${kube_version}" =~ .*"k3s".* ]] || [[ "${kube_version}" == "" ]]; then + export KUBETYPE="k3s" + export KUBECONFIG=/etc/rancher/k3s/k3s.yaml + echo "export KUBECONFIG=/etc/rancher/k3s/k3s.yaml" >> /etc/profile.d/k3s.sh + install_k3s + mkdir -p /root/.kube/ + ln -s /etc/rancher/k3s/k3s.yaml /root/.kube/config +elif [[ "${kube_version}" =~ .*"rke2".* ]]; then + export KUBETYPE="rke2" + ln -s /var/lib/rancher/rke2/bin/kubectl /usr/local/bin/kubectl + export KUBECONFIG=/etc/rancher/rke2/rke2.yaml + echo "export KUBECONFIG=/etc/rancher/rke2/rke2.yaml" >> /etc/profile.d/rke2.sh + install_rke2 + mkdir -p /root/.kube/ + ln -s /etc/rancher/rke2/rke2.yaml /root/.kube/config +else + die "Kubernetes version ${kube_version} not valid" 2 +fi + +DEPLOY_DEMO=false +INSTALL_METALLB=false +INSTALL_RANCHER=false +INSTALL_GLOBAL_INGRESS=false + +%{ if node_type == "control-plane-master" ~} +INSTALL_METALLB=true +%{ if global_ip_cidr != "" ~} +INSTALL_GLOBAL_INGRESS=true +%{ endif ~} +%{ if deploy_demo != "false" ~} +DEPLOY_DEMO=true +%{ endif ~} +%{ if rancher_flavor != "" ~} +INSTALL_RANCHER=true +%{ endif ~} +%{ endif ~} + +%{ if node_type == "all-in-one" ~} +%{ if global_ip_cidr != "" ~} +INSTALL_METALLB=true +INSTALL_GLOBAL_INGRESS=true +%{ endif } +%{ if ip_pool != "" ~} +INSTALL_METALLB=true +%{ endif } +%{ if deploy_demo != "false" ~} +DEPLOY_DEMO=true +%{ endif ~} +%{ if rancher_flavor != "" ~} +INSTALL_RANCHER=true +%{ endif ~} +%{ endif ~} + +[ $${INSTALL_METALLB} == true ] && install_metallb || true + +%{ if API_IP != "" ~} +%{ if node_type == "control-plane-master" ~} +install_eco +%{ endif ~} +%{ endif ~} + +[ $${INSTALL_GLOBAL_INGRESS} == true ] && install_global_ingress || true +[ $${DEPLOY_DEMO} == true ] && deploy_demo || true +[ $${INSTALL_RANCHER} == true ] && install_rancher || true diff --git a/modules/k3s_cluster/variables.tf b/modules/kube_cluster/variables.tf similarity index 57% rename from modules/k3s_cluster/variables.tf rename to modules/kube_cluster/variables.tf index c3860a4..9cddb34 100644 --- a/modules/k3s_cluster/variables.tf +++ b/modules/kube_cluster/variables.tf @@ -17,30 +17,30 @@ variable "deploy_demo" { variable "cluster_name" { type = string description = "Cluster name" - default = "K3s cluster" + default = "Cluster" } variable "plan_control_plane" { type = string - description = "K3s control plane type/size" + description = "Control plane type/size" default = "c3.small.x86" } variable "plan_node" { type = string - description = "K3s node type/size" + description = "Node type/size" default = "c3.small.x86" } variable "node_count" { type = number - description = "Number of K3s nodes" + description = "Number of nodes" default = "0" } -variable "k3s_ha" { +variable "ha" { type = bool - description = "K3s HA (aka 3 control plane nodes)" + description = "HA (aka 3 control plane nodes)" default = false } @@ -62,16 +62,20 @@ variable "node_hostnames" { default = "node" } -variable "custom_k3s_token" { +variable "custom_token" { type = string - description = "K3s token used for nodes to join the cluster (autogenerated otherwise)" + description = "Token used for nodes to join the cluster (autogenerated otherwise)" default = null } variable "ip_pool_count" { type = number - description = "Number of public IPv4 per metro to be used as LoadBalancers with MetalLB" + description = "Number of public IPv4 per metro to be used as LoadBalancers with MetalLB (it needs to be power of 2 between 0 and 256 as required by Equinix Metal)" default = 0 + validation { + condition = contains([0, 1, 2, 4, 8, 16, 32, 64, 128, 256], var.ip_pool_count) + error_message = "The value must be a power of 2 between 0 and 256." + } } variable "global_ip_cidr" { @@ -80,9 +84,9 @@ variable "global_ip_cidr" { default = null } -variable "k3s_version" { +variable "kube_version" { type = string - description = "K3s version to be installed. Empty for latest" + description = "K3s/RKE2 version to be installed. Empty for latest K3s" default = "" } @@ -91,3 +95,21 @@ variable "metallb_version" { description = "MetalLB version to be installed. Empty for latest" default = "" } + +variable "rancher_version" { + type = string + description = "Rancher version to be installed (vX.Y.Z). Empty for latest" + default = "" +} + +variable "rancher_flavor" { + type = string + description = "Rancher flavor to be installed (prime, latest, stable or alpha). Empty to not install it" + default = "" +} + +variable "custom_rancher_password" { + type = string + description = "Rancher initial password (autogenerated if not provided)" + default = null +} diff --git a/modules/k3s_cluster/versions.tf b/modules/kube_cluster/versions.tf similarity index 100% rename from modules/k3s_cluster/versions.tf rename to modules/kube_cluster/versions.tf diff --git a/outputs.tf b/outputs.tf index 432a199..07a3c7b 100644 --- a/outputs.tf +++ b/outputs.tf @@ -8,9 +8,25 @@ output "demo_url" { description = "URL of the demo application to demonstrate a global IP shared across Metros" } -output "k3s_api" { +output "cluster_details" { value = { - for cluster in var.clusters : cluster.name => module.k3s_cluster[cluster.name].k3s_api_ip + for cluster in var.clusters : cluster.name => { + api = module.kube_cluster[cluster.name].kube_api_ip + ingress = module.kube_cluster[cluster.name].ingress_ip + ip_pool_cidr = module.kube_cluster[cluster.name].ip_pool_cidr + nodes = module.kube_cluster[cluster.name].nodes_details + } } - description = "List of Clusters => K3s APIs" + description = "List of Clusters => K8s details" +} + +output "rancher_urls" { + value = { + for cluster in var.clusters : cluster.name => { + rancher_url = cluster.rancher_flavor != "" ? module.kube_cluster[cluster.name].rancher_address : null + rancher_initial_password_base64 = cluster.rancher_flavor != "" ? base64encode(module.kube_cluster[cluster.name].rancher_password) : null + } + if module.kube_cluster[cluster.name].rancher_address != null + } + description = "List of Clusters => Rancher details" } diff --git a/variables.tf b/variables.tf index 490abdb..2aebd38 100644 --- a/variables.tf +++ b/variables.tf @@ -16,21 +16,24 @@ variable "deploy_demo" { } variable "clusters" { - description = "K3s cluster definition" + description = "Cluster definition" type = list(object({ - name = optional(string, "K3s demo cluster") + name = optional(string, "Demo cluster") metro = optional(string, "FR") plan_control_plane = optional(string, "c3.small.x86") plan_node = optional(string, "c3.small.x86") node_count = optional(number, 0) - k3s_ha = optional(bool, false) + ha = optional(bool, false) os = optional(string, "debian_11") - control_plane_hostnames = optional(string, "k3s-cp") - node_hostnames = optional(string, "k3s-node") - custom_k3s_token = optional(string, "") + control_plane_hostnames = optional(string, "cp") + node_hostnames = optional(string, "node") + custom_token = optional(string, "") ip_pool_count = optional(number, 0) - k3s_version = optional(string, "") + kube_version = optional(string, "") metallb_version = optional(string, "") + rancher_flavor = optional(string, "") + rancher_version = optional(string, "") + custom_rancher_password = optional(string, "") })) default = [{}] }