From ae3762f40b338f3a8e5eba9a093a44fbb5b92d56 Mon Sep 17 00:00:00 2001 From: Alessandro Olivero Date: Thu, 12 Sep 2024 17:28:40 +0200 Subject: [PATCH] e2e k3s plugin --- .gitignore | 2 + .../apiserver_interaction_test.go | 6 + test/e2e/cruise/network/network_test.go | 16 +- test/e2e/cruise/storage/storage_test.go | 6 + test/e2e/pipeline/infra/k3s/.gitignore | 2 + .../pipeline/infra/k3s/ansible-blocking-io.py | 18 +++ test/e2e/pipeline/infra/k3s/clean.sh | 14 +- .../pipeline/infra/k3s/inventory.template.yml | 48 ++++++ .../pipeline/infra/k3s/pre-requirements.sh | 28 ++-- test/e2e/pipeline/infra/k3s/setup.sh | 117 ++++++++++++-- test/e2e/pipeline/infra/k3s/vms.template.yaml | 153 ++++++++++++++++++ test/e2e/pipeline/installer/liqoctl/peer.sh | 11 ++ test/e2e/testconsts/consts.go | 2 + 13 files changed, 393 insertions(+), 30 deletions(-) create mode 100644 test/e2e/pipeline/infra/k3s/.gitignore create mode 100644 test/e2e/pipeline/infra/k3s/ansible-blocking-io.py create mode 100644 test/e2e/pipeline/infra/k3s/inventory.template.yml create mode 100644 test/e2e/pipeline/infra/k3s/vms.template.yaml diff --git a/.gitignore b/.gitignore index d8e7036f12..92e7455b41 100644 --- a/.gitignore +++ b/.gitignore @@ -48,3 +48,5 @@ docs/_build # development files /tmp + +/k3s-ansible diff --git a/test/e2e/cruise/apiserverinteraction/apiserver_interaction_test.go b/test/e2e/cruise/apiserverinteraction/apiserver_interaction_test.go index 558dbcd86e..831a8b1a14 100644 --- a/test/e2e/cruise/apiserverinteraction/apiserver_interaction_test.go +++ b/test/e2e/cruise/apiserverinteraction/apiserver_interaction_test.go @@ -26,6 +26,7 @@ import ( "k8s.io/apimachinery/pkg/version" "k8s.io/client-go/discovery" + "github.com/liqotech/liqo/test/e2e/testconsts" "github.com/liqotech/liqo/test/e2e/testutils/apiserver" "github.com/liqotech/liqo/test/e2e/testutils/config" "github.com/liqotech/liqo/test/e2e/testutils/tester" @@ -41,6 +42,11 @@ const ( func TestE2E(t *testing.T) { util.CheckIfTestIsSkipped(t, clustersRequired, testName) + + if util.GetEnvironmentVariableOrDie(testconsts.InfrastructureEnvVar) == testconsts.ProviderK3s { + t.Skipf("Skipping %s test on k3s", testName) + } + RegisterFailHandler(Fail) RunSpecs(t, "Liqo E2E Suite") } diff --git a/test/e2e/cruise/network/network_test.go b/test/e2e/cruise/network/network_test.go index 82e3f5e9a9..9e5f2dfa2e 100644 --- a/test/e2e/cruise/network/network_test.go +++ b/test/e2e/cruise/network/network_test.go @@ -35,6 +35,7 @@ import ( "github.com/liqotech/liqo/pkg/gateway" networkflags "github.com/liqotech/liqo/pkg/liqoctl/test/network/flags" "github.com/liqotech/liqo/pkg/liqoctl/test/network/setup" + "github.com/liqotech/liqo/test/e2e/testconsts" "github.com/liqotech/liqo/test/e2e/testutils/config" "github.com/liqotech/liqo/test/e2e/testutils/tester" "github.com/liqotech/liqo/test/e2e/testutils/util" @@ -49,6 +50,11 @@ const ( func TestE2E(t *testing.T) { util.CheckIfTestIsSkipped(t, clustersRequired, testName) + + if util.GetEnvironmentVariableOrDie(testconsts.InfrastructureEnvVar) == testconsts.ProviderK3s { + t.Skipf("Skipping %s test on k3s", testName) + } + RegisterFailHandler(Fail) RunSpecs(t, "Liqo E2E Suite") } @@ -96,7 +102,9 @@ var _ = BeforeSuite(func() { switch testContext.Infrastructure { case "cluster-api": - ovverideArgsClusterAPI(&args) + overrideArgsClusterAPI(&args) + case "k3s": + overrideArgsK3s(&args) case "kind": overrideArgsKind(&args) case "eks": @@ -209,7 +217,11 @@ func overrideArgsFlannel(args *networkTestsArgs) { args.nodePortNodes = networkflags.NodePortNodesWorkers } -func ovverideArgsClusterAPI(args *networkTestsArgs) { +func overrideArgsClusterAPI(args *networkTestsArgs) { + args.loadBalancer = false +} + +func overrideArgsK3s(args *networkTestsArgs) { args.loadBalancer = false } diff --git a/test/e2e/cruise/storage/storage_test.go b/test/e2e/cruise/storage/storage_test.go index e528b081fa..f75c5d2fea 100644 --- a/test/e2e/cruise/storage/storage_test.go +++ b/test/e2e/cruise/storage/storage_test.go @@ -31,6 +31,7 @@ import ( liqoconst "github.com/liqotech/liqo/pkg/consts" liqoctlmove "github.com/liqotech/liqo/pkg/liqoctl/move" + "github.com/liqotech/liqo/test/e2e/testconsts" "github.com/liqotech/liqo/test/e2e/testutils/config" "github.com/liqotech/liqo/test/e2e/testutils/storage" "github.com/liqotech/liqo/test/e2e/testutils/tester" @@ -46,6 +47,11 @@ const ( func TestE2E(t *testing.T) { util.CheckIfTestIsSkipped(t, clustersRequired, testName) + + if util.GetEnvironmentVariableOrDie(testconsts.InfrastructureEnvVar) == testconsts.ProviderK3s { + t.Skipf("Skipping %s test on k3s", testName) + } + RegisterFailHandler(Fail) RunSpecs(t, "Liqo E2E Suite") } diff --git a/test/e2e/pipeline/infra/k3s/.gitignore b/test/e2e/pipeline/infra/k3s/.gitignore new file mode 100644 index 0000000000..699d734c6f --- /dev/null +++ b/test/e2e/pipeline/infra/k3s/.gitignore @@ -0,0 +1,2 @@ +k3s-ansible/ +inventory.yml diff --git a/test/e2e/pipeline/infra/k3s/ansible-blocking-io.py b/test/e2e/pipeline/infra/k3s/ansible-blocking-io.py new file mode 100644 index 0000000000..6eb25a8a71 --- /dev/null +++ b/test/e2e/pipeline/infra/k3s/ansible-blocking-io.py @@ -0,0 +1,18 @@ +#!/usr/bin/python3 + +# This python script is to fix ansible error in CI test. It's not a bug of ansible, but a side-effect of a different change +# Will remove it later if we do not see ansible error in CI + +import os +import sys + +for handle in (sys.stdin, sys.stdout, sys.stderr): + try: + fd = handle.fileno() + except Exception as e: + print(f"Error: {e}") + continue + + os.set_blocking(fd, True) + +print("Blocking I/O is set for all file descriptors") diff --git a/test/e2e/pipeline/infra/k3s/clean.sh b/test/e2e/pipeline/infra/k3s/clean.sh index d838023da4..65888e4ac7 100755 --- a/test/e2e/pipeline/infra/k3s/clean.sh +++ b/test/e2e/pipeline/infra/k3s/clean.sh @@ -26,8 +26,16 @@ error() { } trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR -# Cleaning all remaining clusters +CLUSTER_NAME=cluster +RUNNER_NAME=${RUNNER_NAME:-"test"} -K3D="${BINDIR}/k3d" +TARGET_NAMESPACE="liqo-ci" -${K3D} cluster delete --all +for i in $(seq 1 "${CLUSTER_NUMBER}"); +do + K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}" + echo "Deleting cluster ${K3S_CLUSTER_NAME}" + "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-control-plane" --ignore-not-found + "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-worker-1" --ignore-not-found + "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-worker-2" --ignore-not-found +done diff --git a/test/e2e/pipeline/infra/k3s/inventory.template.yml b/test/e2e/pipeline/infra/k3s/inventory.template.yml new file mode 100644 index 0000000000..a783c8521c --- /dev/null +++ b/test/e2e/pipeline/infra/k3s/inventory.template.yml @@ -0,0 +1,48 @@ +--- +k3s_cluster: + children: + server: + hosts: + ${CONTROL_PLANE_IP}: + agent: + hosts: + ${WORKER_1_IP}: + ${WORKER_2_IP}: + + # Required Vars + vars: + ansible_port: 22 + ansible_user: ubuntu + k3s_version: ${K8S_VERSION}+k3s1 + # The token should be a random string of reasonable length. You can generate + # one with the following commands: + # - openssl rand -base64 64 + # - pwgen -s 64 1 + # You can use ansible-vault to encrypt this value / keep it secret. + token: "changeme!" + api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}" + extra_server_args: "--cluster-cidr=${POD_CIDR} --service-cidr=${SERVICE_CIDR}" + extra_agent_args: "" + + # Optional vars + + # cluster_context: k3s-ansible + # api_port: 6443 + # k3s_server_location: /var/lib/rancher/k3s + # systemd_dir: /etc/systemd/system + # extra_service_envs: [ 'ENV_VAR1=VALUE1', 'ENV_VAR2=VALUE2' ] + # user_kubectl: true, by default kubectl is symlinked and configured for use by ansible_user. Set to false to only kubectl via root user. + + # Manifests or Airgap should be either full paths or relative to the playbook directory. + # List of locally available manifests to apply to the cluster, useful for PVCs or Traefik modifications. + # extra_manifests: [ '/path/to/manifest1.yaml', '/path/to/manifest2.yaml' ] + # airgap_dir: /tmp/k3s-airgap-images + + # server_config_yaml: | + # This is now an inner yaml file. Maintain the indentation. + # YAML here will be placed as the content of /etc/rancher/k3s/config.yaml + # See https://docs.k3s.io/installation/configuration#configuration-file + # registries_config_yaml: | + # Containerd can be configured to connect to private registries and use them to pull images as needed by the kubelet. + # YAML here will be placed as the content of /etc/rancher/k3s/registries.yaml + # See https://docs.k3s.io/installation/private-registry diff --git a/test/e2e/pipeline/infra/k3s/pre-requirements.sh b/test/e2e/pipeline/infra/k3s/pre-requirements.sh index 468c4e5baa..e8f495a769 100755 --- a/test/e2e/pipeline/infra/k3s/pre-requirements.sh +++ b/test/e2e/pipeline/infra/k3s/pre-requirements.sh @@ -47,18 +47,26 @@ install_kubectl "${OS}" "${ARCH}" "${K8S_VERSION}" install_helm "${OS}" "${ARCH}" -K3D_VERSION="v5.4.7" +# install ansible -echo "Downloading K3D ${K3D_VERSION}" +# ensure pipx is installed +if ! command -v pipx &> /dev/null; then + python3 -m pip install --user pipx + python3 -m pipx ensurepath --force + source "$HOME/.bashrc" || true -if ! command -v docker &> /dev/null; -then - echo "MISSING REQUIREMENT: docker engine could not be found on your system. Please install docker engine to continue: https://docs.docker.com/get-docker/" - return 1 + sudo apt update + sudo apt install -y python3-venv fi -if [[ ! -f "${BINDIR}/k3d" ]]; then - echo "k3d could not be found. Downloading https://k3d.sigs.k8s.io/dl/${K3D_VERSION}/k3d-${OS}-${ARCH} ..." - curl -Lo "${BINDIR}"/k3d "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-${OS}-${ARCH}" - chmod +x "${BINDIR}"/k3d +# ensure envsubst is installed +if ! command -v envsubst &> /dev/null; then + sudo apt update + sudo apt install -y gettext +fi + +# ensure ansible is installed +if ! command -v ansible &> /dev/null; then + pipx install --include-deps ansible + ansible-playbook --version fi diff --git a/test/e2e/pipeline/infra/k3s/setup.sh b/test/e2e/pipeline/infra/k3s/setup.sh index cfd6330e2d..4ae6ac6bdf 100755 --- a/test/e2e/pipeline/infra/k3s/setup.sh +++ b/test/e2e/pipeline/infra/k3s/setup.sh @@ -26,9 +26,50 @@ error() { } trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR -K3D="${BINDIR}/k3d" +check_host_login() { + local host=$1 + local user=$2 + local key=$3 + local timeout=${4:-"600"} + + s=$(date +%s) + local start=${s} + while true; do + if ssh -i "${key}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 "${user}@${host}" exit; then + break + fi + if [[ $(( $(date +%s) - start )) -gt ${timeout} ]]; then + echo "Timeout reached while waiting for the host to be reachable" + exit 1 + fi + sleep 5 + done + + sleep 5 + + # check apt is able to take the lock + start=$(date +%s) + while true; do + if ssh -i "${key}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectTimeout=5 "${user}@${host}" sudo apt update; then + break + fi + if [[ $(( $(date +%s) - start )) -gt ${timeout} ]]; then + echo "Timeout reached while waiting for apt to be available" + exit 1 + fi + sleep 5 + done +} + +# shellcheck disable=SC1091 +source "$HOME/.bashrc" || true CLUSTER_NAME=cluster +RUNNER_NAME=${RUNNER_NAME:-"test"} + +TARGET_NAMESPACE="liqo-ci" + +BASE_DIR=$(dirname "$0") export SERVICE_CIDR=10.100.0.0/16 export POD_CIDR=10.200.0.0/16 @@ -36,21 +77,67 @@ export POD_CIDR_OVERLAPPING=${POD_CIDR_OVERLAPPING:-"false"} for i in $(seq 1 "${CLUSTER_NUMBER}"); do - if [[ ${POD_CIDR_OVERLAPPING} != "true" ]]; then + K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}" + echo "Creating cluster ${K3S_CLUSTER_NAME}" + CLUSTER_NAME="$K3S_CLUSTER_NAME" envsubst < "$BASE_DIR/vms.template.yaml" | "${KUBECTL}" apply -n "${TARGET_NAMESPACE}" -f - +done + +# Wait for the clusters to be ready +for i in $(seq 1 "${CLUSTER_NUMBER}"); +do + K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}" + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vm "${K3S_CLUSTER_NAME}-control-plane" -n "${TARGET_NAMESPACE}" + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vm "${K3S_CLUSTER_NAME}-worker-1" -n "${TARGET_NAMESPACE}" + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vm "${K3S_CLUSTER_NAME}-worker-2" -n "${TARGET_NAMESPACE}" + + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-control-plane" -n "${TARGET_NAMESPACE}" + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-worker-1" -n "${TARGET_NAMESPACE}" + "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-worker-2" -n "${TARGET_NAMESPACE}" +done + +SSH_KEY_FILE="${TMPDIR}/id_rsa" +echo "${SSH_KEY_PATH}" > "${SSH_KEY_FILE}" +chmod 600 "${SSH_KEY_FILE}" + +rm -rf k3s-ansible || true +git clone https://github.com/k3s-io/k3s-ansible.git +cd k3s-ansible + +for i in $(seq 1 "${CLUSTER_NUMBER}"); +do + K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}" + + if [[ ${POD_CIDR_OVERLAPPING} != "true" ]]; then # this should avoid the ipam to reserve a pod CIDR of another cluster as local external CIDR causing remapping export POD_CIDR="10.$((i * 10)).0.0/16" fi - echo "Creating cluster ${CLUSTER_NAME}${i}" - ${K3D} cluster create "${CLUSTER_NAME}${i}" \ - --k3s-arg "--cluster-cidr=${POD_CIDR}@server:*" \ - --k3s-arg "--service-cidr=${SERVICE_CIDR}@server:*" \ - --no-lb \ - --network k3d \ - --verbose - ${K3D} node create "${CLUSTER_NAME}${i}-agent" \ - --cluster "${CLUSTER_NAME}${i}" \ - --role agent \ - --verbose - ${K3D} kubeconfig write "${CLUSTER_NAME}${i}" \ - --output "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" + + _CONTROL_PLANE_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-control-plane" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}') + _WORKER_1_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-worker-1" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}') + _WORKER_2_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-worker-2" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}') + export CONTROL_PLANE_IP="${_CONTROL_PLANE_IP}" + export WORKER_1_IP="${_WORKER_1_IP}" + export WORKER_2_IP="${_WORKER_2_IP}" + + check_host_login "${CONTROL_PLANE_IP}" "ubuntu" "${SSH_KEY_FILE}" + check_host_login "${WORKER_1_IP}" "ubuntu" "${SSH_KEY_FILE}" + check_host_login "${WORKER_2_IP}" "ubuntu" "${SSH_KEY_FILE}" + + # if running in GitHub Actions + if [[ -n "${GITHUB_ACTIONS}" ]]; then + sudo python3 "${BASE_DIR}/ansible-blocking-io.py" + fi + + ansible-playbook --version + envsubst < "$BASE_DIR/inventory.template.yml" > inventory.yml + ansible-playbook playbooks/site.yml -i inventory.yml --key-file "${SSH_KEY_FILE}" + + mkdir -p "${TMPDIR}/kubeconfigs" + scp -i "${SSH_KEY_FILE}" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@"${CONTROL_PLANE_IP}":~/.kube/config "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" + sed -i "s/127.0.0.1/${CONTROL_PLANE_IP}/g" "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" + + # add default namespace to kubeconfig + KUBECONFIG="${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}" "${KUBECTL}" config set-context --current --namespace=default done + +cd .. diff --git a/test/e2e/pipeline/infra/k3s/vms.template.yaml b/test/e2e/pipeline/infra/k3s/vms.template.yaml new file mode 100644 index 0000000000..cd15259ca8 --- /dev/null +++ b/test/e2e/pipeline/infra/k3s/vms.template.yaml @@ -0,0 +1,153 @@ +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: ${CLUSTER_NAME}-control-plane + namespace: liqo-ci +spec: + running: true + template: + metadata: + labels: + kubevirt.io/vmpool: control-plane + spec: + domain: + cpu: + cores: 2 + devices: + disks: + - disk: + bus: virtio + name: root + - disk: + bus: virtio + name: cloud-init + networkInterfaceMultiqueue: true + machine: + type: q35 + resources: + requests: + memory: 4Gi + volumes: + - dataVolume: + name: ${CLUSTER_NAME}-control-plane-data-volume + name: root + - name: cloud-init + cloudInitNoCloud: + secretRef: + name: cloud-init + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: ${CLUSTER_NAME}-control-plane-data-volume + spec: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + source: + registry: + url: docker://ghcr.io/liqotech/ubuntu:22.04 + +--- + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: ${CLUSTER_NAME}-worker-1 + namespace: liqo-ci +spec: + running: true + template: + spec: + domain: + cpu: + cores: 2 + devices: + disks: + - disk: + bus: virtio + name: root + - disk: + bus: virtio + name: cloud-init + networkInterfaceMultiqueue: true + machine: + type: q35 + resources: + requests: + memory: 4Gi + volumes: + - dataVolume: + name: ${CLUSTER_NAME}-worker-1-data-volume + name: root + - name: cloud-init + cloudInitNoCloud: + secretRef: + name: cloud-init + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: ${CLUSTER_NAME}-worker-1-data-volume + spec: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + source: + registry: + url: docker://ghcr.io/liqotech/ubuntu:22.04 + +--- + +apiVersion: kubevirt.io/v1 +kind: VirtualMachine +metadata: + name: ${CLUSTER_NAME}-worker-2 + namespace: liqo-ci +spec: + running: true + template: + spec: + domain: + cpu: + cores: 2 + devices: + disks: + - disk: + bus: virtio + name: root + - disk: + bus: virtio + name: cloud-init + networkInterfaceMultiqueue: true + machine: + type: q35 + resources: + requests: + memory: 4Gi + volumes: + - dataVolume: + name: ${CLUSTER_NAME}-worker-2-data-volume + name: root + - name: cloud-init + cloudInitNoCloud: + secretRef: + name: cloud-init + dataVolumeTemplates: + - metadata: + creationTimestamp: null + name: ${CLUSTER_NAME}-worker-2-data-volume + spec: + storage: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 8Gi + source: + registry: + url: docker://ghcr.io/liqotech/ubuntu:22.04 diff --git a/test/e2e/pipeline/installer/liqoctl/peer.sh b/test/e2e/pipeline/installer/liqoctl/peer.sh index 022555aa33..e5670f00d4 100755 --- a/test/e2e/pipeline/installer/liqoctl/peer.sh +++ b/test/e2e/pipeline/installer/liqoctl/peer.sh @@ -37,8 +37,19 @@ do ARGS=("${ARGS[@]}" --server-service-type NodePort) elif [[ "${INFRA}" == "kind" ]]; then ARGS=("${ARGS[@]}" --server-service-type NodePort) + elif [[ "${INFRA}" == "k3s" ]]; then + ARGS=("${ARGS[@]}" --server-service-type NodePort) fi + echo "Environment variables:" + env + + echo "Kubeconfig consumer:" + cat "${KUBECONFIG}" + + echo "Kubeconfig provider:" + cat "${PROVIDER_KUBECONFIG}" + ARGS=("${ARGS[@]}") "${LIQOCTL}" peer "${ARGS[@]}" diff --git a/test/e2e/testconsts/consts.go b/test/e2e/testconsts/consts.go index 59403312d6..50e066f6f2 100644 --- a/test/e2e/testconsts/consts.go +++ b/test/e2e/testconsts/consts.go @@ -46,6 +46,8 @@ const ( ProviderAWS = "AWS" // ProviderGKE -> provider=GKE. ProviderGKE = "GKE" + // ProviderK3s -> provider=K3s. + ProviderK3s = "k3s" // RegionA -> region=A. RegionA = "A" // RegionB -> region=B.