e2e k3s plugin

.gitignore

@@ -48,3 +48,5 @@ docs/_build
 
 # development files
 /tmp
+
+/k3s-ansible

test/e2e/pipeline/infra/k3s/.gitignore

@@ -0,0 +1,2 @@
+k3s-ansible/
+inventory.yml

test/e2e/pipeline/infra/k3s/clean.sh

@@ -26,8 +26,16 @@ error() {
 }
 trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
 
-# Cleaning all remaining clusters
+CLUSTER_NAME=cluster
+RUNNER_NAME=${RUNNER_NAME:-"test"}
 
-K3D="${BINDIR}/k3d"
+TARGET_NAMESPACE="liqo-ci"
 
-${K3D} cluster delete --all
+for i in $(seq 1 "${CLUSTER_NUMBER}");
+do
+ K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}"
+ echo "Deleting cluster ${K3S_CLUSTER_NAME}"
+ "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-control-plane" --ignore-not-found
+ "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-worker-1" --ignore-not-found
+ "${KUBECTL}" delete -n "${TARGET_NAMESPACE}" vms "${K3S_CLUSTER_NAME}-worker-2" --ignore-not-found
+done

test/e2e/pipeline/infra/k3s/inventory.template.yml

@@ -0,0 +1,48 @@
+---
+k3s_cluster:
+ children:
+ server:
+ hosts:
+ ${CONTROL_PLANE_IP}:
+ agent:
+ hosts:
+ ${WORKER_1_IP}:
+ ${WORKER_2_IP}:
+
+ # Required Vars
+ vars:
+ ansible_port: 22
+ ansible_user: ubuntu
+ k3s_version: ${K8S_VERSION}+k3s1
+ # The token should be a random string of reasonable length. You can generate
+ # one with the following commands:
+ # - openssl rand -base64 64
+ # - pwgen -s 64 1
+ # You can use ansible-vault to encrypt this value / keep it secret.
+ token: "changeme!"
+ api_endpoint: "{{ hostvars[groups['server'][0]]['ansible_host'] | default(groups['server'][0]) }}"
+ extra_server_args: "--cluster-cidr=${POD_CIDR} --service-cidr=${SERVICE_CIDR}"
+ extra_agent_args: ""
+
+ # Optional vars
+
+ # cluster_context: k3s-ansible
+ # api_port: 6443
+ # k3s_server_location: /var/lib/rancher/k3s
+ # systemd_dir: /etc/systemd/system
+ # extra_service_envs: [ 'ENV_VAR1=VALUE1', 'ENV_VAR2=VALUE2' ]
+ # user_kubectl: true, by default kubectl is symlinked and configured for use by ansible_user. Set to false to only kubectl via root user.
+
+ # Manifests or Airgap should be either full paths or relative to the playbook directory.
+ # List of locally available manifests to apply to the cluster, useful for PVCs or Traefik modifications.
+ # extra_manifests: [ '/path/to/manifest1.yaml', '/path/to/manifest2.yaml' ]
+ # airgap_dir: /tmp/k3s-airgap-images
+
+ # server_config_yaml: |
+ # This is now an inner yaml file. Maintain the indentation.
+ # YAML here will be placed as the content of /etc/rancher/k3s/config.yaml
+ # See https://docs.k3s.io/installation/configuration#configuration-file
+ # registries_config_yaml: |
+ # Containerd can be configured to connect to private registries and use them to pull images as needed by the kubelet.
+ # YAML here will be placed as the content of /etc/rancher/k3s/registries.yaml
+ # See https://docs.k3s.io/installation/private-registry

test/e2e/pipeline/infra/k3s/pre-requirements.sh

@@ -47,18 +47,25 @@ install_kubectl "${OS}" "${ARCH}" "${K8S_VERSION}"
 
 install_helm "${OS}" "${ARCH}"
 
-K3D_VERSION="v5.4.7"
+# install ansible
 
-echo "Downloading K3D ${K3D_VERSION}"
+# ensure pipx is installed
+if ! command -v pipx &> /dev/null; then
+ python3 -m pip install --user pipx
+ python3 -m pipx ensurepath
+ source "$HOME/.bashrc"
+
+ sudo apt update
+ sudo apt install -y python3-venv
+fi
 
-if ! command -v docker &> /dev/null;
-then
- echo "MISSING REQUIREMENT: docker engine could not be found on your system. Please install docker engine to continue: https://docs.docker.com/get-docker/"
- return 1
+# ensure envsubst is installed
+if ! command -v envsubst &> /dev/null; then
+ sudo apt update
+ sudo apt install -y gettext
 fi
 
-if [[ ! -f "${BINDIR}/k3d" ]]; then
- echo "k3d could not be found. Downloading https://k3d.sigs.k8s.io/dl/${K3D_VERSION}/k3d-${OS}-${ARCH} ..."
- curl -Lo "${BINDIR}"/k3d "https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-${OS}-${ARCH}"
- chmod +x "${BINDIR}"/k3d
+# ensure ansible is installed
+if ! command -v ansible &> /dev/null; then
+ pipx install ansible
 fi

test/e2e/pipeline/infra/k3s/setup.sh

@@ -26,31 +26,61 @@ error() {
 }
 trap 'error "${BASH_SOURCE}" "${LINENO}"' ERR
 
-K3D="${BINDIR}/k3d"
-
 CLUSTER_NAME=cluster
+RUNNER_NAME=${RUNNER_NAME:-"test"}
+
+TARGET_NAMESPACE="liqo-ci"
+
+BASE_DIR=$(dirname "$0")
 
 export SERVICE_CIDR=10.100.0.0/16
 export POD_CIDR=10.200.0.0/16
 export POD_CIDR_OVERLAPPING=${POD_CIDR_OVERLAPPING:-"false"}
 
 for i in $(seq 1 "${CLUSTER_NUMBER}");
 do
- if [[ ${POD_CIDR_OVERLAPPING} != "true" ]]; then
+ K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}"
+ echo "Creating cluster ${K3S_CLUSTER_NAME}"
+ CLUSTER_NAME="$K3S_CLUSTER_NAME" envsubst < "$BASE_DIR/vms.template.yaml" | "${KUBECTL}" apply -n "${TARGET_NAMESPACE}" -f -
+done
+
+# Wait for the clusters to be ready
+for i in $(seq 1 "${CLUSTER_NUMBER}");
+do
+ K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}"
+ "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-control-plane" -n "${TARGET_NAMESPACE}"
+ "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-worker-1" -n "${TARGET_NAMESPACE}"
+ "${KUBECTL}" wait --for=condition=Ready --timeout=20m vmi "${K3S_CLUSTER_NAME}-worker-2" -n "${TARGET_NAMESPACE}"
+done
+
+sleep 30 # wait for the ssh server to be installed and running
+
+rm -rf k3s-ansible || true
+git clone https://github.com/k3s-io/k3s-ansible.git
+cd k3s-ansible
+
+for i in $(seq 1 "${CLUSTER_NUMBER}");
+do
+ K3S_CLUSTER_NAME="${RUNNER_NAME}-${CLUSTER_NAME}${i}"
+
+ if [[ ${POD_CIDR_OVERLAPPING} != "true" ]]; then
  # this should avoid the ipam to reserve a pod CIDR of another cluster as local external CIDR causing remapping
  export POD_CIDR="10.$((i * 10)).0.0/16"
  fi
- echo "Creating cluster ${CLUSTER_NAME}${i}"
- ${K3D} cluster create "${CLUSTER_NAME}${i}" \
- --k3s-arg "--cluster-cidr=${POD_CIDR}@server:*" \
- --k3s-arg "--service-cidr=${SERVICE_CIDR}@server:*" \
- --no-lb \
- --network k3d \
- --verbose
- ${K3D} node create "${CLUSTER_NAME}${i}-agent" \
- --cluster "${CLUSTER_NAME}${i}" \
- --role agent \
- --verbose
- ${K3D} kubeconfig write "${CLUSTER_NAME}${i}" \
- --output "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
+
+ _CONTROL_PLANE_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-control-plane" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}')
+ _WORKER_1_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-worker-1" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}')
+ _WORKER_2_IP=$("${KUBECTL}" get vmi "${K3S_CLUSTER_NAME}-worker-2" -n "${TARGET_NAMESPACE}" -o jsonpath='{.status.interfaces[0].ipAddress}')
+ export CONTROL_PLANE_IP="${_CONTROL_PLANE_IP}"
+ export WORKER_1_IP="${_WORKER_1_IP}"
+ export WORKER_2_IP="${_WORKER_2_IP}"
+
+ envsubst < "$BASE_DIR/inventory.template.yml" > inventory.yml
+ ansible-playbook playbooks/site.yml -i inventory.yml --key-file "$SSH_KEY_PATH"
+
+ mkdir -p "${TMPDIR}/kubeconfigs"
+ scp -i "$SSH_KEY_PATH" -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null ubuntu@"${CONTROL_PLANE_IP}":~/.kube/config "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
+ sed -i "s/127.0.0.1/${CONTROL_PLANE_IP}/g" "${TMPDIR}/kubeconfigs/liqo_kubeconf_${i}"
 done
+
+cd ..

test/e2e/pipeline/infra/k3s/vms.template.yaml

@@ -0,0 +1,156 @@
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+ name: ${CLUSTER_NAME}-control-plane
+ namespace: liqo-ci
+spec:
+ running: true
+ template:
+ metadata:
+ labels:
+ kubevirt.io/vmpool: control-plane
+ spec:
+ architecture: amd64
+ domain:
+ cpu:
+ cores: 2
+ devices:
+ disks:
+ - disk:
+ bus: virtio
+ name: root
+ - disk:
+ bus: virtio
+ name: cloud-init
+ networkInterfaceMultiqueue: true
+ machine:
+ type: q35
+ resources:
+ requests:
+ memory: 4Gi
+ volumes:
+ - dataVolume:
+ name: ${CLUSTER_NAME}-control-plane-data-volume
+ name: root
+ - name: cloud-init
+ cloudInitNoCloud:
+ secretRef:
+ name: cloud-init
+ dataVolumeTemplates:
+ - metadata:
+ creationTimestamp: null
+ name: ${CLUSTER_NAME}-control-plane-data-volume
+ spec:
+ storage:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 8Gi
+ source:
+ registry:
+ url: docker://ghcr.io/liqotech/ubuntu:22.04
+
+---
+
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+ name: ${CLUSTER_NAME}-worker-1
+ namespace: liqo-ci
+spec:
+ running: true
+ template:
+ spec:
+ architecture: amd64
+ domain:
+ cpu:
+ cores: 2
+ devices:
+ disks:
+ - disk:
+ bus: virtio
+ name: root
+ - disk:
+ bus: virtio
+ name: cloud-init
+ networkInterfaceMultiqueue: true
+ machine:
+ type: q35
+ resources:
+ requests:
+ memory: 4Gi
+ volumes:
+ - dataVolume:
+ name: ${CLUSTER_NAME}-worker-1-data-volume
+ name: root
+ - name: cloud-init
+ cloudInitNoCloud:
+ secretRef:
+ name: cloud-init
+ dataVolumeTemplates:
+ - metadata:
+ creationTimestamp: null
+ name: ${CLUSTER_NAME}-worker-1-data-volume
+ spec:
+ storage:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 8Gi
+ source:
+ registry:
+ url: docker://ghcr.io/liqotech/ubuntu:22.04
+
+---
+
+apiVersion: kubevirt.io/v1
+kind: VirtualMachine
+metadata:
+ name: ${CLUSTER_NAME}-worker-2
+ namespace: liqo-ci
+spec:
+ running: true
+ template:
+ spec:
+ architecture: amd64
+ domain:
+ cpu:
+ cores: 2
+ devices:
+ disks:
+ - disk:
+ bus: virtio
+ name: root
+ - disk:
+ bus: virtio
+ name: cloud-init
+ networkInterfaceMultiqueue: true
+ machine:
+ type: q35
+ resources:
+ requests:
+ memory: 4Gi
+ volumes:
+ - dataVolume:
+ name: ${CLUSTER_NAME}-worker-2-data-volume
+ name: root
+ - name: cloud-init
+ cloudInitNoCloud:
+ secretRef:
+ name: cloud-init
+ dataVolumeTemplates:
+ - metadata:
+ creationTimestamp: null
+ name: ${CLUSTER_NAME}-worker-2-data-volume
+ spec:
+ storage:
+ accessModes:
+ - ReadWriteOnce
+ resources:
+ requests:
+ storage: 8Gi
+ source:
+ registry:
+ url: docker://ghcr.io/liqotech/ubuntu:22.04

test/e2e/pipeline/installer/liqoctl/peer.sh

@@ -37,6 +37,8 @@ do
  ARGS=("${ARGS[@]}" --server-service-type NodePort)
  elif [[ "${INFRA}" == "kind" ]]; then
  ARGS=("${ARGS[@]}" --server-service-type NodePort)
+ elif [[ "${INFRA}" == "k3s" ]]; then
+ ARGS=("${ARGS[@]}" --server-service-type NodePort)
  fi
 
  ARGS=("${ARGS[@]}")