chore: add day-two tests for e2e-qemu

Uses Sidero Labs d2ctl to install a small sample of
common cluster services to ensure the Talos cluster
is capable.

Signed-off-by: Tim Jones <tim.jones@siderolabs.com>

.drone.jsonnet

@@ -403,15 +403,15 @@ local integration_qemu_encrypted_vip = Step("e2e-encrypted-vip", target="e2e-qem
         "IMAGE_REGISTRY": local_registry,
 });
 
-local integration_qemu_csi = Step("e2e-csi", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
+local integration_qemu_day_two = Step("e2e-day-two", target="e2e-qemu", privileged=true, depends_on=[load_artifacts], environment={
         "IMAGE_REGISTRY": local_registry,
         "SHORT_INTEGRATION_TEST": "yes",
         "QEMU_WORKERS": "3",
         "QEMU_CPUS_WORKERS": "4",
         "QEMU_MEMORY_WORKERS": "5120",
         "QEMU_EXTRA_DISKS": "1",
         "QEMU_EXTRA_DISKS_SIZE": "12288",
-        "WITH_TEST": "run_csi_tests",
+        "WITH_TEST": "run_day_two_tests",
 });
 
 local integration_images = Step("images", target="images", depends_on=[load_artifacts], environment={"IMAGE_REGISTRY": local_registry});
@@ -455,7 +455,7 @@ local integration_pipelines = [
 , integration_cilium, integration_bios, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan]) + integration_trigger(['integration-misc']),
   Pipeline('integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip]) + integration_trigger(['integration-qemu-encrypted-vip']),
   Pipeline('integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race]) + integration_trigger(['integration-qemu-race']),
-  Pipeline('integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi]) + integration_trigger(['integration-qemu-csi']),
+  Pipeline('integration-qemu-day-two', default_pipeline_steps + [integration_qemu_day_two]) + integration_trigger(['integration-qemu-day-two']),
   Pipeline('integration-images', default_pipeline_steps + [integration_images, integration_sbcs]) + integration_trigger(['integration-images']),
 
   // cron pipelines, triggered on schedule events
@@ -467,7 +467,7 @@ local integration_pipelines = [
 , integration_cilium, integration_bios, integration_disk_image, integration_canal_reset, integration_no_cluster_discovery, integration_kubespan], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-qemu-encrypted-vip', default_pipeline_steps + [integration_qemu_encrypted_vip], [default_cron_pipeline]) + cron_trigger(['thrice-daily', 'nightly']),
   Pipeline('cron-integration-qemu-race', default_pipeline_steps + [build_race, integration_qemu_race], [default_cron_pipeline]) + cron_trigger(['nightly']),
-  Pipeline('cron-integration-qemu-csi', default_pipeline_steps + [integration_qemu_csi], [default_cron_pipeline]) + cron_trigger(['nightly']),
+  Pipeline('cron-integration-qemu-day-two', default_pipeline_steps + [integration_qemu_day_two], [default_cron_pipeline]) + cron_trigger(['nightly']),
   Pipeline('cron-integration-images', default_pipeline_steps + [integration_images, integration_sbcs], [default_cron_pipeline]) + cron_trigger(['nightly']),
 ];
 

Makefile

@@ -31,6 +31,8 @@ KUBECTL_URL ?= https://storage.googleapis.com/kubernetes-release/release/v1.23.5
 KUBESTR_URL ?= https://github.com/kastenhq/kubestr/releases/download/v0.4.31/kubestr_0.4.31_Linux_amd64.tar.gz
 CLUSTERCTL_VERSION ?= 1.0.4
 CLUSTERCTL_URL ?= https://github.com/kubernetes-sigs/cluster-api/releases/download/v$(CLUSTERCTL_VERSION)/clusterctl-$(OPERATING_SYSTEM)-amd64
+D2CTL_URL ?= https://github.com/talos-systems/day-two/releases/download/v0.1.0-alpha.1/d2ctl-$(OPERATING_SYSTEM)-amd64
+PULUMI_URL ?= https://get.pulumi.com/releases/sdk/pulumi-v3.26.1-$(OPERATING_SYSTEM)-x64.tar.gz
 TESTPKGS ?= github.com/talos-systems/talos/...
 RELEASES ?= v0.13.4 v0.14.1
 SHORT_INTEGRATION_TEST ?=
@@ -173,7 +175,7 @@ docker-%: ## Builds the specified target defined in the Dockerfile using the doc
 registry-%: ## Builds the specified target defined in the Dockerfile using the image/registry output type. The build result will be pushed to the registry if PUSH=true.
 	@$(MAKE) target-$* TARGET_ARGS="--output type=image,name=$(REGISTRY_AND_USERNAME)/$*:$(IMAGE_TAG) $(TARGET_ARGS)"
 
-hack-test-%: ## Runs the specied script in ./hack/test with well known environment variables.
+hack-test-%: ## Runs the specified script in ./hack/test with well known environment variables.
 	@./hack/test/$*.sh
 
 # Generators
@@ -318,7 +320,16 @@ $(ARTIFACTS)/clusterctl:
 	@curl -L -o $(ARTIFACTS)/clusterctl "$(CLUSTERCTL_URL)"
 	@chmod +x $(ARTIFACTS)/clusterctl
 
-e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr ## Runs the E2E test for the specified platform (e.g. e2e-docker).
+$(ARTIFACTS)/d2ctl:
+	@mkdir -p $(ARTIFACTS)
+	@curl -L -o $(ARTIFACTS)/d2ctl "$(D2CTL_URL)"
+	@chmod +x $(ARTIFACTS)/d2ctl
+
+$(ARTIFACTS)/pulumi:
+	@mkdir -p $(ARTIFACTS)
+	@curl -L "$(PULUMI_URL)" | tar xzf - -C $(ARTIFACTS) --strip-components 1 pulumi/pulumi
+
+e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubectl $(ARTIFACTS)/clusterctl $(ARTIFACTS)/kubestr $(ARTIFACTS)/d2ctl $(ARTIFACTS)/pulumi ## Runs the E2E test for the specified platform (e.g. e2e-docker).
 	@$(MAKE) hack-test-$@ \
 		PLATFORM=$* \
 		TAG=$(TAG) \
@@ -333,7 +344,9 @@ e2e-%: $(ARTIFACTS)/$(INTEGRATION_TEST_DEFAULT_TARGET)-amd64 $(ARTIFACTS)/kubect
 		CUSTOM_CNI_URL=$(CUSTOM_CNI_URL) \
 		KUBECTL=$(PWD)/$(ARTIFACTS)/kubectl \
 		KUBESTR=$(PWD)/$(ARTIFACTS)/kubestr \
-		CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl
+		CLUSTERCTL=$(PWD)/$(ARTIFACTS)/clusterctl \
+		D2CTL=$(PWD)/$(ARTIFACTS)/d2ctl \
+		PULUMI=$(PWD)/$(ARTIFACTS)/pulumi
 
 provision-tests-prepare: release-artifacts $(ARTIFACTS)/$(INTEGRATION_TEST_PROVISION_DEFAULT_TARGET)-amd64
 

hack/test/day-two/config.yaml

@@ -0,0 +1,31 @@
+charts:
+  loki:
+    namespace: loki
+    repo: https://grafana.github.io/helm-charts
+    chart: loki-stack
+    valuesPath: hack/test/day-two/loki-values.yaml      
+
+  metallb:
+    namespace: metallb
+    repo: https://metallb.github.io/metallb
+    chart: metallb
+    valuesPath: hack/test/day-two/metallb-values.yaml      
+
+  ingress-nginx:
+    namespace: ingress
+    repo: https://kubernetes.github.io/ingress-nginx
+    chart: ingress-nginx
+    depends:
+      - metallb
+
+  rook:
+    namespace: rook-ceph
+    repo: https://charts.rook.io/release
+    chart: rook-ceph
+
+  ceph:
+    namespace: rook-ceph
+    repo: https://charts.rook.io/release
+    chart: rook-ceph-cluster
+    depends:
+      - rook

hack/test/day-two/loki-values.yaml

@@ -0,0 +1,12 @@
+grafana:
+  enabled: true
+
+prometheus:
+  enabled: true
+  alertmanager:
+    persistentVolume:
+      enabled: false
+
+  server:
+    persistentVolume:
+      enabled: false

hack/test/day-two/metallb-values.yaml

@@ -0,0 +1,6 @@
+configInline:
+  address-pools:
+    - name: default
+      protocol: layer2
+      addresses:
+        - 10.5.0.240-10.5.0.242

hack/test/e2e.sh

@@ -225,20 +225,26 @@ function run_extensions_test {
   curl http://172.20.1.2/ | grep Hello
 }
 
-function run_csi_tests {
-  rm -rf "${TMP}/rook"
-  git clone --depth=1 --single-branch --branch v1.8.2 https://github.com/rook/rook.git "${TMP}/rook"
-  pushd "${TMP}/rook/deploy/examples"
-  ${KUBECTL} apply -f crds.yaml -f common.yaml -f operator.yaml
-  ${KUBECTL} apply -f cluster.yaml
+function run_day_two_tests {
+  rm -rf "${TMP}/day-two-state"
+  # pulumi needs to be in $PATH temporarily 
+  PATH="${PATH}:$(dirname ${PULUMI})" ${D2CTL} up --state-path "${TMP}/day-two-state" --config-path "${PWD}/hack/test/day-two/config.yaml"
+
+  ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-kube-state-metrics
+  ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-prometheus-server
+  ${KUBECTL} --namespace loki wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/loki-grafana
+
+  ${KUBECTL} --namespace metallb wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/metallb-controller
+
+  ${KUBECTL} --namespace ingress wait --timeout 900s --for=jsonpath='{.status.availableReplicas}=1' deployment/ingress-nginx-controller
+
   # wait for the controller to populate the status field
-  sleep 30
-  ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.phase}=Ready' cephclusters.ceph.rook.io/rook-ceph
-  ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.state}=Created' cephclusters.ceph.rook.io/rook-ceph
-  # .status.ceph is populated later only
-  sleep 60
-  ${KUBECTL} --namespace rook-ceph wait --timeout=900s --for=jsonpath='{.status.ceph.health}=HEALTH_OK' cephclusters.ceph.rook.io/rook-ceph
-  ${KUBECTL} create -f csi/rbd/storageclass.yaml
+  sleep 10
+  ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.phase}=Ready' cephclusters.ceph.rook.io/rook-ceph
+  ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.state}=Created' cephclusters.ceph.rook.io/rook-ceph
+  # .status.ceph is populated only after the cluster comes up
+  sleep 20
+  ${KUBECTL} --namespace rook-ceph wait --timeout=1800s --for=jsonpath='{.status.ceph.health}=HEALTH_OK' cephclusters.ceph.rook.io/rook-ceph
   # hack until https://github.com/kastenhq/kubestr/issues/101 is addressed
-  KUBERNETES_SERVICE_HOST= KUBECONFIG="${TMP}/kubeconfig" ${KUBESTR} fio --storageclass rook-ceph-block --size 10G
+  KUBERNETES_SERVICE_HOST= KUBECONFIG="${TMP}/kubeconfig" ${KUBESTR} fio --storageclass ceph-block --size 10G 
 }