diff --git a/Makefile b/Makefile index 8eb3721c1..b2f480005 100644 --- a/Makefile +++ b/Makefile @@ -171,6 +171,8 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml + $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml touch $(E2E_OUT_DIR)/clusterclass.yaml .PHONY: clusterclass-templates diff --git a/test/e2e/config/e2e_conf.yaml b/test/e2e/config/e2e_conf.yaml index 9fcea4d52..22d3c39d7 100644 --- a/test/e2e/config/e2e_conf.yaml +++ b/test/e2e/config/e2e_conf.yaml @@ -117,6 +117,10 @@ providers: targetName: "cluster-template-centos.yaml" - sourcePath: "../_out/clusterclass.yaml" targetName: "clusterclass-test-clusterclass.yaml" + - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml" + targetName: "cluster-template-centos-md-remediation.yaml" + - sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml" + targetName: "cluster-template-ubuntu-md-remediation.yaml" variables: CNI: "/tmp/calico.yaml" diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/kustomization.yaml new file mode 100644 index 000000000..276c554b2 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- ../bases/ippool +- ../bases/centos-kubeadm-config + +- mhc.yaml +patchesStrategicMerge: +- md.yaml diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/md.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/md.yaml new file mode 100644 index 000000000..e96eece1c --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/md.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: cluster.x-k8s.io/${CAPI_VERSION} +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + nodepool: nodepool-0 + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + metadata: + labels: + "e2e.remediation.label": "" diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/mhc.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/mhc.yaml new file mode 100644 index 000000000..bb0ec12ea --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/mhc.yaml @@ -0,0 +1,19 @@ +--- +# MachineHealthCheck object with +# - a selector that targets all the machines with label e2e.remediation.label="" +# - unhealthyConditions triggering remediation after 10s the condition is set +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" + namespace: ${NAMESPACE} +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + e2e.remediation.label: "" + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 10s diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/kustomization.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/kustomization.yaml new file mode 100644 index 000000000..27d42f978 --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/kustomization.yaml @@ -0,0 +1,7 @@ +resources: +- ../bases/ippool +- ../bases/ubuntu-kubeadm-config + +- mhc.yaml +patchesStrategicMerge: +- md.yaml diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/md.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/md.yaml new file mode 100644 index 000000000..e96eece1c --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/md.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: cluster.x-k8s.io/${CAPI_VERSION} +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + nodepool: nodepool-0 + name: ${CLUSTER_NAME} + namespace: ${NAMESPACE} +spec: + template: + metadata: + labels: + "e2e.remediation.label": "" diff --git a/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/mhc.yaml b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/mhc.yaml new file mode 100644 index 000000000..bb0ec12ea --- /dev/null +++ b/test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/mhc.yaml @@ -0,0 +1,19 @@ +--- +# MachineHealthCheck object with +# - a selector that targets all the machines with label e2e.remediation.label="" +# - unhealthyConditions triggering remediation after 10s the condition is set +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineHealthCheck +metadata: + name: "${CLUSTER_NAME}-mhc-0" + namespace: ${NAMESPACE} +spec: + clusterName: "${CLUSTER_NAME}" + maxUnhealthy: 100% + selector: + matchLabels: + e2e.remediation.label: "" + unhealthyConditions: + - type: e2e.remediation.condition + status: "False" + timeout: 10s diff --git a/test/e2e/md_remediations_test.go b/test/e2e/md_remediations_test.go new file mode 100644 index 000000000..b35392e52 --- /dev/null +++ b/test/e2e/md_remediations_test.go @@ -0,0 +1,56 @@ +package e2e + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/utils/ptr" + capi_e2e "sigs.k8s.io/cluster-api/test/e2e" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var _ = Describe("When testing MachineDeployment remediation [healthcheck] [remediation] [features]", Label("healthcheck", "remediation", "features"), func() { + BeforeEach(func() { + osType := strings.ToLower(os.Getenv("OS")) + Expect(osType).ToNot(Equal("")) + validateGlobals(specName) + // We need to override clusterctl apply log folder to avoid getting our credentials exposed. + clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName()) + }) + capi_e2e.MachineDeploymentRemediationSpec(ctx, func() capi_e2e.MachineDeploymentRemediationSpecInput { + return capi_e2e.MachineDeploymentRemediationSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + PostNamespaceCreated: postNamespaceCreated, + Flavor: ptr.To(fmt.Sprintf("%s-md-remediation", osType)), + } + }) + AfterEach(func() { + // Recreate bmh that was used in capi namespace in metal3 + //#nosec G204 -- We need to pass in the file name here. + cmd := exec.Command("bash", "-c", "kubectl apply -f bmhosts_crs.yaml -n metal3") + cmd.Dir = workDir + output, err := cmd.CombinedOutput() + Logf("Applying bmh to metal3 namespace : \n %v", string(output)) + Expect(err).ToNot(HaveOccurred()) + // wait for all bmh to become available + bootstrapClient := bootstrapClusterProxy.GetClient() + ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace)) + WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{ + Client: bootstrapClient, + Options: []client.ListOption{client.InNamespace(namespace)}, + Replicas: 4, + Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-available"), + }) + ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace)) + }) +})