🌱 Add md_remediation test

Makefile

@@ -171,6 +171,8 @@ cluster-templates: $(KUSTOMIZE) ## Generate cluster templates
  $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu > $(E2E_OUT_DIR)/cluster-template-ubuntu.yaml
  $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos > $(E2E_OUT_DIR)/cluster-template-centos.yaml
  $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-upgrade-workload > $(E2E_OUT_DIR)/cluster-template-upgrade-workload.yaml
+ $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-centos-md-remediation > $(E2E_OUT_DIR)/cluster-template-centos-md-remediation.yaml
+ $(KUSTOMIZE) build $(E2E_TEMPLATES_DIR)/cluster-template-ubuntu-md-remediation > $(E2E_OUT_DIR)/cluster-template-ubuntu-md-remediation.yaml
  touch $(E2E_OUT_DIR)/clusterclass.yaml
 
 .PHONY: clusterclass-templates

test/e2e/config/e2e_conf.yaml

@@ -117,6 +117,10 @@ providers:
  targetName: "cluster-template-centos.yaml"
  - sourcePath: "../_out/clusterclass.yaml"
  targetName: "clusterclass-test-clusterclass.yaml"
+ - sourcePath: "../_out/cluster-template-centos-md-remediation.yaml"
+ targetName: "cluster-template-centos-md-remediation.yaml"
+ - sourcePath: "../_out/cluster-template-ubuntu-md-remediation.yaml"
+ targetName: "cluster-template-ubuntu-md-remediation.yaml"
 
 variables:
  CNI: "/tmp/calico.yaml"

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- ../bases/ippool
+- ../bases/centos-kubeadm-config
+
+- mhc.yaml
+patchesStrategicMerge:
+- md.yaml

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/md.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.x-k8s.io/${CAPI_VERSION}
+kind: MachineDeployment
+metadata:
+ labels:
+ cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+ nodepool: nodepool-0
+ name: ${CLUSTER_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ template:
+ metadata:
+ labels:
+ "e2e.remediation.label": ""

test/e2e/data/infrastructure-metal3/cluster-template-centos-md-remediation/mhc.yaml

@@ -0,0 +1,19 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label e2e.remediation.label=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineHealthCheck
+metadata:
+ name: "${CLUSTER_NAME}-mhc-0"
+ namespace: ${NAMESPACE}
+spec:
+ clusterName: "${CLUSTER_NAME}"
+ maxUnhealthy: 100%
+ selector:
+ matchLabels:
+ e2e.remediation.label: ""
+ unhealthyConditions:
+ - type: e2e.remediation.condition
+ status: "False"
+ timeout: 10s

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+- ../bases/ippool
+- ../bases/ubuntu-kubeadm-config
+
+- mhc.yaml
+patchesStrategicMerge:
+- md.yaml

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/md.yaml

@@ -0,0 +1,14 @@
+---
+apiVersion: cluster.x-k8s.io/${CAPI_VERSION}
+kind: MachineDeployment
+metadata:
+ labels:
+ cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+ nodepool: nodepool-0
+ name: ${CLUSTER_NAME}
+ namespace: ${NAMESPACE}
+spec:
+ template:
+ metadata:
+ labels:
+ "e2e.remediation.label": ""

test/e2e/data/infrastructure-metal3/cluster-template-ubuntu-md-remediation/mhc.yaml

@@ -0,0 +1,19 @@
+---
+# MachineHealthCheck object with
+# - a selector that targets all the machines with label e2e.remediation.label=""
+# - unhealthyConditions triggering remediation after 10s the condition is set
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineHealthCheck
+metadata:
+ name: "${CLUSTER_NAME}-mhc-0"
+ namespace: ${NAMESPACE}
+spec:
+ clusterName: "${CLUSTER_NAME}"
+ maxUnhealthy: 100%
+ selector:
+ matchLabels:
+ e2e.remediation.label: ""
+ unhealthyConditions:
+ - type: e2e.remediation.condition
+ status: "False"
+ timeout: 10s

test/e2e/md_remediations_test.go

@@ -0,0 +1,56 @@
+package e2e
+
+import (
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+
+ bmov1alpha1 "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "k8s.io/utils/ptr"
+ capi_e2e "sigs.k8s.io/cluster-api/test/e2e"
+ "sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+var _ = Describe("When testing MachineDeployment remediation [healthcheck] [remediation] [features]", Label("healthcheck", "remediation", "features"), func() {
+ BeforeEach(func() {
+ osType := strings.ToLower(os.Getenv("OS"))
+ Expect(osType).ToNot(Equal(""))
+ validateGlobals(specName)
+ // We need to override clusterctl apply log folder to avoid getting our credentials exposed.
+ clusterctlLogFolder = filepath.Join(os.TempDir(), "clusters", bootstrapClusterProxy.GetName())
+ })
+ capi_e2e.MachineDeploymentRemediationSpec(ctx, func() capi_e2e.MachineDeploymentRemediationSpecInput {
+ return capi_e2e.MachineDeploymentRemediationSpecInput{
+ E2EConfig: e2eConfig,
+ ClusterctlConfigPath: clusterctlConfigPath,
+ BootstrapClusterProxy: bootstrapClusterProxy,
+ ArtifactFolder: artifactFolder,
+ SkipCleanup: skipCleanup,
+ PostNamespaceCreated: postNamespaceCreated,
+ Flavor: ptr.To(fmt.Sprintf("%s-md-remediation", osType)),
+ }
+ })
+ AfterEach(func() {
+ // Recreate bmh that was used in capi namespace in metal3
+ //#nosec G204 -- We need to pass in the file name here.
+ cmd := exec.Command("bash", "-c", "kubectl apply -f bmhosts_crs.yaml -n metal3")
+ cmd.Dir = workDir
+ output, err := cmd.CombinedOutput()
+ Logf("Applying bmh to metal3 namespace : \n %v", string(output))
+ Expect(err).ToNot(HaveOccurred())
+ // wait for all bmh to become available
+ bootstrapClient := bootstrapClusterProxy.GetClient()
+ ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
+ WaitForNumBmhInState(ctx, bmov1alpha1.StateAvailable, WaitForNumInput{
+ Client: bootstrapClient,
+ Options: []client.ListOption{client.InNamespace(namespace)},
+ Replicas: 4,
+ Intervals: e2eConfig.GetIntervals(specName, "wait-bmh-available"),
+ })
+ ListBareMetalHosts(ctx, bootstrapClient, client.InNamespace(namespace))
+ })
+})