From b1f51e94962433e7d74b5afb972de363d3aeaeac Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 09:59:53 -0700
Subject: [PATCH 01/19] MLE-22135: change liveness probe

---
 pkg/k8sutil/scripts/liveness-probe.sh | 31 ---------------------------
 pkg/k8sutil/statefulset.go            |  7 ++++--
 2 files changed, 5 insertions(+), 33 deletions(-)
 delete mode 100644 pkg/k8sutil/scripts/liveness-probe.sh

diff --git a/pkg/k8sutil/scripts/liveness-probe.sh b/pkg/k8sutil/scripts/liveness-probe.sh
deleted file mode 100644
index 388d426..0000000
--- a/pkg/k8sutil/scripts/liveness-probe.sh
+++ /dev/null
@@ -1,31 +0,0 @@
-#!/bin/bash
-
-log () {
-    local TIMESTAMP=$(date +"%Y-%m-%d %T.%3N")
-    # Check to make sure pod doesn't terminate if PID value is empty for any reason
-    if [ -n "$pid" ]; then
-        echo "${TIMESTAMP} $@" > /proc/$pid/fd/1
-    fi
-}
-
-pid=$(pgrep -fn start.marklogic)
-
-# Check if ML service is running. Exit with 1 if it is other than running
-ml_status=$(/etc/init.d/MarkLogic status)
-
-if [[ "$ml_status" =~ "running" ]]; then
-    http_code=$(curl -o /tmp/probe_response.txt -s -w "%{http_code}" "http://${HOSTNAME}:8001/admin/v1/timestamp")
-    curl_code=$?
-    http_resp=$(cat /tmp/probe_response.txt)
-
-    if [[ $curl_code -ne 0 && $http_code -ne 401 ]]; then
-        log "Info: [Liveness Probe] Error with MarkLogic"
-        log "Info: [Liveness Probe] Curl response code: "$curl_code
-        log "Info: [Liveness Probe] Http response code: "$http_code
-        log "Info: [Liveness Probe] Http response message: "$http_resp 
-    fi
-    rm -f /tmp/probe_response.txt
-    exit 0
-else
-    exit 1
-fi
\ No newline at end of file
diff --git a/pkg/k8sutil/statefulset.go b/pkg/k8sutil/statefulset.go
index 8c690f3..080b805 100644
--- a/pkg/k8sutil/statefulset.go
+++ b/pkg/k8sutil/statefulset.go
@@ -696,8 +696,11 @@ func getLivenessProbe(probe marklogicv1.ContainerProbe) *corev1.Probe {
 		TimeoutSeconds:      probe.TimeoutSeconds,
 		SuccessThreshold:    probe.SuccessThreshold,
 		ProbeHandler: corev1.ProbeHandler{
-			Exec: &corev1.ExecAction{
-				Command: []string{"/bin/bash", "/tmp/helm-scripts/liveness-probe.sh"},
+			TCPSocket: &corev1.TCPSocketAction{
+				Port: intstr.IntOrString{
+					Type:   intstr.Int,
+					IntVal: 8001,
+				},
 			},
 		},
 	}

From 04cfc620c6443a6980d374f0b9dc7286714ff697 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 10:51:04 -0700
Subject: [PATCH 02/19] add retry logic to group configuration

---
 pkg/k8sutil/scripts/poststart-hook.sh | 23 +++++++++++++++++++++--
 1 file changed, 21 insertions(+), 2 deletions(-)

diff --git a/pkg/k8sutil/scripts/poststart-hook.sh b/pkg/k8sutil/scripts/poststart-hook.sh
index 0cbac3d..3abedba 100644
--- a/pkg/k8sutil/scripts/poststart-hook.sh
+++ b/pkg/k8sutil/scripts/poststart-hook.sh
@@ -53,6 +53,25 @@ log () {
     echo $message >> /tmp/script.log
 }
 
+# Function to retry a command based on the return code
+# $1: The number of retries
+# $2: The command to run
+retry() {
+    local retries=$1
+    shift
+    local count=0
+    until "$@"; do
+        exit_code=$?
+        count=$((count + 1))
+        if [ $count -ge $retries ]; then
+        echo "Command failed after $retries attempts."
+        return $exit_code
+        fi
+        echo "Attempt $count failed. Retrying..."
+        sleep 5
+    done
+}
+
 ###############################################################
 # Function to get the current host protocol
 # $1: The host name
@@ -699,10 +718,10 @@ if [[ "$IS_BOOTSTRAP_HOST" == "true" ]]; then
     if [[ "${MARKLOGIC_CLUSTER_TYPE}" == "bootstrap" ]]; then
         log "Info:  bootstrap host is ready"
         init_security_db
-        configure_group
+        retry 5 configure_group
     else 
         log "Info:  bootstrap host is ready"
-        configure_group
+        retry 5 configure_group
         join_cluster $HOST_FQDN
     fi
     configure_path_based_routing

From b40bc805fe0829829fcabc25770e980b972dc4b5 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 10:58:55 -0700
Subject: [PATCH 03/19] update haproxy image

---
 api/v1/common_types.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/api/v1/common_types.go b/api/v1/common_types.go
index 73d66d1..e9b05cc 100644
--- a/api/v1/common_types.go
+++ b/api/v1/common_types.go
@@ -82,7 +82,7 @@ type NetworkPolicy struct {
 }
 type HAProxy struct {
 	Enabled bool `json:"enabled,omitempty"`
-	// +kubebuilder:default:="haproxytech/haproxy-alpine:3.2"
+	// +kubebuilder:default:="haproxytech/haproxy-alpine:3.2.1"
 	Image            string                        `json:"image,omitempty"`
 	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 	// +kubebuilder:default:=1

From 60aa994a1e450a04b79c9383cacf68687d7b33bf Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 14:25:43 -0700
Subject: [PATCH 04/19] adjust testing time

---
 test/e2e/2_marklogic_cluster_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/e2e/2_marklogic_cluster_test.go b/test/e2e/2_marklogic_cluster_test.go
index a8e34fc..f7a6146 100644
--- a/test/e2e/2_marklogic_cluster_test.go
+++ b/test/e2e/2_marklogic_cluster_test.go
@@ -228,7 +228,7 @@ func TestMarklogicCluster(t *testing.T) {
 		client := c.Client()
 
 		podName := "node-0"
-		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 180*time.Second)
+		err := utils.WaitForPod(ctx, t, client, mlNamespace, podName, 240*time.Second)
 		if err != nil {
 			t.Fatalf("Failed to wait for pod creation: %v", err)
 		}

From 46962f558be3a073b8f5762b61f6f6ae5f7507a7 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 22:43:58 -0700
Subject: [PATCH 05/19] update marklogic image version

---
 Makefile | 26 ++++++++++++++------------
 1 file changed, 14 insertions(+), 12 deletions(-)

diff --git a/Makefile b/Makefile
index 156b14e..232da13 100644
--- a/Makefile
+++ b/Makefile
@@ -11,7 +11,7 @@ VERIFY_HUGE_PAGES ?= false
 export E2E_DOCKER_IMAGE ?= $(IMG)
 export E2E_KUSTOMIZE_VERSION ?= $(KUSTOMIZE_VERSION)
 export E2E_CONTROLLER_TOOLS_VERSION ?= $(CONTROLLER_TOOLS_VERSION)
-export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.0
+export E2E_MARKLOGIC_IMAGE_VERSION ?= progressofficial/marklogic-db:11.3.1-ubi-rootless-2.1.3
 export E2E_KUBERNETES_VERSION ?= v1.31.0
 
 # ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
@@ -134,22 +134,23 @@ test: manifests generate fmt vet envtest ## Run tests.
 e2e-test: 
 	@echo "=====Check Huges pages test is enabled or not for e2e test"
 ifeq ($(VERIFY_HUGE_PAGES), true)
-	@echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
-	sudo sysctl -w vm.nr_hugepages=1280
+	# @echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
+	# sudo sysctl -w vm.nr_hugepages=1280
 
-	@echo "=====Restart minikube cluster to apply hugepages value"
-	minikube stop
-	minikube start
+	# @echo "=====Restart minikube cluster to apply hugepages value"
+	# minikube stop
+	# minikube start
 
 	@echo "=====Running e2e test including hugepages test"
-	go test -v -count=1 -timeout 30m ./test/e2e -verifyHugePages
+	# go test -v -count=1 -timeout 30m ./test/e2e -verifyHugePages
+	go test -v -count=1 -timeout 30m ./test/e2e -args --labels="type=cluster-test"
 
-	@echo "=====Resetting hugepages value to 0"
-	sudo sysctl -w vm.nr_hugepages=0
+	# @echo "=====Resetting hugepages value to 0"
+	# sudo sysctl -w vm.nr_hugepages=0
 
-	@echo "=====Restart minikube cluster"
-	minikube stop
-	minikube start
+	# @echo "=====Restart minikube cluster"
+	# minikube stop
+	# minikube start
 else
 	@echo "=====Running e2e test without hugepages test"
 	go test -v -count=1 -timeout 30m ./test/e2e
@@ -162,6 +163,7 @@ e2e-setup-minikube: kustomize controller-gen build docker-build
 	minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2
 	minikube addons enable ingress
 	minikube image load $(IMG)
+	minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION)
 
 .PHONY: e2e-cleanup-minikube
 e2e-cleanup-minikube:

From 73477c0152bf612f4e95ea604bfe3d822f811919 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 8 Jul 2025 23:03:21 -0700
Subject: [PATCH 06/19] remove image

---
 Makefile | 1 -
 1 file changed, 1 deletion(-)

diff --git a/Makefile b/Makefile
index 232da13..050d86f 100644
--- a/Makefile
+++ b/Makefile
@@ -163,7 +163,6 @@ e2e-setup-minikube: kustomize controller-gen build docker-build
 	minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2
 	minikube addons enable ingress
 	minikube image load $(IMG)
-	minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION)
 
 .PHONY: e2e-cleanup-minikube
 e2e-cleanup-minikube:

From 188d5e86d0297696c7ddbfc60f6fd717b45408a1 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 07:42:40 -0700
Subject: [PATCH 07/19] update image version in Jenkins

---
 Jenkinsfile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 681dbd3..67d3b90 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -164,7 +164,7 @@ pipeline {
     // }
 
     parameters {
-        string(name: 'dockerImage', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi:latest-11', description: 'Docker image to use for tests.', trim: true)
+        string(name: 'E2E_MARKLOGIC_IMAGE_VERSION', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:latest-12', description: 'Docker image to use for tests.', trim: true)
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
     }
 

From 521229c4631e8fbb25e3248289669d7d1244c76c Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 08:16:47 -0700
Subject: [PATCH 08/19] fix makefile for pipeline

---
 Makefile               | 1 +
 api/v1/common_types.go | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 050d86f..232da13 100644
--- a/Makefile
+++ b/Makefile
@@ -163,6 +163,7 @@ e2e-setup-minikube: kustomize controller-gen build docker-build
 	minikube start --driver=docker --kubernetes-version=$(E2E_KUBERNETES_VERSION) --memory=8192 --cpus=2
 	minikube addons enable ingress
 	minikube image load $(IMG)
+	minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION)
 
 .PHONY: e2e-cleanup-minikube
 e2e-cleanup-minikube:
diff --git a/api/v1/common_types.go b/api/v1/common_types.go
index e9b05cc..73d66d1 100644
--- a/api/v1/common_types.go
+++ b/api/v1/common_types.go
@@ -82,7 +82,7 @@ type NetworkPolicy struct {
 }
 type HAProxy struct {
 	Enabled bool `json:"enabled,omitempty"`
-	// +kubebuilder:default:="haproxytech/haproxy-alpine:3.2.1"
+	// +kubebuilder:default:="haproxytech/haproxy-alpine:3.2"
 	Image            string                        `json:"image,omitempty"`
 	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
 	// +kubebuilder:default:=1

From e53bd98c12b61fde1e335ccb21ebc16b3594f9a2 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 09:47:18 -0700
Subject: [PATCH 09/19] fix validate rule and image load

---
 Makefile                         | 2 ++
 api/v1/marklogiccluster_types.go | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 232da13..700fb4a 100644
--- a/Makefile
+++ b/Makefile
@@ -164,6 +164,8 @@ e2e-setup-minikube: kustomize controller-gen build docker-build
 	minikube addons enable ingress
 	minikube image load $(IMG)
 	minikube image load $(E2E_MARKLOGIC_IMAGE_VERSION)
+	minikube image load "docker.io/haproxytech/haproxy-alpine:3.2"
+	minikube image ls
 
 .PHONY: e2e-cleanup-minikube
 e2e-cleanup-minikube:
diff --git a/api/v1/marklogiccluster_types.go b/api/v1/marklogiccluster_types.go
index d213666..a255168 100644
--- a/api/v1/marklogiccluster_types.go
+++ b/api/v1/marklogiccluster_types.go
@@ -27,7 +27,7 @@ import (
 
 // MarklogicClusterSpec defines the desired state of MarklogicCluster
 
-// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
+// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || self.image.split(':')[0].matches('.*latest.*') || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
 type MarklogicClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file

From c9ab1d036f82355d57c7571e1a92915b847eb777 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 09:55:09 -0700
Subject: [PATCH 10/19] push changes from build

---
 config/crd/bases/marklogic.progress.com_marklogicclusters.yaml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
index c5b9d03..b7646dd 100644
--- a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
+++ b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
@@ -11037,7 +11037,8 @@ spec:
             - message: HAProxy and Pathbased Routing is enabled. PathBasedRouting
                 is only supported for MarkLogic 11.1 and above
               rule: '!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting
-                == true) || int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1])
+                == true) || self.image.split('':'')[0].matches(''.*latest.*'') ||
+                int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1])
                 >= 111'
           status:
             description: MarklogicClusterStatus defines the observed state of MarklogicCluster

From 62cc96b5668b899f250eabead4fe8bfd620b0a68 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 22:43:47 -0700
Subject: [PATCH 11/19] MLE-22902: fix operator pipeline bug

---
 Jenkinsfile           | 1 +
 test/e2e/main_test.go | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/Jenkinsfile b/Jenkinsfile
index 67d3b90..41b46ce 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -165,6 +165,7 @@ pipeline {
 
     parameters {
         string(name: 'E2E_MARKLOGIC_IMAGE_VERSION', defaultValue: 'ml-docker-db-dev-tierpoint.bed-artifactory.bedford.progress.com/marklogic/marklogic-server-ubi-rootless:latest-12', description: 'Docker image to use for tests.', trim: true)
+        string(name: 'IMG', defaultValue: 'testrepo/marklogic-operator-image-dev:internal', description: 'Docker image for Running Operator Container', trim: true)
         string(name: 'emailList', defaultValue: emailList, description: 'List of email for build notification', trim: true)
     }
 
diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go
index 4af29f9..29be6b7 100644
--- a/test/e2e/main_test.go
+++ b/test/e2e/main_test.go
@@ -94,8 +94,8 @@ func TestMain(m *testing.M) {
 			log.Println("Deploying controller-manager resources...")
 			p := utils.RunCommand(`kubectl version`)
 			log.Printf("Output of kubectl: %s", p.Result())
-			p = utils.RunCommand(`bash -c "kustomize build config/default | kubectl apply --server-side -f -"`)
-			log.Printf("Output: %s", p.Result())
+			p = utils.RunCommand(`make deploy`)
+			log.Printf("Output of make deploy: %s", p.Result())
 			if p.Err() != nil {
 				log.Printf("Failed to deploy resource configurations: %s: %s", p.Err(), p.Result())
 				return ctx, p.Err()

From 8f655ef8ed60202f7745a77869ee7ba3cd0fb194 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Thu, 10 Jul 2025 23:23:36 -0700
Subject: [PATCH 12/19] ignore docker cache

---
 Makefile | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 700fb4a..a31ccc4 100644
--- a/Makefile
+++ b/Makefile
@@ -65,7 +65,8 @@ OPERATOR_SDK_VERSION ?= v1.34.2
 
 # Image URL to use all building/pushing image targets
 # Image for dev: ml-marklogic-operator-dev.bed-artifactory.bedford.progress.com/marklogic-operator-kubernetes
-IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION)
+# IMG ?= progressofficial/marklogic-operator-kubernetes:$(VERSION)
+IMG = "testrepo/marklogic-operator-image-dev:1.0.0"
 
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -204,7 +205,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager. to build for linux, add --platform="linux/amd64"
-	$(CONTAINER_TOOL) buildx build -t ${IMG} .
+	$(CONTAINER_TOOL) buildx build --no-cache -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.

From be4bb572f9c1f0dd315df0c452e6e9037acebd96 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Fri, 11 Jul 2025 09:56:45 -0700
Subject: [PATCH 13/19] fix validating rule

---
 api/v1/marklogiccluster_types.go                               | 2 +-
 config/crd/bases/marklogic.progress.com_marklogicclusters.yaml | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/api/v1/marklogiccluster_types.go b/api/v1/marklogiccluster_types.go
index a255168..5453f7b 100644
--- a/api/v1/marklogiccluster_types.go
+++ b/api/v1/marklogiccluster_types.go
@@ -27,7 +27,7 @@ import (
 
 // MarklogicClusterSpec defines the desired state of MarklogicCluster
 
-// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || self.image.split(':')[0].matches('.*latest.*') || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
+// +kubebuilder:validation:XValidation:rule="!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting == true) || self.image.split(':')[1].matches('.*latest.*') || int(self.image.split(':')[1].split('.')[0] + self.image.split(':')[1].split('.')[1]) >= 111", message="HAProxy and Pathbased Routing is enabled. PathBasedRouting is only supported for MarkLogic 11.1 and above"
 type MarklogicClusterSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
diff --git a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
index b7646dd..8eee952 100644
--- a/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
+++ b/config/crd/bases/marklogic.progress.com_marklogicclusters.yaml
@@ -11037,7 +11037,7 @@ spec:
             - message: HAProxy and Pathbased Routing is enabled. PathBasedRouting
                 is only supported for MarkLogic 11.1 and above
               rule: '!(self.haproxy.enabled == true && self.haproxy.pathBasedRouting
-                == true) || self.image.split('':'')[0].matches(''.*latest.*'') ||
+                == true) || self.image.split('':'')[1].matches(''.*latest.*'') ||
                 int(self.image.split('':'')[1].split(''.'')[0] + self.image.split('':'')[1].split(''.'')[1])
                 >= 111'
           status:

From 6a7df0e39c0173a77c68189e8f459f75762a22ed Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Mon, 14 Jul 2025 09:59:12 -0700
Subject: [PATCH 14/19] add debug message for Jenkins Pipeline

---
 test/utils/utils.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/test/utils/utils.go b/test/utils/utils.go
index f01e627..787c624 100644
--- a/test/utils/utils.go
+++ b/test/utils/utils.go
@@ -154,6 +154,9 @@ func WaitForPod(ctx context.Context, t *testing.T, client klient.Client, namespa
 	pod := &corev1.Pod{}
 	p := utils.RunCommand(`kubectl get ns`)
 	t.Logf("Kubernetes namespace: %s", p.Result())
+	p = utils.RunCommand("kubectl get pods --namespace " + "marklogic-operator-system" + " -o wide")
+	t.Logf("Kubernetes Operator Running Status: %s", p.Result())
+
 	for {
 		t.Logf("Waiting for pod %s in namespace %s ", podName, namespace)
 		p := utils.RunCommand("kubectl get pods --namespace " + namespace)

From 931d4e739d4b62a1fceb9537b0ef3e17176d4766 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Mon, 14 Jul 2025 15:26:16 -0700
Subject: [PATCH 15/19] fix test with bug: index out of range [0] with length 0

---
 test/e2e/2_marklogic_cluster_test.go | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/test/e2e/2_marklogic_cluster_test.go b/test/e2e/2_marklogic_cluster_test.go
index f7a6146..4e3986e 100644
--- a/test/e2e/2_marklogic_cluster_test.go
+++ b/test/e2e/2_marklogic_cluster_test.go
@@ -134,14 +134,17 @@ func TestMarklogicCluster(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to install grafana helm chart: %v", err)
 		}
-
+		// Wait for Grafana pod to be ready
+		time.Sleep(5 * time.Second) // Give some time for Grafana to start
 		podList := &corev1.PodList{}
 		if err := client.Resources().List(ctx, podList, func(lo *metav1.ListOptions) {
 			lo.FieldSelector = "metadata.namespace=" + "grafana"
 		}); err != nil {
 			t.Fatal(err)
 		}
-
+		if len(podList.Items) == 0 {
+			t.Fatal("No Grafana pods found")
+		}
 		grafanaPodName := podList.Items[0].Name
 		err = utils.WaitForPod(ctx, t, client, "grafana", grafanaPodName, 120*time.Second)
 		if err != nil {
@@ -245,6 +248,10 @@ func TestMarklogicCluster(t *testing.T) {
 		}); err != nil {
 			t.Fatal(err)
 		}
+		time.Sleep(5 * time.Second) // Wait for Grafana to be fully ready
+		if len(podList.Items) == 0 {
+			t.Fatal("No Grafana pods found")
+		}
 		grafanaPodName := podList.Items[0].Name
 		grafanaAdminUser, grafanaAdminPassword, err := utils.GetSecretData(ctx, client, "grafana", "grafana", "admin-user", "admin-password")
 		if err != nil {
@@ -337,7 +344,7 @@ func TestMarklogicCluster(t *testing.T) {
 			if err := client.Resources().Get(ctx, "marklogicclusters", mlNamespace, &mlcluster); err != nil {
 				t.Fatal(err)
 			}
-
+			
 			mlcluster.Spec.MarkLogicGroups[0].Resources = &resources
 			if err := client.Resources().Update(ctx, &mlcluster); err != nil {
 				t.Log("Failed to update MarkLogic group resources")

From 33de6c08f6c74d8d03cb2b658c00301f797864f2 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Mon, 14 Jul 2025 16:01:09 -0700
Subject: [PATCH 16/19] fix test bug with index out of range [1] with length 1

---
 test/e2e/4_tls_test.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/test/e2e/4_tls_test.go b/test/e2e/4_tls_test.go
index a645313..3e85d54 100644
--- a/test/e2e/4_tls_test.go
+++ b/test/e2e/4_tls_test.go
@@ -240,6 +240,9 @@ func TestTlsWithNamedCert(t *testing.T) {
 		}
 		certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array()
 		t.Log("Certificates URL list", certURIs)
+		if len(certURIs) < 2 {
+			t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs))
+		}
 		cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0])
 		cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1])
 		command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url)
@@ -406,6 +409,9 @@ func TestTlsWithMultiNode(t *testing.T) {
 		}
 		certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array()
 		t.Log("Dnode Cert Url", certURIs)
+		if len(certURIs) < 2 {
+			t.Fatalf("Expected at least 2 certificates, found %d", len(certURIs))
+		}
 		cert0Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[0])
 		cert1Url := fmt.Sprintf("https://localhost:8002%s?format=json", certURIs[1])
 		command = fmt.Sprintf("curl -k --anyauth -u %s:%s %s", adminUsername, adminPassword, cert0Url)

From c8626c8a0ec8c1645b588b490c74b5d5c80e5414 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 15 Jul 2025 06:41:24 -0700
Subject: [PATCH 17/19] add debug info

---
 test/e2e/4_tls_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/e2e/4_tls_test.go b/test/e2e/4_tls_test.go
index 3e85d54..b1c564e 100644
--- a/test/e2e/4_tls_test.go
+++ b/test/e2e/4_tls_test.go
@@ -407,6 +407,7 @@ func TestTlsWithMultiNode(t *testing.T) {
 		if err != nil {
 			t.Fatalf("Failed to get certificates list: %v", err)
 		}
+		t.Log("Certificates list", certs)
 		certURIs := gjson.Get(certs, `certificate-default-list.list-items.list-item.#.uriref`).Array()
 		t.Log("Dnode Cert Url", certURIs)
 		if len(certURIs) < 2 {

From 9a0a3e2b389e2d2f1b52e9c0cb314b49ef2affb5 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 15 Jul 2025 12:40:24 -0700
Subject: [PATCH 18/19] revert changes back to makefile

---
 Makefile | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/Makefile b/Makefile
index a31ccc4..6288a73 100644
--- a/Makefile
+++ b/Makefile
@@ -135,23 +135,22 @@ test: manifests generate fmt vet envtest ## Run tests.
 e2e-test: 
 	@echo "=====Check Huges pages test is enabled or not for e2e test"
 ifeq ($(VERIFY_HUGE_PAGES), true)
-	# @echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
-	# sudo sysctl -w vm.nr_hugepages=1280
+	@echo "=====Setting hugepages value to 1280 for hugepages-e2e test"
+	sudo sysctl -w vm.nr_hugepages=1280
 
-	# @echo "=====Restart minikube cluster to apply hugepages value"
-	# minikube stop
-	# minikube start
+	@echo "=====Restart minikube cluster to apply hugepages value"
+	minikube stop
+	minikube start
 
 	@echo "=====Running e2e test including hugepages test"
-	# go test -v -count=1 -timeout 30m ./test/e2e -verifyHugePages
-	go test -v -count=1 -timeout 30m ./test/e2e -args --labels="type=cluster-test"
+	go test -v -count=1 -timeout 30m ./test/e2e -verifyHugePages
 
 	# @echo "=====Resetting hugepages value to 0"
-	# sudo sysctl -w vm.nr_hugepages=0
+	sudo sysctl -w vm.nr_hugepages=0
 
 	# @echo "=====Restart minikube cluster"
-	# minikube stop
-	# minikube start
+	minikube stop
+	minikube start
 else
 	@echo "=====Running e2e test without hugepages test"
 	go test -v -count=1 -timeout 30m ./test/e2e
@@ -205,7 +204,7 @@ run: manifests generate fmt vet ## Run a controller from your host.
 # More info: https://docs.docker.com/develop/develop-images/build_enhancements/
 .PHONY: docker-build
 docker-build: ## Build docker image with the manager. to build for linux, add --platform="linux/amd64"
-	$(CONTAINER_TOOL) buildx build --no-cache -t ${IMG} .
+	$(CONTAINER_TOOL) buildx build -t ${IMG} .
 
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.

From 604f70569fcc9115ba3f7840edbef1ea19a416b2 Mon Sep 17 00:00:00 2001
From: Peng Zhou <peng.zhou@progress.com>
Date: Tue, 15 Jul 2025 12:40:46 -0700
Subject: [PATCH 19/19] uncomment changes

---
 Makefile | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/Makefile b/Makefile
index 6288a73..de1ee0c 100644
--- a/Makefile
+++ b/Makefile
@@ -145,10 +145,10 @@ ifeq ($(VERIFY_HUGE_PAGES), true)
 	@echo "=====Running e2e test including hugepages test"
 	go test -v -count=1 -timeout 30m ./test/e2e -verifyHugePages
 
-	# @echo "=====Resetting hugepages value to 0"
+	@echo "=====Resetting hugepages value to 0"
 	sudo sysctl -w vm.nr_hugepages=0
 
-	# @echo "=====Restart minikube cluster"
+	@echo "=====Restart minikube cluster"
 	minikube stop
 	minikube start
 else