debug

Signed-off-by: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com>

Makefile

@@ -161,8 +161,10 @@ $(KIND_KUBECONFIG):
 	kind create cluster --name $(KIND_NAME) $(KIND_ARGS)
 	kind get kubeconfig --name $(KIND_NAME) > $(KIND_KUBECONFIG)
 	kind get kubeconfig --name $(KIND_NAME) --internal > $(KIND_KUBECONFIG_INTERNAL)
-	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.64.1/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml
-	KUBECONFIG=$(KIND_KUBECONFIG) kubectl create -f https://raw.githubusercontent.com/openshift/api/release-4.12/route/v1/route.crd.yaml
+	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f \
+		https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.64.1/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml
+	KUBECONFIG=$(KIND_KUBECONFIG) kubectl create -f \
+		https://raw.githubusercontent.com/openshift/api/release-4.12/route/v1/route.crd.yaml
 
 .PHONY: kind-delete-cluster
 kind-delete-cluster: ## Delete a kind cluster.
@@ -183,16 +185,25 @@ kind-deploy-registration-operator-hub: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy
 	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager -n open-cluster-management -c condition=Available -m 90
 	KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager-placement-controller -n open-cluster-management-hub -c condition=Available -m 90
 	@echo installing Policy CRD on hub
-	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/open-cluster-management-io/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml
+	KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f \
+		https://raw.githubusercontent.com/open-cluster-management-io/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml
 
 .PHONY: kind-deploy-registration-operator-managed
 kind-deploy-registration-operator-managed: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster.
-	cd $(OCM_REPO) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-operator
-	cd $(OCM_REPO) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make apply-spoke-cr
+	cd $(OCM_REPO) && \
+	KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \
+	KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-operator
+	cd $(OCM_REPO) && \
+	KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \
+	KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make apply-spoke-cr
 
 .PHONY: kind-deploy-registration-operator-managed-hosted
 kind-deploy-registration-operator-managed-hosted: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster in hosted mode.
-	cd $(OCM_REPO) && KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-hosted
+	cd $(OCM_REPO) && \
+	$(SED) -i 's/mode: Singleton/mode: SingletonHosted/' test/e2e-test.mk && \
+	KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \
+	HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) \
+	KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-hosted
 
 .PHONY: kind-approve-cluster
 kind-approve-cluster: $(KIND_KUBECONFIG) ## Approve managed cluster in the kind cluster.

go.mod

@@ -126,7 +126,7 @@ require (
 	k8s.io/api v0.29.0 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
 	k8s.io/apiserver v0.29.0 // indirect
-	k8s.io/klog v1.0.0 // indirect
+	k8s.io/klog v1.0.0
 	k8s.io/kms v0.29.0 // indirect
 	k8s.io/kube-aggregator v0.29.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect

pkg/addon/common.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/klog"
 	"open-cluster-management.io/addon-framework/pkg/addonmanager"
 	"open-cluster-management.io/addon-framework/pkg/agent"
 	"open-cluster-management.io/addon-framework/pkg/utils"
@@ -190,7 +191,11 @@ func (pa *PolicyAgentAddon) Manifests(
 			return nil, err
 		}
 
+		klog.Infof("HOSTING CLUSTER: %+v\n", hostingCluster)
+
 		pa.hostingCluster = hostingCluster
+
+		klog.Infof("AGENT_ADDON: %+v\n", pa)
 	}
 
 	return pa.AgentAddon.Manifests(cluster, addon)

test/e2e/case1_framework_deployment_test.go

@@ -35,6 +35,10 @@ var _ = Describe("Test framework deployment", Ordered, func() {
 	})
 
 	AfterAll(func() {
+		if CurrentSpecReport().Failed() {
+			debugCollection(case1PodSelector)
+		}
+
 		By("Deleting the default governance-policy-framework ClusterManagementAddon from the hub cluster")
 		Kubectl("delete", "-f", case1ClusterManagementAddOnCRDefault)
 	})
@@ -718,7 +722,7 @@ func checkContainersAndAvailabilityInNamespace(cluster managedClusterConfig, clu
 			g.Expect(err).ToNot(HaveOccurred())
 
 			g.Expect(available).To(Equal(replicas), "available replicas should equal expected replicas")
-		}, 240, 1).Should(Succeed())
+		}, 300, 1).Should(Succeed())
 	}
 
 	By(logPrefix + "verifying one framework pod is running")

test/e2e/case2_config_deployment_test.go

@@ -62,7 +62,7 @@ func verifyConfigPolicyDeployment(
 			g.Expect(err).ToNot(HaveOccurred())
 
 			g.Expect(available).To(Equal(replicas), "available replicas should equal expected replicas")
-		}, 240, 1).Should(Succeed())
+		}, 300, 1).Should(Succeed())
 	}
 
 	By(logPrefix + "verifying a running config-policy-controller pod")
@@ -94,6 +94,10 @@ var _ = Describe("Test config-policy-controller deployment", Ordered, func() {
 	})
 
 	AfterAll(func() {
+		if CurrentSpecReport().Failed() {
+			debugCollection(case2PodSelector)
+		}
+
 		By("Deleting the default config-policy-controller ClusterManagementAddon from the hub cluster")
 		Kubectl("delete", "-f", case2ClusterManagementAddOnCRDefault)
 	})

test/e2e/utils_test.go

@@ -179,3 +179,40 @@ func getAddonStatus(addon *unstructured.Unstructured) bool {
 
 	return false
 }
+
+func debugCollection(podSelector string) {
+	By("Recording debug logs")
+
+	output := Kubectl("get", "all", "-n", addonNamespace)
+	output += "===\n"
+
+	for i, cluster := range managedClusterList {
+		for _, suffix := range []string{"", "-hosted"} {
+			managedClusterNamespace := cluster.clusterName + suffix
+			output += fmt.Sprintf("All objects in namespace %s:\n", managedClusterNamespace)
+			output += Kubectl("get", "all", "-n", managedClusterNamespace)
+			output += Kubectl("get", "secrets", "-n", managedClusterNamespace)
+			output += "===\n"
+			output += fmt.Sprintf("Pod logs for label %s in namespace %s:\n", podSelector, managedClusterNamespace)
+			output += Kubectl("describe", "pod", "-n", managedClusterNamespace, "-l", podSelector)
+			output += Kubectl("logs", "-n", managedClusterNamespace, "-l", podSelector, "--ignore-errors")
+			output += "===\n"
+		}
+
+		output += fmt.Sprintf("All objects in namespace %s:\n", addonNamespace)
+		output += Kubectl("get", "all", "-n", addonNamespace)
+		output += Kubectl("get", "secrets", "-n", addonNamespace)
+		output += fmt.Sprintf("Pod logs for label %s in namespace %s for cluster %s:\n",
+			podSelector, addonNamespace, cluster.clusterName)
+		output += Kubectl(
+			"describe", "pod", "-n", addonNamespace, "-l", podSelector,
+			fmt.Sprintf("--kubeconfig=%s%d_e2e", kubeconfigFilename, i+1),
+		)
+		output += Kubectl(
+			"logs", "-n", addonNamespace, "-l", podSelector, "--ignore-errors",
+			fmt.Sprintf("--kubeconfig=%s%d_e2e", kubeconfigFilename, i+1),
+		)
+	}
+
+	GinkgoWriter.Print(output)
+}