From 75bf6683360fd737645e6eff180e5d1f5ef3747c Mon Sep 17 00:00:00 2001 From: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com> Date: Fri, 8 Mar 2024 11:20:24 -0500 Subject: [PATCH] debug Signed-off-by: Dale Haiducek <19750917+dhaiducek@users.noreply.github.com> --- Makefile | 23 +++++++++---- go.mod | 2 +- pkg/addon/common.go | 5 +++ test/e2e/case1_framework_deployment_test.go | 6 +++- test/e2e/case2_config_deployment_test.go | 6 +++- test/e2e/utils_test.go | 37 +++++++++++++++++++++ 6 files changed, 70 insertions(+), 9 deletions(-) diff --git a/Makefile b/Makefile index 78113e1f..37871836 100644 --- a/Makefile +++ b/Makefile @@ -161,8 +161,10 @@ $(KIND_KUBECONFIG): kind create cluster --name $(KIND_NAME) $(KIND_ARGS) kind get kubeconfig --name $(KIND_NAME) > $(KIND_KUBECONFIG) kind get kubeconfig --name $(KIND_NAME) --internal > $(KIND_KUBECONFIG_INTERNAL) - KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.64.1/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml - KUBECONFIG=$(KIND_KUBECONFIG) kubectl create -f https://raw.githubusercontent.com/openshift/api/release-4.12/route/v1/route.crd.yaml + KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f \ + https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.64.1/example/prometheus-operator-crd-full/monitoring.coreos.com_servicemonitors.yaml + KUBECONFIG=$(KIND_KUBECONFIG) kubectl create -f \ + https://raw.githubusercontent.com/openshift/api/release-4.12/route/v1/route.crd.yaml .PHONY: kind-delete-cluster kind-delete-cluster: ## Delete a kind cluster. @@ -183,16 +185,25 @@ kind-deploy-registration-operator-hub: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager -n open-cluster-management -c condition=Available -m 90 KUBECONFIG=$(KIND_KUBECONFIG) $(KUBEWAIT) -r deploy/cluster-manager-placement-controller -n open-cluster-management-hub -c condition=Available -m 90 @echo installing Policy CRD on hub - KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f https://raw.githubusercontent.com/open-cluster-management-io/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml + KUBECONFIG=$(KIND_KUBECONFIG) kubectl apply -f \ + https://raw.githubusercontent.com/open-cluster-management-io/governance-policy-propagator/main/deploy/crds/policy.open-cluster-management.io_policies.yaml .PHONY: kind-deploy-registration-operator-managed kind-deploy-registration-operator-managed: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster. - cd $(OCM_REPO) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-operator - cd $(OCM_REPO) && KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make apply-spoke-cr + cd $(OCM_REPO) && \ + KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \ + KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-operator + cd $(OCM_REPO) && \ + KUBECONFIG=$(KIND_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \ + KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make apply-spoke-cr .PHONY: kind-deploy-registration-operator-managed-hosted kind-deploy-registration-operator-managed-hosted: $(OCM_REPO) $(KIND_KUBECONFIG) ## Deploy the ocm registration operator to the kind cluster in hosted mode. - cd $(OCM_REPO) && KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-hosted + cd $(OCM_REPO) && \ + $(SED) -i 's/mode: Singleton/mode: SingletonHosted/' test/e2e-test.mk && \ + KUBECONFIG=$(HUB_KUBECONFIG) MANAGED_CLUSTER_NAME=$(CLUSTER_NAME) HUB_KUBECONFIG=$(HUB_KUBECONFIG_INTERNAL) \ + HOSTED_CLUSTER_MANAGER_NAME=$(HUB_CLUSTER_NAME) EXTERNAL_MANAGED_KUBECONFIG=$(KIND_KUBECONFIG_INTERNAL) \ + KUSTOMIZE_VERSION=$(KUSTOMIZE_VERSION_CLEAN) make deploy-spoke-hosted .PHONY: kind-approve-cluster kind-approve-cluster: $(KIND_KUBECONFIG) ## Approve managed cluster in the kind cluster. diff --git a/go.mod b/go.mod index 3d0d8715..d6cba836 100644 --- a/go.mod +++ b/go.mod @@ -126,7 +126,7 @@ require ( k8s.io/api v0.29.0 // indirect k8s.io/apiextensions-apiserver v0.29.0 // indirect k8s.io/apiserver v0.29.0 // indirect - k8s.io/klog v1.0.0 // indirect + k8s.io/klog v1.0.0 k8s.io/kms v0.29.0 // indirect k8s.io/kube-aggregator v0.29.0 // indirect k8s.io/kube-openapi v0.0.0-20231010175941-2dd684a91f00 // indirect diff --git a/pkg/addon/common.go b/pkg/addon/common.go index caecce34..41a04d53 100644 --- a/pkg/addon/common.go +++ b/pkg/addon/common.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" + "k8s.io/klog" "open-cluster-management.io/addon-framework/pkg/addonmanager" "open-cluster-management.io/addon-framework/pkg/agent" "open-cluster-management.io/addon-framework/pkg/utils" @@ -190,7 +191,11 @@ func (pa *PolicyAgentAddon) Manifests( return nil, err } + klog.Infof("HOSTING CLUSTER: %+v\n", hostingCluster) + pa.hostingCluster = hostingCluster + + klog.Infof("AGENT_ADDON: %+v\n", pa) } return pa.AgentAddon.Manifests(cluster, addon) diff --git a/test/e2e/case1_framework_deployment_test.go b/test/e2e/case1_framework_deployment_test.go index 6f0da73e..a5d1cc24 100644 --- a/test/e2e/case1_framework_deployment_test.go +++ b/test/e2e/case1_framework_deployment_test.go @@ -35,6 +35,10 @@ var _ = Describe("Test framework deployment", Ordered, func() { }) AfterAll(func() { + if CurrentSpecReport().Failed() { + debugCollection(case1PodSelector) + } + By("Deleting the default governance-policy-framework ClusterManagementAddon from the hub cluster") Kubectl("delete", "-f", case1ClusterManagementAddOnCRDefault) }) @@ -718,7 +722,7 @@ func checkContainersAndAvailabilityInNamespace(cluster managedClusterConfig, clu g.Expect(err).ToNot(HaveOccurred()) g.Expect(available).To(Equal(replicas), "available replicas should equal expected replicas") - }, 240, 1).Should(Succeed()) + }, 300, 1).Should(Succeed()) } By(logPrefix + "verifying one framework pod is running") diff --git a/test/e2e/case2_config_deployment_test.go b/test/e2e/case2_config_deployment_test.go index 5a0c2fd5..44630dc0 100644 --- a/test/e2e/case2_config_deployment_test.go +++ b/test/e2e/case2_config_deployment_test.go @@ -62,7 +62,7 @@ func verifyConfigPolicyDeployment( g.Expect(err).ToNot(HaveOccurred()) g.Expect(available).To(Equal(replicas), "available replicas should equal expected replicas") - }, 240, 1).Should(Succeed()) + }, 300, 1).Should(Succeed()) } By(logPrefix + "verifying a running config-policy-controller pod") @@ -94,6 +94,10 @@ var _ = Describe("Test config-policy-controller deployment", Ordered, func() { }) AfterAll(func() { + if CurrentSpecReport().Failed() { + debugCollection(case2PodSelector) + } + By("Deleting the default config-policy-controller ClusterManagementAddon from the hub cluster") Kubectl("delete", "-f", case2ClusterManagementAddOnCRDefault) }) diff --git a/test/e2e/utils_test.go b/test/e2e/utils_test.go index 51ad2c58..e418634c 100644 --- a/test/e2e/utils_test.go +++ b/test/e2e/utils_test.go @@ -179,3 +179,40 @@ func getAddonStatus(addon *unstructured.Unstructured) bool { return false } + +func debugCollection(podSelector string) { + By("Recording debug logs") + + output := Kubectl("get", "all", "-n", addonNamespace) + output += "===\n" + + for i, cluster := range managedClusterList { + for _, suffix := range []string{"", "-hosted"} { + managedClusterNamespace := cluster.clusterName + suffix + output += fmt.Sprintf("All objects in namespace %s:\n", managedClusterNamespace) + output += Kubectl("get", "all", "-n", managedClusterNamespace) + output += Kubectl("get", "secrets", "-n", managedClusterNamespace) + output += "===\n" + output += fmt.Sprintf("Pod logs for label %s in namespace %s:\n", podSelector, managedClusterNamespace) + output += Kubectl("describe", "pod", "-n", managedClusterNamespace, "-l", podSelector) + output += Kubectl("logs", "-n", managedClusterNamespace, "-l", podSelector, "--ignore-errors") + output += "===\n" + } + + output += fmt.Sprintf("All objects in namespace %s:\n", addonNamespace) + output += Kubectl("get", "all", "-n", addonNamespace) + output += Kubectl("get", "secrets", "-n", addonNamespace) + output += fmt.Sprintf("Pod logs for label %s in namespace %s for cluster %s:\n", + podSelector, addonNamespace, cluster.clusterName) + output += Kubectl( + "describe", "pod", "-n", addonNamespace, "-l", podSelector, + fmt.Sprintf("--kubeconfig=%s%d_e2e", kubeconfigFilename, i+1), + ) + output += Kubectl( + "logs", "-n", addonNamespace, "-l", podSelector, "--ignore-errors", + fmt.Sprintf("--kubeconfig=%s%d_e2e", kubeconfigFilename, i+1), + ) + } + + GinkgoWriter.Print(output) +}