From 952be59e5af60f603cff2a405dd8ef9386930944 Mon Sep 17 00:00:00 2001 From: Christian Schlotter Date: Fri, 23 Feb 2024 17:28:20 +0100 Subject: [PATCH] debug cert-manager --- test/e2e/clusterctl_upgrade_test.go | 60 ++++++++++--------- test/framework/alltypes_helpers.go | 7 ++- .../clusterctl/clusterctl_helpers.go | 31 ++++++++++ test/framework/controller_helpers.go | 18 ++++++ 4 files changed, 86 insertions(+), 30 deletions(-) diff --git a/test/e2e/clusterctl_upgrade_test.go b/test/e2e/clusterctl_upgrade_test.go index 652d35b07510..ec7823c50a29 100644 --- a/test/e2e/clusterctl_upgrade_test.go +++ b/test/e2e/clusterctl_upgrade_test.go @@ -26,35 +26,37 @@ import ( "sigs.k8s.io/cluster-api/test/framework" ) -var _ = Describe("When testing clusterctl upgrades (v0.4=>current)", func() { - ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput { - return ClusterctlUpgradeSpecInput{ - E2EConfig: e2eConfig, - ClusterctlConfigPath: clusterctlConfigPath, - BootstrapClusterProxy: bootstrapClusterProxy, - ArtifactFolder: artifactFolder, - SkipCleanup: skipCleanup, - InfrastructureProvider: pointer.String("docker"), - InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}", - InitWithProvidersContract: "v1alpha4", - // NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.4/bases. - InitWithKubernetesVersion: "v1.23.17", - WorkloadKubernetesVersion: "v1.23.17", - MgmtFlavor: "topology", - WorkloadFlavor: "", - // This check ensures that ownerReference apiVersions are updated for all types after the upgrade. - PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) { - framework.AssertOwnerReferences(namespace, proxy.GetKubeconfigPath(), - framework.CoreOwnerReferenceAssertion, - framework.ExpOwnerReferenceAssertions, - framework.DockerInfraOwnerReferenceAssertions, - framework.KubeadmBootstrapOwnerReferenceAssertions, - framework.KubeadmControlPlaneOwnerReferenceAssertions, - framework.KubernetesReferenceAssertions, - ) - }, - } - }) +var _ = FDescribe("When testing clusterctl upgrades (v0.4=>current)", func() { + for i := 0; i < 10; i++ { + ClusterctlUpgradeSpec(ctx, func() ClusterctlUpgradeSpecInput { + return ClusterctlUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: pointer.String("docker"), + InitWithBinary: "https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.4.8/clusterctl-{OS}-{ARCH}", + InitWithProvidersContract: "v1alpha4", + // NOTE: If this version is changed here the image and SHA must also be updated in all DockerMachineTemplates in `test/data/infrastructure-docker/v0.4/bases. + InitWithKubernetesVersion: "v1.23.17", + WorkloadKubernetesVersion: "v1.23.17", + MgmtFlavor: "topology", + WorkloadFlavor: "", + // This check ensures that ownerReference apiVersions are updated for all types after the upgrade. + PostUpgrade: func(proxy framework.ClusterProxy, namespace, clusterName string) { + framework.AssertOwnerReferences(namespace, proxy.GetKubeconfigPath(), + framework.CoreOwnerReferenceAssertion, + framework.ExpOwnerReferenceAssertions, + framework.DockerInfraOwnerReferenceAssertions, + framework.KubeadmBootstrapOwnerReferenceAssertions, + framework.KubeadmControlPlaneOwnerReferenceAssertions, + framework.KubernetesReferenceAssertions, + ) + }, + } + }) + } }) var _ = Describe("When testing clusterctl upgrades (v1.0=>current)", func() { diff --git a/test/framework/alltypes_helpers.go b/test/framework/alltypes_helpers.go index d5ce38bb5f5c..681271b5922e 100644 --- a/test/framework/alltypes_helpers.go +++ b/test/framework/alltypes_helpers.go @@ -88,8 +88,12 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { Eventually(func() error { return lister.List(ctx, crdList, capiProviderOptions()...) }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") + crdListCertManager := &apiextensionsv1.CustomResourceDefinitionList{} + Eventually(func() error { + return lister.List(ctx, crdListCertManager, client.MatchingLabels{"app": "cert-manager"}) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "failed to list CRDs for CAPI providers") - for _, crd := range crdList.Items { + for _, crd := range append(crdList.Items, crdListCertManager.Items...) { for _, version := range crd.Spec.Versions { if !version.Storage { continue @@ -104,6 +108,7 @@ func getClusterAPITypes(ctx context.Context, lister Lister) []metav1.TypeMeta { }) } } + return discoveredTypes } diff --git a/test/framework/clusterctl/clusterctl_helpers.go b/test/framework/clusterctl/clusterctl_helpers.go index d970a5e3f229..c9cbd8ff4628 100644 --- a/test/framework/clusterctl/clusterctl_helpers.go +++ b/test/framework/clusterctl/clusterctl_helpers.go @@ -18,6 +18,7 @@ package clusterctl import ( "context" + "fmt" "os" "path/filepath" "time" @@ -128,6 +129,36 @@ func InitManagementClusterAndWatchControllerLogs(ctx context.Context, input Init }) } } + + certManagerDeployments := framework.GetCertManagerDeployments(ctx, framework.GetControllerDeploymentsInput{ + Lister: client, + }) + for _, deployment := range certManagerDeployments { + fmt.Printf("Pod status of %q:\n%++v\n", deployment.Namespace, deployment.Name, deployment.Status) + // framework.WaitForDeploymentsAvailable(ctx, framework.WaitForDeploymentsAvailableInput{ + // Getter: client, + // Deployment: deployment, + // }, intervals...) + + // Start streaming logs from all controller providers + framework.WatchDeploymentLogsByName(ctx, framework.WatchDeploymentLogsByNameInput{ + GetLister: client, + Cache: input.ClusterProxy.GetCache(ctx), + ClientSet: input.ClusterProxy.GetClientSet(), + Deployment: deployment, + LogPath: filepath.Join(input.LogFolder, "logs", deployment.GetNamespace()), + }) + + // if !input.DisableMetricsCollection { + // framework.WatchPodMetrics(ctx, framework.WatchPodMetricsInput{ + // GetLister: client, + // ClientSet: input.ClusterProxy.GetClientSet(), + // Deployment: deployment, + // MetricsPath: filepath.Join(input.LogFolder, "metrics", deployment.GetNamespace()), + // }) + // } + } + } // UpgradeManagementClusterAndWaitInput is the input type for UpgradeManagementClusterAndWait. diff --git a/test/framework/controller_helpers.go b/test/framework/controller_helpers.go index a4327a37264d..28558ddb5a87 100644 --- a/test/framework/controller_helpers.go +++ b/test/framework/controller_helpers.go @@ -21,6 +21,7 @@ import ( . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // GetControllerDeploymentsInput is the input for GetControllerDeployments. @@ -46,6 +47,23 @@ func GetControllerDeployments(ctx context.Context, input GetControllerDeployment return deployments } +// GetControllerDeployments returns all the deployment for the cluster API controllers existing in a management cluster. +func GetCertManagerDeployments(ctx context.Context, input GetControllerDeploymentsInput) []*appsv1.Deployment { + deploymentList := &appsv1.DeploymentList{} + Eventually(func() error { + return input.Lister.List(ctx, deploymentList, client.InNamespace("cert-manager")) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to list deployments for the cluster API controllers") + + deployments := make([]*appsv1.Deployment, 0, len(deploymentList.Items)) + for i := range deploymentList.Items { + d := &deploymentList.Items[i] + if !skipDeployment(d, input.ExcludeNamespaces) { + deployments = append(deployments, d) + } + } + return deployments +} + func skipDeployment(d *appsv1.Deployment, excludeNamespaces []string) bool { if !d.DeletionTimestamp.IsZero() { return true