From 1a3b22b31d21d02bc4dfb4e6b47bf7fac5096817 Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov Date: Mon, 9 Sep 2024 14:09:49 +0300 Subject: [PATCH] Replace use of `public.ecr.aws/nginx/nginx` with `ghcr.io/jitesoft/alpine` image --- docs/usage/registry-cache/configuration.md | 3 ++ test/common/common.go | 54 ++++++++++++------- ...create_enable_add_remove_disable_delete.go | 26 ++++----- ..._enabled_delete_shoot_system_components.go | 4 +- test/e2e/cache/create_enabled_force_delete.go | 4 +- ...eate_enabled_hibernate_reconcile_delete.go | 4 +- .../shoot/enable_disable_test.go | 6 +-- ...hibernate_reconcile_wakeup_disable_test.go | 12 ++--- 8 files changed, 65 insertions(+), 48 deletions(-) diff --git a/docs/usage/registry-cache/configuration.md b/docs/usage/registry-cache/configuration.md index c816382a..6ab06a3e 100644 --- a/docs/usage/registry-cache/configuration.md +++ b/docs/usage/registry-cache/configuration.md @@ -96,6 +96,9 @@ The `providerConfig.caches[].secretReferenceName` is the name of the reference f > [!NOTE] > It is only possible to provide one set of credentials for one private upstream registry. +> [!WARNING] +> The `public.ecr.aws` upstream for Amazon ECR Public Gallery is currently not supported. For details see [here](https://github.com/distribution/distribution/issues/4383). + ## Garbage Collection When the registry cache receives a request for an image that is not present in its local store, it fetches the image from the upstream, returns it to the client and stores the image in the local store. The registry cache runs a scheduler that deletes images when their time to live (ttl) expires. When adding an image to the local store, the registry cache also adds a time to live for the image. The ttl defaults to `168h` (7 days) and is configurable. The garbage collection can be disabled by setting the ttl to `0s`. Requesting an image from the registry cache does not extend the time to live of the image. Hence, an image is always garbage collected from the registry cache store when its ttl expires. diff --git a/test/common/common.go b/test/common/common.go index ec8ecefd..af08f357 100644 --- a/test/common/common.go +++ b/test/common/common.go @@ -29,19 +29,19 @@ import ( ) const ( - // PublicEcrAwsNginx1230Image is the public.ecr.aws/nginx/nginx:1.23.0 image. - PublicEcrAwsNginx1230Image = "public.ecr.aws/nginx/nginx:1.23.0" - // PublicEcrAwsNginx1240Image is the public.ecr.aws/nginx/nginx:1.24.0 image. - PublicEcrAwsNginx1240Image = "public.ecr.aws/nginx/nginx:1.24.0" - // PublicEcrAwsNginx1250Image is the public.ecr.aws/nginx/nginx:1.25.0 image. - PublicEcrAwsNginx1250Image = "public.ecr.aws/nginx/nginx:1.25.0" + // GithubRegistryJitesoftAlpine3189Image is the ghcr.io/jitesoft/alpine:3.18.9 image. + GithubRegistryJitesoftAlpine3189Image = "ghcr.io/jitesoft/alpine:3.18.9" + // GithubRegistryJitesoftAlpine3194Image is the ghcr.io/jitesoft/alpine:3.19.4 image. + GithubRegistryJitesoftAlpine3194Image = "ghcr.io/jitesoft/alpine:3.19.4" + // GithubRegistryJitesoftAlpine3203Image is the ghcr.io/jitesoft/alpine:3.20.3 image. + GithubRegistryJitesoftAlpine3203Image = "ghcr.io/jitesoft/alpine:3.20.3" // ArtifactRegistryNginx1176Image is the europe-docker.pkg.dev/gardener-project/releases/3rd/nginx:1.17.6 image (copy of docker.io/library/nginx:1.17.6). ArtifactRegistryNginx1176Image = "europe-docker.pkg.dev/gardener-project/releases/3rd/nginx:1.17.6" // RegistryK8sNginx1154Image is the registry.k8s.io/e2e-test-images/nginx:1.15-4 image. RegistryK8sNginx1154Image = "registry.k8s.io/e2e-test-images/nginx:1.15-4" - // GithubRegistryNginx1261Image is the ghcr.io/linuxserver/nginx:1.26.1 image. - GithubRegistryNginx1261Image = "ghcr.io/linuxserver/nginx:1.26.1" + // GitlabRegistryJitesoftAlpine31710Image is the registry.gitlab.com/jitesoft/dockerfiles/alpine:3.17.10 image. + GitlabRegistryJitesoftAlpine31710Image = "registry.gitlab.com/jitesoft/dockerfiles/alpine:3.17.10" // jqExtractRegistryLocation is a jq command that extracts the source location of the '/var/lib/registry' mount from the container's config.json file. jqExtractRegistryLocation = `jq -j '.mounts[] | select(.destination=="/var/lib/registry") | .source' /run/containerd/io.containerd.runtime.v2.task/k8s.io/%s/config.json` @@ -167,41 +167,55 @@ func VerifyHostsTOMLFilesDeletedForAllNodes(ctx context.Context, log logr.Logger } } +// SetupPodFn is an optional function to change the Pod specification depending on the image used. +type SetupPodFn func(pod *corev1.Pod) *corev1.Pod + +// SleepInfinity is SetupPodFn that keeps the container running indefinitely. +func SleepInfinity(pod *corev1.Pod) *corev1.Pod { + pod.Spec.Containers[0].Command = []string{"sleep"} + pod.Spec.Containers[0].Args = []string{"infinity"} + return pod +} + // VerifyRegistryCache verifies that a registry cache works as expected. // // The verification consists of the following steps: -// 1. It deploys an nginx Pod with the given image. +// 1. It deploys a Pod with the given image. // 2. It waits until the Pod is running. // 3. It verifies that the image is present in the registry's volume. // This is a verification that the image pull happened via the registry cache (and the containerd didn't fall back to the upstream). -func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient kubernetes.Interface, nginxImage string) { - By("Create nginx Pod") +func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient kubernetes.Interface, image string, setup SetupPodFn) { + upstream, path, tag := splitImage(image) + name := strings.ReplaceAll(path, "/", "-") + By(fmt.Sprintf("Create %s Pod", name)) ctx, cancel := context.WithTimeout(parentCtx, 5*time.Minute) defer cancel() pod := &corev1.Pod{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "nginx-", + GenerateName: name + "-", Namespace: corev1.NamespaceDefault, }, Spec: corev1.PodSpec{ Containers: []corev1.Container{ { - Name: "nginx", - Image: nginxImage, + Name: name, + Image: image, }, }, }, } + if setup != nil { + pod = setup(pod) + } ExpectWithOffset(1, shootClient.Client().Create(ctx, pod)).To(Succeed()) - By("Wait until nginx Pod is running") + By(fmt.Sprintf("Wait until %s Pod is running", name)) ExpectWithOffset(1, framework.WaitUntilPodIsRunning(ctx, log, pod.Name, pod.Namespace, shootClient)).To(Succeed()) - By("Verify the registry cache pulled the nginx image") + By(fmt.Sprintf("Verify the registry cache pulled the %s image", image)) ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute) defer cancel() - upstream, path, tag := splitImage(nginxImage) selector := labels.SelectorFromSet(labels.Set(map[string]string{"upstream-host": strings.Replace(upstream, ":", "-", 1)})) EventuallyWithOffset(1, ctx, func() error { registryPod, err := framework.GetFirstRunningPodWithLabels(ctx, selector, metav1.NamespaceSystem, shootClient) @@ -222,20 +236,20 @@ func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient imageDigest, err := rootPodExecutor.Execute(ctx, fmt.Sprintf("cat %s/docker/registry/v2/repositories/%s/_manifests/tags/%s/current/link", string(registryRootPath), path, tag)) if err != nil { - return fmt.Errorf("failed to get the %s image digest: %w", nginxImage, err) + return fmt.Errorf("failed to get the %s image digest: %w", image, err) } imageSha256Value := strings.TrimPrefix(string(imageDigest), "sha256:") imageIndexPath := fmt.Sprintf("sha256/%s/%s", imageSha256Value[:2], imageSha256Value) _, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(jqCountManifests, string(registryRootPath), imageIndexPath)) if err != nil { - return fmt.Errorf("failed to get the %s image index manifests count: %w", nginxImage, err) + return fmt.Errorf("failed to get the %s image index manifests count: %w", image, err) } return nil }).WithPolling(10*time.Second).Should(Succeed(), "Expected to successfully find the nginx image in the registry's volume") - By("Delete nginx Pod") + By(fmt.Sprintf("Delete %s Pod", name)) timeout := 5 * time.Minute ctx, cancel = context.WithTimeout(parentCtx, timeout) defer cancel() diff --git a/test/e2e/cache/create_enable_add_remove_disable_delete.go b/test/e2e/cache/create_enable_add_remove_disable_delete.go index ffb32f46..f774bdb8 100644 --- a/test/e2e/cache/create_enable_add_remove_disable_delete.go +++ b/test/e2e/cache/create_enable_add_remove_disable_delete.go @@ -37,47 +37,47 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { Expect(f.UpdateShoot(ctx, f.Shoot, func(shoot *gardencorev1beta1.Shoot) error { size := resource.MustParse("2Gi") common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) return nil })).To(Succeed()) - By("[public.ecr.aws] Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.PublicEcrAwsNginx1230Image) + By("[ghcr.io] Verify registry-cache works") + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.GithubRegistryJitesoftAlpine3189Image, common.SleepInfinity) - By("Add the ghcr.io upstream to the registry-cache extension") + By("Add the registry.gitlab.com upstream to the registry-cache extension") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) defer cancel() Expect(f.UpdateShoot(ctx, f.Shoot, func(shoot *gardencorev1beta1.Shoot) error { size := resource.MustParse("2Gi") common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "registry.gitlab.com", Volume: &v1alpha3.Volume{Size: &size}}, }) return nil })).To(Succeed()) - By("[ghcr.io] Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.GithubRegistryNginx1261Image) + By("[registry.gitlab.com] Verify registry-cache works") + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.GitlabRegistryJitesoftAlpine31710Image, common.SleepInfinity) - By("Remove the ghcr.io upstream from the registry-cache extension") + By("Remove the registry.gitlab.com upstream from the registry-cache extension") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) defer cancel() Expect(f.UpdateShoot(ctx, f.Shoot, func(shoot *gardencorev1beta1.Shoot) error { size := resource.MustParse("2Gi") common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) return nil })).To(Succeed()) - By("[ghcr.io] Verify registry configuration is removed") + By("[registry.gitlab.com] Verify registry configuration is removed") ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute) defer cancel() - common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootFramework.ShootClient, []string{"ghcr.io"}) + common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootFramework.ShootClient, []string{"registry.gitlab.com"}) By("Disable the registry-cache extension") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) @@ -88,10 +88,10 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { return nil })).To(Succeed()) - By("[public.ecr.aws] Verify registry configuration is removed") + By("[ghcr.io] Verify registry configuration is removed") ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute) defer cancel() - common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootFramework.ShootClient, []string{"public.ecr.aws"}) + common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootFramework.ShootClient, []string{"ghcr.io"}) By("Delete Shoot") ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) diff --git a/test/e2e/cache/create_enabled_delete_shoot_system_components.go b/test/e2e/cache/create_enabled_delete_shoot_system_components.go index 58179620..b0d2a238 100644 --- a/test/e2e/cache/create_enabled_delete_shoot_system_components.go +++ b/test/e2e/cache/create_enabled_delete_shoot_system_components.go @@ -41,10 +41,10 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { f.Verify() By("[europe-docker.pkg.dev] Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.ArtifactRegistryNginx1176Image) + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.ArtifactRegistryNginx1176Image, nil) By("[registry.k8s.io] Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.RegistryK8sNginx1154Image) + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.RegistryK8sNginx1154Image, nil) By("Delete Shoot") ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) diff --git a/test/e2e/cache/create_enabled_force_delete.go b/test/e2e/cache/create_enabled_force_delete.go index 0813cbe8..bdca2b2e 100644 --- a/test/e2e/cache/create_enabled_force_delete.go +++ b/test/e2e/cache/create_enabled_force_delete.go @@ -24,7 +24,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { shoot := e2e.DefaultShoot("e2e-cache-fd") size := resource.MustParse("2Gi") common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) f.Shoot = shoot @@ -36,7 +36,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { f.Verify() By("Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.PublicEcrAwsNginx1230Image) + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.GithubRegistryJitesoftAlpine3189Image, common.SleepInfinity) By("Force Delete Shoot") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) diff --git a/test/e2e/cache/create_enabled_hibernate_reconcile_delete.go b/test/e2e/cache/create_enabled_hibernate_reconcile_delete.go index e6d6c5d3..abd5bd2e 100644 --- a/test/e2e/cache/create_enabled_hibernate_reconcile_delete.go +++ b/test/e2e/cache/create_enabled_hibernate_reconcile_delete.go @@ -26,7 +26,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { shoot := e2e.DefaultShoot("e2e-cache-hib") size := resource.MustParse("2Gi") common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) f.Shoot = shoot @@ -38,7 +38,7 @@ var _ = Describe("Registry Cache Extension Tests", Label("cache"), func() { f.Verify() By("Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.PublicEcrAwsNginx1230Image) + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootFramework.ShootClient, common.GithubRegistryJitesoftAlpine3189Image, common.SleepInfinity) By("Hibernate Shoot") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) diff --git a/test/testmachinery/shoot/enable_disable_test.go b/test/testmachinery/shoot/enable_disable_test.go index d71b3672..f82eec58 100644 --- a/test/testmachinery/shoot/enable_disable_test.go +++ b/test/testmachinery/shoot/enable_disable_test.go @@ -38,14 +38,14 @@ var _ = Describe("Shoot registry cache testing", func() { } common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) return nil })).To(Succeed()) By("Verify registry-cache works") - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.PublicEcrAwsNginx1230Image) + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.GithubRegistryJitesoftAlpine3189Image, common.SleepInfinity) By("Disable the registry-cache extension") ctx, cancel = context.WithTimeout(parentCtx, 10*time.Minute) @@ -59,7 +59,7 @@ var _ = Describe("Shoot registry cache testing", func() { By("Verify registry configuration is removed") ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute) defer cancel() - common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootClient, []string{"public.ecr.aws"}) + common.VerifyHostsTOMLFilesDeletedForAllNodes(ctx, f.Logger, f.ShootClient, []string{"ghcr.io"}) }, defaultTestTimeout, framework.WithCAfterTest(func(ctx context.Context) { if common.HasRegistryCacheExtension(f.Shoot) { By("Disable the registry-cache extension") diff --git a/test/testmachinery/shoot/enable_hibernate_reconcile_wakeup_disable_test.go b/test/testmachinery/shoot/enable_hibernate_reconcile_wakeup_disable_test.go index 9da8a3df..06d9b47c 100644 --- a/test/testmachinery/shoot/enable_hibernate_reconcile_wakeup_disable_test.go +++ b/test/testmachinery/shoot/enable_hibernate_reconcile_wakeup_disable_test.go @@ -40,16 +40,16 @@ var _ = Describe("Shoot registry cache testing", func() { } common.AddOrUpdateRegistryCacheExtension(shoot, []v1alpha3.RegistryCache{ - {Upstream: "public.ecr.aws", Volume: &v1alpha3.Volume{Size: &size}}, + {Upstream: "ghcr.io", Volume: &v1alpha3.Volume{Size: &size}}, }) return nil })).To(Succeed()) By("Verify registry-cache works") - // We are using nginx:1.24.0 as nginx:1.23.0 is already used by the "should enable and disable the registry-cache extension" test. - // Hence, nginx:1.23.0 will be present in the Node. - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.PublicEcrAwsNginx1240Image) + // We are using ghcr.io/jitesoft/alpine:3.19.4 as ghcr.io/jitesoft/alpine:3.18.9 is already used by the "should enable and disable the registry-cache extension" test. + // Hence, ghcr.io/jitesoft/alpine:3.18.9 will be present in the Node. + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.GithubRegistryJitesoftAlpine3194Image, common.SleepInfinity) By("Hibernate Shoot") ctx, cancel = context.WithTimeout(parentCtx, 15*time.Minute) @@ -72,8 +72,8 @@ var _ = Describe("Shoot registry cache testing", func() { Expect(f.WakeUpShoot(ctx)).To(Succeed()) By("Verify registry-cache works after wake up") - // We are using nginx:1.25.0 as nginx:1.24.0 is already used above and already present in the Node and in the registry cache. - common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.PublicEcrAwsNginx1250Image) + // We are using ghcr.io/jitesoft/alpine:3.20.3 as ghcr.io/jitesoft/alpine:3.19.4 is already used above and already present in the Node and in the registry cache. + common.VerifyRegistryCache(parentCtx, f.Logger, f.ShootClient, common.GithubRegistryJitesoftAlpine3203Image, common.SleepInfinity) }, hibernationTestTimeout, framework.WithCAfterTest(func(ctx context.Context) { if v1beta1helper.HibernationIsEnabled(f.Shoot) { By("Wake up Shoot")