From 24658b3317cd0604f9fba1f9a75a307f1fe78912 Mon Sep 17 00:00:00 2001 From: Dimitar Kostadinov <51451517+dimitar-kostadinov@users.noreply.github.com> Date: Tue, 8 Oct 2024 16:46:23 +0300 Subject: [PATCH] Improve image pull check (#255) * Improve image pull check * Address PR review feedback * Add select condition for platform.os linux and address PR review feedback --- test/common/common.go | 71 +++++++++++++++++++++++++++++++++++-------- 1 file changed, 59 insertions(+), 12 deletions(-) diff --git a/test/common/common.go b/test/common/common.go index 548902e9..ac284ade 100644 --- a/test/common/common.go +++ b/test/common/common.go @@ -7,6 +7,7 @@ package common import ( "context" "encoding/json" + "errors" "fmt" "slices" "strings" @@ -23,6 +24,7 @@ import ( "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "sigs.k8s.io/controller-runtime/pkg/client" mirrorv1alpha1 "github.com/gardener/gardener-extension-registry-cache/pkg/apis/mirror/v1alpha1" registryv1alpha3 "github.com/gardener/gardener-extension-registry-cache/pkg/apis/registry/v1alpha3" @@ -50,9 +52,11 @@ const ( // jqExtractRegistryLocation is a jq command that extracts the source location of the '/var/lib/registry' mount from the container's config.json file. jqExtractRegistryLocation = `jq -j '.mounts[] | select(.destination=="/var/lib/registry") | .source' /run/containerd/io.containerd.runtime.v2.task/k8s.io/%s/config.json` - // jqCountManifests is a jq command that counts the number of image manifests in the manifest index. - // Ref: https://github.com/opencontainers/image-spec/blob/main/image-index.md#example-image-index. - jqCountManifests = `jq -j '.manifests | length' %s/docker/registry/v2/blobs/%s/data` + // jqExtractManifestDigest is a jq command that extracts the manifest digest for the current OS architecture. + jqExtractManifestDigest = `jq -j '.manifests[] | select(.platform.os=="linux" and .platform.architecture=="%s") | .digest' %s/docker/registry/v2/blobs/%s/data` + // jqExtractLayersDigests is a jq command that extracts layers digests from the manifest. + // Ref: https://github.com/opencontainers/image-spec/blob/main/manifest.md. + jqExtractLayersDigests = `jq -r '.layers[].digest' %s/docker/registry/v2/blobs/%s/data` ) // AddOrUpdateRegistryCacheExtension adds or updates registry-cache extension with the given caches to the given Shoot. @@ -217,6 +221,13 @@ func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient By(fmt.Sprintf("Wait until %s Pod is running", name)) ExpectWithOffset(1, framework.WaitUntilPodIsRunning(ctx, log, pod.Name, pod.Namespace, shootClient)).To(Succeed()) + // Get the architecture of Node the Pod is running on + ExpectWithOffset(1, shootClient.Client().Get(ctx, client.ObjectKeyFromObject(pod), pod)).To(Succeed()) + node := &corev1.Node{} + ExpectWithOffset(1, shootClient.Client().Get(ctx, client.ObjectKey{Name: pod.Spec.NodeName}, node)).To(Succeed()) + arch := node.Status.NodeInfo.Architecture + log.Info("Node architecture", "name", node.Name, "arch", arch) + By(fmt.Sprintf("Verify the registry cache pulled the %s image", image)) ctx, cancel = context.WithTimeout(parentCtx, 2*time.Minute) defer cancel() @@ -234,24 +245,50 @@ func VerifyRegistryCache(parentCtx context.Context, log logr.Logger, shootClient }(ctx, rootPodExecutor) containerID := strings.TrimPrefix(registryPod.Status.ContainerStatuses[0].ContainerID, "containerd://") - registryRootPath, err := rootPodExecutor.Execute(ctx, fmt.Sprintf(jqExtractRegistryLocation, containerID)) + log.Info("Registry container ID", "containerID", containerID) + output, err := rootPodExecutor.Execute(ctx, fmt.Sprintf(jqExtractRegistryLocation, containerID)) + if err != nil { + log.Error(err, "Failed to extract the source location of the '/var/lib/registry' mount from the container's config.json file", "output", string(output)) + return fmt.Errorf("failed to extract the source location of the '/var/lib/registry' mount from the container's config.json file: command failed with err %w", err) + } + registryRootPath := string(output) + log.Info("Registry root path on node", "registryRootPath", registryRootPath) + + output, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(`cat %s/docker/registry/v2/repositories/%s/_manifests/tags/%s/current/link`, registryRootPath, path, tag)) if err != nil { - return fmt.Errorf("failed to extract the source localtion of the '/var/lib/registry' mount from the container's config.json file: %w", err) + log.Error(err, "Failed to get the image index digest", "image", image, "output", string(output)) + return fmt.Errorf("failed to get the %s image index digest: %w", image, err) } + imageIndexPath := sha256Path(string(output)) + log.Info("Image index path under /docker/registry/v2/blobs/", "imageIndexPath", imageIndexPath) - imageDigest, err := rootPodExecutor.Execute(ctx, fmt.Sprintf("cat %s/docker/registry/v2/repositories/%s/_manifests/tags/%s/current/link", string(registryRootPath), path, tag)) + output, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(jqExtractManifestDigest, arch, registryRootPath, imageIndexPath)) if err != nil { - return fmt.Errorf("failed to get the %s image digest: %w", image, err) + log.Error(err, "Failed to get the image manifests digest", "image", image, "output", string(output)) + return fmt.Errorf("failed to get the %s image manifests digest: %w", image, err) } - imageSha256Value := strings.TrimPrefix(string(imageDigest), "sha256:") - imageIndexPath := fmt.Sprintf("sha256/%s/%s", imageSha256Value[:2], imageSha256Value) + manifestPath := sha256Path(string(output)) + log.Info("Image manifest path under /docker/registry/v2/blobs/", "image", image, "manifestPath", manifestPath) - _, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(jqCountManifests, string(registryRootPath), imageIndexPath)) + output, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(jqExtractLayersDigests, registryRootPath, manifestPath)) if err != nil { - return fmt.Errorf("failed to get the %s image index manifests count: %w", image, err) + log.Error(err, "Failed to get the image layers digests", "image", image, "output", string(output)) + return fmt.Errorf("failed to get the %s image layers digests: %w", image, err) + } + layerDigests := strings.Split(strings.TrimSpace(string(output)), "\n") + log.Info("Image layers", "count", len(layerDigests)) + + var errs []error + for _, layerDigest := range layerDigests { + _, err = rootPodExecutor.Execute(ctx, fmt.Sprintf(`[ -f %s/docker/registry/v2/blobs/%s/data ]`, registryRootPath, sha256Path(layerDigest))) + if err != nil { + log.Error(err, "Failed to find image layer", "image", image, "digest", layerDigest) + errs = append(errs, fmt.Errorf("failed to find image %s layer with digest %s", image, layerDigest)) + } + log.Info("Image layer exists", "image", image, "digest", layerDigest) } - return nil + return errors.Join(errs...) }).WithPolling(10*time.Second).Should(Succeed(), fmt.Sprintf("Expected to successfully find the %s image in the registry's volume", image)) By(fmt.Sprintf("Delete %s Pod", name)) @@ -271,3 +308,13 @@ func splitImage(image string) (upstream, path, tag string) { path = path[:index] return } + +// sha256Path construct the path under /docker/registry/v2/blobs/ +// e.g. sha256:d72807a326fbca3a3bf68a9add2f10248a19205557ddd44b5ad629d8d6c0f805 -> sha256/d7/d72807a326fbca3a3bf68a9add2f10248a19205557ddd44b5ad629d8d6c0f805 +func sha256Path(digest string) string { + if len(digest) < 64 { + return digest + } + value := strings.TrimPrefix(digest, "sha256:") + return fmt.Sprintf("sha256/%s/%s", value[:2], value) +}