Support pulling OCI Image Index manifests

- Add Accept header application/vnd.oci.image.index.v1+json during base image pull - Re-use the logic from V22ManifestListTemplate to select the target platform diget via a new interface ManifestListTemplate
GoogleContainerTools · Aug 13, 2022 · 0b3ff8a · 0b3ff8a
1 parent 9027c8d
commit 0b3ff8a
Show file tree

Hide file tree

Showing 12 changed files with 226 additions and 26 deletions.
diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java
@@ -41,6 +41,7 @@
 import com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;
 import com.google.cloud.tools.jib.image.json.JsonToImageTranslator;
 import com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;
+import com.google.cloud.tools.jib.image.json.ManifestListTemplate;
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
 import com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException;
 import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;
@@ -315,8 +316,7 @@ private List<Image> pullBaseImages(
             JsonToImageTranslator.toImage(imageManifest, containerConfig));
       }
 
-      // TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it.
-      Verify.verify(manifestTemplate instanceof V22ManifestListTemplate);
+      Verify.verify(manifestTemplate instanceof ManifestListTemplate);
 
       List<ManifestAndConfigTemplate> manifestsAndConfigs = new ArrayList<>();
       ImmutableList.Builder<Image> images = ImmutableList.builder();
@@ -332,7 +332,7 @@ private List<Image> pullBaseImages(
 
           String manifestDigest =
               lookUpPlatformSpecificImageManifest(
-                  (V22ManifestListTemplate) manifestTemplate, platform);
+                  (ManifestListTemplate) manifestTemplate, platform);
           // TODO: pull multiple manifests (+ container configs) in parallel.
           ManifestAndDigest<?> imageManifestAndDigest = registryClient.pullManifest(manifestDigest);
           progressDispatcher2.dispatchProgress(1);
@@ -360,10 +360,9 @@ private List<Image> pullBaseImages(
    * Looks through a manifest list for the manifest matching the {@code platform} and returns the
    * digest of the first manifest it finds.
    */
-  // TODO: support OciIndexTemplate once AbstractManifestPuller starts to accept it.
   @VisibleForTesting
   String lookUpPlatformSpecificImageManifest(
-      V22ManifestListTemplate manifestListTemplate, Platform platform)
+      ManifestListTemplate manifestListTemplate, Platform platform)
       throws UnlistedPlatformInManifestListException {
     EventHandlers eventHandlers = buildContext.getEventHandlers();
 

diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/BuildableManifestTemplate.java
@@ -59,7 +59,7 @@ class ContentDescriptorTemplate implements JsonTemplate {
 
     /** Necessary for Jackson to create from JSON. */
     @SuppressWarnings("unused")
-    private ContentDescriptorTemplate() {}
+    protected ContentDescriptorTemplate() {}
 
     public long getSize() {
       return size;

diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/ManifestListTemplate.java
@@ -0,0 +1,38 @@
+/*
+ * Copyright 2022 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.image.json;
+
+import java.util.List;
+
+/**
+ * Parent class for manifest lists.
+ *
+ * @see V22ManifestListTemplate Docker V2.2 format
+ * @see OciIndexTemplate OCI format
+ */
+public interface ManifestListTemplate extends ManifestTemplate {
+
+  /**
+   * Returns a list of digests for a specific platform found in the manifest list. see
+   * <a>https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list</a>
+   *
+   * @param architecture the architecture of the target platform
+   * @param os the os of the target platform
+   * @return a list of matching digests
+   */
+  List<String> getDigestsForPlatform(String architecture, String os);
+}
diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/OciIndexTemplate.java
@@ -16,11 +16,17 @@
 
 package com.google.cloud.tools.jib.image.json;
 
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
+import com.google.cloud.tools.jib.api.DescriptorDigest;
 import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.json.JsonTemplate;
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.collect.ImmutableMap;
 import java.util.ArrayList;
 import java.util.List;
+import java.util.Objects;
+import java.util.stream.Collectors;
+import javax.annotation.Nullable;
 
 /**
  * JSON template for OCI archive "index.json" file.
@@ -36,6 +42,10 @@
  *       "mediaType": "application/vnd.oci.image.manifest.v1+json",
  *       "digest": "sha256:e684b1dceef404268f17d4adf7f755fd9912b8ae64864b3954a83ebb8aa628b3",
  *       "size": 1132,
+ *       "platform": {
+ *         "architecture": "ppc64le",
+ *         "os": "linux"
+ *       },
  *       "annotations": {
  *         "org.opencontainers.image.ref.name": "gcr.io/project/image:tag"
  *       }
@@ -47,16 +57,15 @@
  * @see <a href="https://github.com/opencontainers/image-spec/blob/master/image-index.md">OCI Image
  *     Index Specification</a>
  */
-public class OciIndexTemplate implements ManifestTemplate {
+public class OciIndexTemplate implements ManifestListTemplate {
 
   /** The OCI Index media type. */
   public static final String MEDIA_TYPE = "application/vnd.oci.image.index.v1+json";
 
   private final int schemaVersion = 2;
   private final String mediaType = MEDIA_TYPE;
 
-  private final List<BuildableManifestTemplate.ContentDescriptorTemplate> manifests =
-      new ArrayList<>();
+  private final List<ManifestDescriptorTemplate> manifests = new ArrayList<>();
 
   @Override
   public int getSchemaVersion() {
@@ -75,16 +84,84 @@ public String getManifestMediaType() {
    * @param imageReferenceName the image reference name
    */
   public void addManifest(BlobDescriptor descriptor, String imageReferenceName) {
-    BuildableManifestTemplate.ContentDescriptorTemplate contentDescriptorTemplate =
-        new BuildableManifestTemplate.ContentDescriptorTemplate(
+    ManifestDescriptorTemplate contentDescriptorTemplate =
+        new ManifestDescriptorTemplate(
             OciManifestTemplate.MANIFEST_MEDIA_TYPE, descriptor.getSize(), descriptor.getDigest());
     contentDescriptorTemplate.setAnnotations(
         ImmutableMap.of("org.opencontainers.image.ref.name", imageReferenceName));
     manifests.add(contentDescriptorTemplate);
   }
 
   @VisibleForTesting
-  public List<BuildableManifestTemplate.ContentDescriptorTemplate> getManifests() {
+  public List<ManifestDescriptorTemplate> getManifests() {
     return manifests;
   }
+
+  @Override
+  public List<String> getDigestsForPlatform(String architecture, String os) {
+    return getManifests().stream()
+        .filter(
+            manifest ->
+                manifest.platform != null
+                    && os.equals(manifest.platform.os)
+                    && architecture.equals(manifest.platform.architecture))
+        .map(ManifestDescriptorTemplate::getDigest)
+        .filter(Objects::nonNull)
+        .map(DescriptorDigest::toString)
+        .collect(Collectors.toList());
+  }
+
+  /**
+   * Template for inner JSON object representing a single platform specific manifest. See <a
+   * href="https://github.com/opencontainers/image-spec/blob/main/image-index.md">OCI Image Index
+   * Specification</a>
+   */
+  public static class ManifestDescriptorTemplate
+      extends BuildableManifestTemplate.ContentDescriptorTemplate {
+
+    ManifestDescriptorTemplate(String mediaType, long size, DescriptorDigest digest) {
+      super(mediaType, size, digest);
+    }
+
+    /** Necessary for Jackson to create from JSON. */
+    @SuppressWarnings("unused")
+    private ManifestDescriptorTemplate() {
+      super();
+    }
+
+    @JsonIgnoreProperties(ignoreUnknown = true)
+    public static class Platform implements JsonTemplate {
+      @Nullable private String architecture;
+      @Nullable private String os;
+
+      @Nullable
+      public String getArchitecture() {
+        return architecture;
+      }
+
+      @Nullable
+      public String getOs() {
+        return os;
+      }
+    }
+
+    @Nullable private OciIndexTemplate.ManifestDescriptorTemplate.Platform platform;
+
+    /**
+     * Sets a platform.
+     *
+     * @param architecture the manifest architecture
+     * @param os the manifest os
+     */
+    public void setPlatform(String architecture, String os) {
+      platform = new OciIndexTemplate.ManifestDescriptorTemplate.Platform();
+      platform.architecture = architecture;
+      platform.os = os;
+    }
+
+    @Nullable
+    public OciIndexTemplate.ManifestDescriptorTemplate.Platform getPlatform() {
+      return platform;
+    }
+  }
 }
diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java
@@ -63,7 +63,7 @@
  * @see <a href="https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list">Image Manifest
  *     Version 2, Schema 2: Manifest List</a>
  */
-public class V22ManifestListTemplate implements ManifestTemplate {
+public class V22ManifestListTemplate implements ManifestListTemplate {
 
   public static final String MANIFEST_MEDIA_TYPE =
       "application/vnd.docker.distribution.manifest.list.v2+json";
@@ -101,14 +101,7 @@ public List<ManifestDescriptorTemplate> getManifests() {
     return Preconditions.checkNotNull(manifests);
   }
 
-  /**
-   * Returns a list of digests for a specific platform found in the manifest list. see
-   * <a>https://docs.docker.com/registry/spec/manifest-v2-2/#manifest-list</a>
-   *
-   * @param architecture the architecture of the target platform
-   * @param os the os of the target platform
-   * @return a list of matching digests
-   */
+  @Override
   public List<String> getDigestsForPlatform(String architecture, String os) {
     return getManifests().stream()
         .filter(

diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/AbstractManifestPuller.java
@@ -80,12 +80,16 @@ public List<String> getAccept() {
     if (manifestTemplateClass.equals(V22ManifestListTemplate.class)) {
       return Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE);
     }
+    if (manifestTemplateClass.equals(OciIndexTemplate.class)) {
+      return Collections.singletonList(OciIndexTemplate.MEDIA_TYPE);
+    }
 
     return Arrays.asList(
         OciManifestTemplate.MANIFEST_MEDIA_TYPE,
         V22ManifestTemplate.MANIFEST_MEDIA_TYPE,
         V21ManifestTemplate.MEDIA_TYPE,
-        V22ManifestListTemplate.MANIFEST_MEDIA_TYPE);
+        V22ManifestListTemplate.MANIFEST_MEDIA_TYPE,
+        OciIndexTemplate.MEDIA_TYPE);
   }
 
   /** Parses the response body into a {@link ManifestAndDigest}. */
@@ -174,6 +178,10 @@ private T getManifestTemplateFromJson(String jsonString)
         return manifestTemplateClass.cast(
             JsonTemplateMapper.readJson(jsonString, V22ManifestListTemplate.class));
       }
+      if (OciIndexTemplate.MEDIA_TYPE.equals(mediaType)) {
+        return manifestTemplateClass.cast(
+            JsonTemplateMapper.readJson(jsonString, OciIndexTemplate.class));
+      }
       throw new UnknownManifestFormatException("Unknown mediaType: " + mediaType);
     }
     throw new UnknownManifestFormatException(

diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStepTest.java
@@ -38,6 +38,7 @@
 import com.google.cloud.tools.jib.image.json.ContainerConfigurationTemplate;
 import com.google.cloud.tools.jib.image.json.ImageMetadataTemplate;
 import com.google.cloud.tools.jib.image.json.ManifestAndConfigTemplate;
+import com.google.cloud.tools.jib.image.json.OciIndexTemplate;
 import com.google.cloud.tools.jib.image.json.PlatformNotFoundInBaseImageException;
 import com.google.cloud.tools.jib.image.json.UnlistedPlatformInManifestListException;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
@@ -202,7 +203,7 @@ public void testCall_offlineMode_cached()
   }
 
   @Test
-  public void testLookUpPlatformSpecificImageManifest()
+  public void testLookUpPlatformSpecificDockerImageManifest()
       throws IOException, UnlistedPlatformInManifestListException {
     String manifestListJson =
         " {\n"
@@ -241,6 +242,46 @@ public void testLookUpPlatformSpecificImageManifest()
         "sha256:2222222222222222222222222222222222222222222222222222222222222222", manifestDigest);
   }
 
+  @Test
+  public void testLookUpPlatformSpecificOciManifest()
+      throws IOException, UnlistedPlatformInManifestListException {
+    String manifestListJson =
+        " {\n"
+            + "   \"schemaVersion\": 2,\n"
+            + "   \"mediaType\": \"application/vnd.oci.image.index.v1+json\",\n"
+            + "   \"manifests\": [\n"
+            + "      {\n"
+            + "         \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n"
+            + "         \"size\": 424,\n"
+            + "         \"digest\": \"sha256:1111111111111111111111111111111111111111111111111111111111111111\",\n"
+            + "         \"platform\": {\n"
+            + "            \"architecture\": \"arm64\",\n"
+            + "            \"os\": \"linux\"\n"
+            + "         }\n"
+            + "      },\n"
+            + "      {\n"
+            + "         \"mediaType\": \"application/vnd.oci.image.manifest.v1+json\",\n"
+            + "         \"size\": 425,\n"
+            + "         \"digest\": \"sha256:2222222222222222222222222222222222222222222222222222222222222222\",\n"
+            + "         \"platform\": {\n"
+            + "            \"architecture\": \"targetArchitecture\",\n"
+            + "            \"os\": \"targetOS\"\n"
+            + "         }\n"
+            + "      }\n"
+            + "   ]\n"
+            + "}";
+
+    OciIndexTemplate manifestList =
+        JsonTemplateMapper.readJson(manifestListJson, OciIndexTemplate.class);
+
+    String manifestDigest =
+        pullBaseImageStep.lookUpPlatformSpecificImageManifest(
+            manifestList, new Platform("targetArchitecture", "targetOS"));
+
+    Assert.assertEquals(
+        "sha256:2222222222222222222222222222222222222222222222222222222222222222", manifestDigest);
+  }
+
   @Test
   public void testGetCachedBaseImages_emptyCache()
       throws InvalidImageReferenceException, IOException, CacheCorruptedException,

diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageReaderTest.java
@@ -297,7 +297,7 @@ public void testRetrieveMetadata_ociImageIndex()
 
     MatcherAssert.assertThat(
         metadata.getManifestList(), CoreMatchers.instanceOf(OciIndexTemplate.class));
-    List<ContentDescriptorTemplate> manifestDescriptors =
+    List<? extends ContentDescriptorTemplate> manifestDescriptors =
         ((OciIndexTemplate) metadata.getManifestList()).getManifests();
 
     Assert.assertEquals(1, manifestDescriptors.size());

diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/cache/CacheStorageWriterTest.java
@@ -291,7 +291,7 @@ public void testWriteMetadata_oci()
 
     MatcherAssert.assertThat(
         savedMetadata.getManifestList(), CoreMatchers.instanceOf(OciIndexTemplate.class));
-    List<ContentDescriptorTemplate> savedManifestDescriptors =
+    List<? extends ContentDescriptorTemplate> savedManifestDescriptors =
         ((OciIndexTemplate) savedMetadata.getManifestList()).getManifests();
 
     Assert.assertEquals(1, savedManifestDescriptors.size());

diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/image/json/OciIndexTemplateTest.java
@@ -71,4 +71,19 @@ public void testFromJson() throws IOException, URISyntaxException, DigestExcepti
         "regis.try/repo:tag", manifest.getAnnotations().get("org.opencontainers.image.ref.name"));
     Assert.assertEquals(1000, manifest.getSize());
   }
+
+  @Test
+  public void testFromJsonWithPlatform() throws IOException, URISyntaxException, DigestException {
+    // Loads the JSON string.
+    Path jsonFile = Paths.get(Resources.getResource("core/json/ociindex_platforms.json").toURI());
+
+    // Deserializes into a manifest JSON object.
+    OciIndexTemplate ociIndexJson =
+        JsonTemplateMapper.readJsonFromFile(jsonFile, OciIndexTemplate.class);
+
+    Assert.assertEquals(2, ociIndexJson.getManifests().size());
+    Assert.assertEquals(
+        "sha256:e692418e4cbaf90ca69d05a66403747baa33ee08806650b51fab815ad7fc331f",
+        ociIndexJson.getDigestsForPlatform("ppc64le", "linux").get(0));
+  }
 }
diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java
@@ -318,7 +318,8 @@ public void testGetAccept() {
             OciManifestTemplate.MANIFEST_MEDIA_TYPE,
             V22ManifestTemplate.MANIFEST_MEDIA_TYPE,
             V21ManifestTemplate.MEDIA_TYPE,
-            V22ManifestListTemplate.MANIFEST_MEDIA_TYPE),
+            V22ManifestListTemplate.MANIFEST_MEDIA_TYPE,
+            OciIndexTemplate.MEDIA_TYPE),
         testManifestPuller.getAccept());
 
     Assert.assertEquals(