Support manifest lists when referenced by sha256

jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java

@@ -18,6 +18,7 @@
 
 import com.google.cloud.tools.jib.Command;
 import com.google.cloud.tools.jib.registry.LocalRegistry;
+import com.google.cloud.tools.jib.registry.ManifestPullerIntegrationTest;
 import java.io.IOException;
 import java.nio.file.Path;
 import java.util.Optional;
@@ -188,4 +189,22 @@ public void testProvidedExecutorNotDisposed()
 
     executorService.shutdown();
   }
+
+  @Test
+  public void testManifestListReferenceByShaDoesNotFail()
+      throws InvalidImageReferenceException, IOException, InterruptedException, ExecutionException,
+          RegistryException, CacheDirectoryCreationException {
+    ImageReference sourceImageReferenceAsManifestList =
+        ImageReference.of(
+            "registry-1.docker.io",
+            "library/openjdk",
+            ManifestPullerIntegrationTest.KNOWN_MANIFEST_LIST_SHA);
+    Containerizer containerizer =
+        Containerizer.to(
+            TarImage.named("whatever")
+                .saveTo(cacheFolder.newFolder("goose").toPath().resolve("moose")));
+
+    Jib.from(sourceImageReferenceAsManifestList).containerize(containerizer);
+    // pass, no exceptions thrown
+  }
 }

jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPullerIntegrationTest.java

@@ -18,8 +18,10 @@
 
 import com.google.cloud.tools.jib.api.RegistryException;
 import com.google.cloud.tools.jib.event.EventHandlers;
+import com.google.cloud.tools.jib.http.Authorization;
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
+import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate;
 import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import java.io.IOException;
 import org.hamcrest.CoreMatchers;
@@ -30,6 +32,10 @@
 /** Integration tests for {@link ManifestPuller}. */
 public class ManifestPullerIntegrationTest {
 
+  /** A known manifest list sha for openjdk:11-jre-slim */
+  public static final String KNOWN_MANIFEST_LIST_SHA =
+      "sha256:8ab7b3078b01ba66b937b7fbe0b9eccf60449cc101c42e99aeefaba0e1781155";
+
   @ClassRule public static LocalRegistry localRegistry = new LocalRegistry(5000);
 
   @Test
@@ -58,6 +64,40 @@ public void testPull_v22() throws IOException, RegistryException, InterruptedExc
     Assert.assertTrue(v22ManifestTemplate.getLayers().size() > 0);
   }
 
+  @Test
+  public void testPull_v22ManifestList() throws IOException, RegistryException {
+    RegistryClient.Factory factory =
+        RegistryClient.factory(EventHandlers.NONE, "registry-1.docker.io", "library/openjdk");
+    Authorization authorization =
+        factory.newRegistryClient().getRegistryAuthenticator().authenticatePull(null);
+    RegistryClient registryClient = factory.setAuthorization(authorization).newRegistryClient();
+
+    // Ensure 11-jre-slim is a manifest list
+    V22ManifestListTemplate manifestListTemplate =
+        registryClient.pullManifest("11-jre-slim", V22ManifestListTemplate.class);
+    Assert.assertEquals(2, manifestListTemplate.getSchemaVersion());
+    Assert.assertTrue(manifestListTemplate.getManifests().size() > 0);
+
+    // Generic call to 11-jre-slim should NOT pull a manifest list (delegate to registry default)
+    ManifestTemplate manifestTemplate = registryClient.pullManifest("11-jre-slim");
+    Assert.assertEquals(2, manifestTemplate.getSchemaVersion());
+    Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestTemplate.class));
+
+    // Make sure we can't cast a v22ManifestTemplate to v22ManifestListTemplate in ManifestPuller
+    try {
+      registryClient.pullManifest(KNOWN_MANIFEST_LIST_SHA, V22ManifestTemplate.class);
+      Assert.fail();
+    } catch (ClassCastException ex) {
+      // pass
+    }
+
+    // Referencing a manifest list by sha256, should return a manifest list
+    ManifestTemplate sha256ManifestList = registryClient.pullManifest(KNOWN_MANIFEST_LIST_SHA);
+    Assert.assertEquals(2, sha256ManifestList.getSchemaVersion());
+    Assert.assertThat(sha256ManifestList, CoreMatchers.instanceOf(V22ManifestListTemplate.class));
+    Assert.assertTrue(((V22ManifestListTemplate) sha256ManifestList).getManifests().size() > 0);
+  }
+
   @Test
   public void testPull_unknownManifest()
       throws RegistryException, IOException, InterruptedException {

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java

@@ -44,12 +44,14 @@
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
 import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
+import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate;
 import com.google.cloud.tools.jib.json.JsonTemplateMapper;
 import com.google.cloud.tools.jib.registry.RegistryAuthenticator;
 import com.google.cloud.tools.jib.registry.RegistryClient;
 import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException;
 import com.google.common.annotations.VisibleForTesting;
 import java.io.IOException;
+import java.util.List;
 import java.util.Optional;
 import java.util.concurrent.Callable;
 import javax.annotation.Nullable;
@@ -207,6 +209,19 @@ private Image pullBaseImage(
     ManifestTemplate manifestTemplate =
         registryClient.pullManifest(buildConfiguration.getBaseImageConfiguration().getImageTag());
 
+    // special handling if we happen upon a manifest list, redirect to a manifest and continue
+    // handling it normally
+    if (manifestTemplate instanceof V22ManifestListTemplate) {
+      buildConfiguration
+          .getEventHandlers()
+          .dispatch(
+              LogEvent.lifecycle(
+                  "The base image reference is manifest list, searching for linux/amd64"));
+      manifestTemplate =
+          obtainPlatformSpecificImageManifest(
+              registryClient, (V22ManifestListTemplate) manifestTemplate);
+    }
+
     // TODO: Make schema version be enum.
     switch (manifestTemplate.getSchemaVersion()) {
       case 1:
@@ -258,6 +273,25 @@ private Image pullBaseImage(
     throw new IllegalStateException("Unknown manifest schema version");
   }
 
+  /**
+   * Looks through a manifest list for any amd64/linux manifest and downloads and returns the first
+   * manifest it finds.
+   */
+  private ManifestTemplate obtainPlatformSpecificImageManifest(
+      RegistryClient registryClient, V22ManifestListTemplate manifestListTemplate)
+      throws RegistryException, IOException {
+
+    List<String> digests = manifestListTemplate.getDigestsForPlatform("amd64", "linux");
+    if (digests.size() == 0) {
+      String errorMessage =
+          "Unable to find amd64/linux manifest in manifest list at: "
+              + buildConfiguration.getBaseImageConfiguration().getImage();
+      buildConfiguration.getEventHandlers().dispatch(LogEvent.error(errorMessage));
+      throw new RegistryException(errorMessage);
+    }
+    return registryClient.pullManifest(digests.get(0));
+  }
+
   /**
    * Retrieves the cached base image.
    *

jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java

@@ -76,7 +76,7 @@ public int getSchemaVersion() {
   @Nullable private List<ManifestDescriptorTemplate> manifests;
 
   @VisibleForTesting
-  List<ManifestDescriptorTemplate> getManifests() {
+  public List<ManifestDescriptorTemplate> getManifests() {
     return Preconditions.checkNotNull(manifests);
   }
 
@@ -93,10 +93,10 @@ public List<String> getDigestsForPlatform(String architecture, String os) {
   }
 
   /** Template for inner JSON object representing a single platform specific manifest. */
-  static class ManifestDescriptorTemplate implements JsonTemplate {
+  public static class ManifestDescriptorTemplate implements JsonTemplate {
 
     @JsonIgnoreProperties(ignoreUnknown = true)
-    static class Platform implements JsonTemplate {
+    public static class Platform implements JsonTemplate {
       @Nullable private String architecture;
       @Nullable private String os;
     }

jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java

@@ -25,6 +25,7 @@
 import com.google.cloud.tools.jib.image.json.OCIManifestTemplate;
 import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
+import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate;
 import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import com.google.cloud.tools.jib.json.JsonTemplateMapper;
 import com.google.common.io.CharStreams;
@@ -71,7 +72,13 @@ public List<String> getAccept() {
     if (manifestTemplateClass.equals(OCIManifestTemplate.class)) {
       return Collections.singletonList(OCIManifestTemplate.MANIFEST_MEDIA_TYPE);
     }
+    if (manifestTemplateClass.equals(V22ManifestListTemplate.class)) {
+      return Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE);
+    }
 
+    // V22ManifestListTemplate is not included by default, we don't explicitly accept
+    // it, we only handle it if referenced by sha256 (see getManifestTemplateFromJson) in which
+    // case registries ignore the "accept" directive and just return a manifest list anyway.
     return Arrays.asList(
         OCIManifestTemplate.MANIFEST_MEDIA_TYPE,
         V22ManifestTemplate.MANIFEST_MEDIA_TYPE,
@@ -118,10 +125,6 @@ private T getManifestTemplateFromJson(String jsonString)
       throw new UnknownManifestFormatException("Cannot find field 'schemaVersion' in manifest");
     }
 
-    if (!manifestTemplateClass.equals(ManifestTemplate.class)) {
-      return JsonTemplateMapper.readJson(jsonString, manifestTemplateClass);
-    }
-
     int schemaVersion = node.get("schemaVersion").asInt(-1);
     if (schemaVersion == -1) {
       throw new UnknownManifestFormatException("`schemaVersion` field is not an integer");
@@ -142,6 +145,10 @@ private T getManifestTemplateFromJson(String jsonString)
         return manifestTemplateClass.cast(
             JsonTemplateMapper.readJson(jsonString, OCIManifestTemplate.class));
       }
+      if (V22ManifestListTemplate.MANIFEST_MEDIA_TYPE.equals(mediaType)) {
+        return manifestTemplateClass.cast(
+            JsonTemplateMapper.readJson(jsonString, V22ManifestListTemplate.class));
+      }
       throw new UnknownManifestFormatException("Unknown mediaType: " + mediaType);
     }
     throw new UnknownManifestFormatException(

jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java

@@ -28,10 +28,9 @@
 import com.google.cloud.tools.jib.event.EventHandlers;
 import com.google.cloud.tools.jib.global.JibSystemProperties;
 import com.google.cloud.tools.jib.http.Authorization;
+import com.google.cloud.tools.jib.http.Response;
 import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate;
 import com.google.cloud.tools.jib.image.json.ManifestTemplate;
-import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
-import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import com.google.cloud.tools.jib.json.JsonTemplate;
 import com.google.cloud.tools.jib.json.JsonTemplateMapper;
 import com.google.common.annotations.VisibleForTesting;
@@ -282,7 +281,8 @@ public RegistryAuthenticator getRegistryAuthenticator() throws IOException, Regi
    * @param <T> child type of ManifestTemplate
    * @param imageTag the tag to pull on
    * @param manifestTemplateClass the specific version of manifest template to pull, or {@link
-   *     ManifestTemplate} to pull either {@link V22ManifestTemplate} or {@link V21ManifestTemplate}
+   *     ManifestTemplate} to pull predefined subclasses; see: {@link
+   *     ManifestPuller#handleResponse(Response)}
    * @return the manifest template
    * @throws IOException if communicating with the endpoint fails
    * @throws RegistryException if communicating with the endpoint fails

jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java

@@ -21,6 +21,7 @@
 import com.google.cloud.tools.jib.image.json.OCIManifestTemplate;
 import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException;
 import com.google.cloud.tools.jib.image.json.V21ManifestTemplate;
+import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate;
 import com.google.cloud.tools.jib.image.json.V22ManifestTemplate;
 import com.google.common.io.Resources;
 import java.io.ByteArrayInputStream;
@@ -89,6 +90,42 @@ public void testHandleResponse_v22()
     Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestTemplate.class));
   }
 
+  @Test
+  public void testHandleResponse_v22ManifestListFailsWhenParsedAsV22Manifest()
+      throws URISyntaxException, IOException, UnknownManifestFormatException {
+    Path v22ManifestListFile =
+        Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI());
+    InputStream v22ManifestList = new ByteArrayInputStream(Files.readAllBytes(v22ManifestListFile));
+
+    Mockito.when(mockResponse.getBody()).thenReturn(v22ManifestList);
+    try {
+      new ManifestPuller<>(
+              fakeRegistryEndpointRequestProperties, "test-image-tag", V22ManifestTemplate.class)
+          .handleResponse(mockResponse);
+      Assert.fail();
+    } catch (ClassCastException ex) {
+      // pass
+    }
+  }
+
+  @Test
+  public void testHandleResponse_v22ManifestList()
+      throws URISyntaxException, IOException, UnknownManifestFormatException {
+    Path v22ManifestListFile =
+        Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI());
+    InputStream v22ManifestList = new ByteArrayInputStream(Files.readAllBytes(v22ManifestListFile));
+
+    Mockito.when(mockResponse.getBody()).thenReturn(v22ManifestList);
+    ManifestTemplate manifestTemplate =
+        new ManifestPuller<>(
+                fakeRegistryEndpointRequestProperties,
+                "test-image-tag",
+                V22ManifestListTemplate.class)
+            .handleResponse(mockResponse);
+
+    Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestListTemplate.class));
+  }
+
   @Test
   public void testHandleResponse_noSchemaVersion() throws IOException {
     Mockito.when(mockResponse.getBody()).thenReturn(stringToInputStreamUtf8("{}"));
@@ -175,5 +212,12 @@ public void testGetAccept() {
         new ManifestPuller<>(
                 fakeRegistryEndpointRequestProperties, "test-image-tag", V21ManifestTemplate.class)
             .getAccept());
+    Assert.assertEquals(
+        Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE),
+        new ManifestPuller<>(
+                fakeRegistryEndpointRequestProperties,
+                "test-image-tag",
+                V22ManifestListTemplate.class)
+            .getAccept());
   }
 }