diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java index 7124df8d56..8ad467771a 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/api/JibIntegrationTest.java @@ -18,6 +18,7 @@ import com.google.cloud.tools.jib.Command; import com.google.cloud.tools.jib.registry.LocalRegistry; +import com.google.cloud.tools.jib.registry.ManifestPullerIntegrationTest; import java.io.IOException; import java.nio.file.Path; import java.util.Optional; @@ -188,4 +189,22 @@ public void testProvidedExecutorNotDisposed() executorService.shutdown(); } + + @Test + public void testManifestListReferenceByShaDoesNotFail() + throws InvalidImageReferenceException, IOException, InterruptedException, ExecutionException, + RegistryException, CacheDirectoryCreationException { + ImageReference sourceImageReferenceAsManifestList = + ImageReference.of( + "registry-1.docker.io", + "library/openjdk", + ManifestPullerIntegrationTest.KNOWN_MANIFEST_LIST_SHA); + Containerizer containerizer = + Containerizer.to( + TarImage.named("whatever") + .saveTo(cacheFolder.newFolder("goose").toPath().resolve("moose"))); + + Jib.from(sourceImageReferenceAsManifestList).containerize(containerizer); + // pass, no exceptions thrown + } } diff --git a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPullerIntegrationTest.java b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPullerIntegrationTest.java index e4f82691f5..43b8c80e98 100644 --- a/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPullerIntegrationTest.java +++ b/jib-core/src/integration-test/java/com/google/cloud/tools/jib/registry/ManifestPullerIntegrationTest.java @@ -18,8 +18,10 @@ import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.event.EventHandlers; +import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; +import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import java.io.IOException; import org.hamcrest.CoreMatchers; @@ -31,6 +33,10 @@ /** Integration tests for {@link ManifestPuller}. */ public class ManifestPullerIntegrationTest { + /** A known manifest list sha for openjdk:11-jre-slim */ + public static final String KNOWN_MANIFEST_LIST_SHA = + "sha256:8ab7b3078b01ba66b937b7fbe0b9eccf60449cc101c42e99aeefaba0e1781155"; + @ClassRule public static LocalRegistry localRegistry = new LocalRegistry(5000); @BeforeClass @@ -62,6 +68,40 @@ public void testPull_v22() throws IOException, RegistryException { Assert.assertTrue(v22ManifestTemplate.getLayers().size() > 0); } + @Test + public void testPull_v22ManifestList() throws IOException, RegistryException { + RegistryClient.Factory factory = + RegistryClient.factory(EventHandlers.NONE, "registry-1.docker.io", "library/openjdk"); + Authorization authorization = + factory.newRegistryClient().getRegistryAuthenticator().authenticatePull(null); + RegistryClient registryClient = factory.setAuthorization(authorization).newRegistryClient(); + + // Ensure 11-jre-slim is a manifest list + V22ManifestListTemplate manifestListTemplate = + registryClient.pullManifest("11-jre-slim", V22ManifestListTemplate.class); + Assert.assertEquals(2, manifestListTemplate.getSchemaVersion()); + Assert.assertTrue(manifestListTemplate.getManifests().size() > 0); + + // Generic call to 11-jre-slim should NOT pull a manifest list (delegate to registry default) + ManifestTemplate manifestTemplate = registryClient.pullManifest("11-jre-slim"); + Assert.assertEquals(2, manifestTemplate.getSchemaVersion()); + Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestTemplate.class)); + + // Make sure we can't cast a v22ManifestTemplate to v22ManifestListTemplate in ManifestPuller + try { + registryClient.pullManifest(KNOWN_MANIFEST_LIST_SHA, V22ManifestTemplate.class); + Assert.fail(); + } catch (ClassCastException ex) { + // pass + } + + // Referencing a manifest list by sha256, should return a manifest list + ManifestTemplate sha256ManifestList = registryClient.pullManifest(KNOWN_MANIFEST_LIST_SHA); + Assert.assertEquals(2, sha256ManifestList.getSchemaVersion()); + Assert.assertThat(sha256ManifestList, CoreMatchers.instanceOf(V22ManifestListTemplate.class)); + Assert.assertTrue(((V22ManifestListTemplate) sha256ManifestList).getManifests().size() > 0); + } + @Test public void testPull_unknownManifest() throws RegistryException, IOException { try { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java index c6dab7ed2f..bab08c12cf 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullBaseImageStep.java @@ -44,12 +44,14 @@ import com.google.cloud.tools.jib.image.json.ManifestTemplate; import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; +import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.cloud.tools.jib.registry.RegistryAuthenticator; import com.google.cloud.tools.jib.registry.RegistryClient; import com.google.cloud.tools.jib.registry.credentials.CredentialRetrievalException; import com.google.common.annotations.VisibleForTesting; import java.io.IOException; +import java.util.List; import java.util.Optional; import java.util.concurrent.Callable; import javax.annotation.Nullable; @@ -207,6 +209,19 @@ private Image pullBaseImage( ManifestTemplate manifestTemplate = registryClient.pullManifest(buildConfiguration.getBaseImageConfiguration().getImageTag()); + // special handling if we happen upon a manifest list, redirect to a manifest and continue + // handling it normally + if (manifestTemplate instanceof V22ManifestListTemplate) { + buildConfiguration + .getEventHandlers() + .dispatch( + LogEvent.lifecycle( + "The base image reference is manifest list, searching for linux/amd64")); + manifestTemplate = + obtainPlatformSpecificImageManifest( + registryClient, (V22ManifestListTemplate) manifestTemplate); + } + // TODO: Make schema version be enum. switch (manifestTemplate.getSchemaVersion()) { case 1: @@ -258,6 +273,25 @@ private Image pullBaseImage( throw new IllegalStateException("Unknown manifest schema version"); } + /** + * Looks through a manifest list for any amd64/linux manifest and downloads and returns the first + * manifest it finds. + */ + private ManifestTemplate obtainPlatformSpecificImageManifest( + RegistryClient registryClient, V22ManifestListTemplate manifestListTemplate) + throws RegistryException, IOException { + + List digests = manifestListTemplate.getDigestsForPlatform("amd64", "linux"); + if (digests.size() == 0) { + String errorMessage = + "Unable to find amd64/linux manifest in manifest list at: " + + buildConfiguration.getBaseImageConfiguration().getImage(); + buildConfiguration.getEventHandlers().dispatch(LogEvent.error(errorMessage)); + throw new RegistryException(errorMessage); + } + return registryClient.pullManifest(digests.get(0)); + } + /** * Retrieves the cached base image. * diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java index f758411e27..de5437aea6 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/image/json/V22ManifestListTemplate.java @@ -76,7 +76,7 @@ public int getSchemaVersion() { @Nullable private List manifests; @VisibleForTesting - List getManifests() { + public List getManifests() { return Preconditions.checkNotNull(manifests); } @@ -93,10 +93,10 @@ public List getDigestsForPlatform(String architecture, String os) { } /** Template for inner JSON object representing a single platform specific manifest. */ - static class ManifestDescriptorTemplate implements JsonTemplate { + public static class ManifestDescriptorTemplate implements JsonTemplate { @JsonIgnoreProperties(ignoreUnknown = true) - static class Platform implements JsonTemplate { + public static class Platform implements JsonTemplate { @Nullable private String architecture; @Nullable private String os; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java index 615a4a9500..84918eb0ed 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/ManifestPuller.java @@ -25,6 +25,7 @@ import com.google.cloud.tools.jib.image.json.OCIManifestTemplate; import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; +import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.io.CharStreams; @@ -71,7 +72,13 @@ public List getAccept() { if (manifestTemplateClass.equals(OCIManifestTemplate.class)) { return Collections.singletonList(OCIManifestTemplate.MANIFEST_MEDIA_TYPE); } + if (manifestTemplateClass.equals(V22ManifestListTemplate.class)) { + return Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE); + } + // V22ManifestListTemplate is not included by default, we don't explicitly accept + // it, we only handle it if referenced by sha256 (see getManifestTemplateFromJson) in which + // case registries ignore the "accept" directive and just return a manifest list anyway. return Arrays.asList( OCIManifestTemplate.MANIFEST_MEDIA_TYPE, V22ManifestTemplate.MANIFEST_MEDIA_TYPE, @@ -118,10 +125,6 @@ private T getManifestTemplateFromJson(String jsonString) throw new UnknownManifestFormatException("Cannot find field 'schemaVersion' in manifest"); } - if (!manifestTemplateClass.equals(ManifestTemplate.class)) { - return JsonTemplateMapper.readJson(jsonString, manifestTemplateClass); - } - int schemaVersion = node.get("schemaVersion").asInt(-1); if (schemaVersion == -1) { throw new UnknownManifestFormatException("`schemaVersion` field is not an integer"); @@ -142,6 +145,10 @@ private T getManifestTemplateFromJson(String jsonString) return manifestTemplateClass.cast( JsonTemplateMapper.readJson(jsonString, OCIManifestTemplate.class)); } + if (V22ManifestListTemplate.MANIFEST_MEDIA_TYPE.equals(mediaType)) { + return manifestTemplateClass.cast( + JsonTemplateMapper.readJson(jsonString, V22ManifestListTemplate.class)); + } throw new UnknownManifestFormatException("Unknown mediaType: " + mediaType); } throw new UnknownManifestFormatException( diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java index 4dfe8dd52b..9e8cf51bd9 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/registry/RegistryClient.java @@ -28,10 +28,9 @@ import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.global.JibSystemProperties; import com.google.cloud.tools.jib.http.Authorization; +import com.google.cloud.tools.jib.http.Response; import com.google.cloud.tools.jib.image.json.BuildableManifestTemplate; import com.google.cloud.tools.jib.image.json.ManifestTemplate; -import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; -import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.cloud.tools.jib.json.JsonTemplate; import com.google.cloud.tools.jib.json.JsonTemplateMapper; import com.google.common.annotations.VisibleForTesting; @@ -282,7 +281,8 @@ public RegistryAuthenticator getRegistryAuthenticator() throws IOException, Regi * @param child type of ManifestTemplate * @param imageTag the tag to pull on * @param manifestTemplateClass the specific version of manifest template to pull, or {@link - * ManifestTemplate} to pull either {@link V22ManifestTemplate} or {@link V21ManifestTemplate} + * ManifestTemplate} to pull predefined subclasses; see: {@link + * ManifestPuller#handleResponse(Response)} * @return the manifest template * @throws IOException if communicating with the endpoint fails * @throws RegistryException if communicating with the endpoint fails diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java index 0c5a37fdd4..cd583c5381 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/registry/ManifestPullerTest.java @@ -21,6 +21,7 @@ import com.google.cloud.tools.jib.image.json.OCIManifestTemplate; import com.google.cloud.tools.jib.image.json.UnknownManifestFormatException; import com.google.cloud.tools.jib.image.json.V21ManifestTemplate; +import com.google.cloud.tools.jib.image.json.V22ManifestListTemplate; import com.google.cloud.tools.jib.image.json.V22ManifestTemplate; import com.google.common.io.Resources; import java.io.ByteArrayInputStream; @@ -89,6 +90,60 @@ public void testHandleResponse_v22() Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestTemplate.class)); } + @Test + public void testHandleResponse_v22ManifestListFailsWhenParsedAsV22Manifest() + throws URISyntaxException, IOException, UnknownManifestFormatException { + Path v22ManifestListFile = + Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI()); + InputStream v22ManifestList = new ByteArrayInputStream(Files.readAllBytes(v22ManifestListFile)); + + Mockito.when(mockResponse.getBody()).thenReturn(v22ManifestList); + try { + new ManifestPuller<>( + fakeRegistryEndpointRequestProperties, "test-image-tag", V22ManifestTemplate.class) + .handleResponse(mockResponse); + Assert.fail(); + } catch (ClassCastException ex) { + // pass + } + } + + @Test + public void testHandleResponse_v22ManifestListFromParentType() + throws URISyntaxException, IOException, UnknownManifestFormatException { + Path v22ManifestListFile = + Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI()); + InputStream v22ManifestList = new ByteArrayInputStream(Files.readAllBytes(v22ManifestListFile)); + + Mockito.when(mockResponse.getBody()).thenReturn(v22ManifestList); + ManifestTemplate manifestTemplate = + new ManifestPuller<>( + fakeRegistryEndpointRequestProperties, "test-image-tag", ManifestTemplate.class) + .handleResponse(mockResponse); + + Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestListTemplate.class)); + Assert.assertTrue(((V22ManifestListTemplate) manifestTemplate).getManifests().size() > 0); + } + + @Test + public void testHandleResponse_v22ManifestList() + throws URISyntaxException, IOException, UnknownManifestFormatException { + Path v22ManifestListFile = + Paths.get(Resources.getResource("core/json/v22manifest_list.json").toURI()); + InputStream v22ManifestList = new ByteArrayInputStream(Files.readAllBytes(v22ManifestListFile)); + + Mockito.when(mockResponse.getBody()).thenReturn(v22ManifestList); + V22ManifestListTemplate manifestTemplate = + new ManifestPuller<>( + fakeRegistryEndpointRequestProperties, + "test-image-tag", + V22ManifestListTemplate.class) + .handleResponse(mockResponse); + + Assert.assertThat(manifestTemplate, CoreMatchers.instanceOf(V22ManifestListTemplate.class)); + Assert.assertTrue(manifestTemplate.getManifests().size() > 0); + } + @Test public void testHandleResponse_noSchemaVersion() throws IOException { Mockito.when(mockResponse.getBody()).thenReturn(stringToInputStreamUtf8("{}")); @@ -175,5 +230,12 @@ public void testGetAccept() { new ManifestPuller<>( fakeRegistryEndpointRequestProperties, "test-image-tag", V21ManifestTemplate.class) .getAccept()); + Assert.assertEquals( + Collections.singletonList(V22ManifestListTemplate.MANIFEST_MEDIA_TYPE), + new ManifestPuller<>( + fakeRegistryEndpointRequestProperties, + "test-image-tag", + V22ManifestListTemplate.class) + .getAccept()); } }