Skip downloading base image layers if they exists in target repository

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java

@@ -34,7 +34,7 @@
 import java.util.concurrent.Callable;
 
 /** Builds and caches application layers. */
-class BuildAndCacheApplicationLayerStep implements Callable<CachedLayerAndName> {
+class BuildAndCacheApplicationLayerStep implements Callable<PreparedLayer> {
 
   private static final String DESCRIPTION = "Building %s layer";
 
@@ -86,7 +86,7 @@ private BuildAndCacheApplicationLayerStep(
   }
 
   @Override
-  public CachedLayerAndName call() throws IOException, CacheCorruptedException {
+  public PreparedLayer call() throws IOException, CacheCorruptedException {
     String description = String.format(DESCRIPTION, layerName);
 
     EventHandlers eventHandlers = buildConfiguration.getEventHandlers();
@@ -101,7 +101,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException {
       Optional<CachedLayer> optionalCachedLayer =
           cache.retrieve(layerConfiguration.getLayerEntries());
       if (optionalCachedLayer.isPresent()) {
-        return new CachedLayerAndName(optionalCachedLayer.get(), layerName);
+        return new PreparedLayer.Builder(optionalCachedLayer.get()).setName(layerName).build();
       }
 
       Blob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries()).build();
@@ -110,7 +110,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException {
 
       eventHandlers.dispatch(LogEvent.debug(description + " built " + cachedLayer.getDigest()));
 
-      return new CachedLayerAndName(cachedLayer, layerName);
+      return new PreparedLayer.Builder(cachedLayer).setName(layerName).build();
     }
   }
 }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java

@@ -25,7 +25,6 @@
 import com.google.cloud.tools.jib.image.Image;
 import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException;
 import com.google.cloud.tools.jib.image.json.HistoryEntry;
-import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
 import java.time.Instant;
 import java.util.List;
@@ -40,15 +39,15 @@ class BuildImageStep implements Callable<Image> {
   private final BuildConfiguration buildConfiguration;
   private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;
   private final Image baseImage;
-  private final List<CachedLayerAndName> baseImageLayers;
-  private final List<CachedLayerAndName> applicationLayers;
+  private final List<PreparedLayer> baseImageLayers;
+  private final List<PreparedLayer> applicationLayers;
 
   BuildImageStep(
       BuildConfiguration buildConfiguration,
       ProgressEventDispatcher.Factory progressEventDispatcherFactory,
       Image baseImage,
-      List<CachedLayerAndName> baseImageLayers,
-      List<CachedLayerAndName> applicationLayers) {
+      List<PreparedLayer> baseImageLayers,
+      List<PreparedLayer> applicationLayers) {
     this.buildConfiguration = buildConfiguration;
     this.progressEventDispatcherFactory = progressEventDispatcherFactory;
     this.baseImage = baseImage;
@@ -68,9 +67,7 @@ public Image call() throws LayerPropertyNotFoundException {
           buildConfiguration.getContainerConfiguration();
 
       // Base image layers
-      for (CachedLayerAndName baseImageLayer : baseImageLayers) {
-        imageBuilder.addLayer(baseImageLayer.getCachedLayer());
-      }
+      baseImageLayers.stream().forEach(imageBuilder::addLayer);
 
       // Passthrough config and count non-empty history entries
       int nonEmptyLayerCount = 0;
@@ -104,15 +101,15 @@ public Image call() throws LayerPropertyNotFoundException {
       }
 
       // Add built layers/configuration
-      for (CachedLayerAndName applicationLayer : applicationLayers) {
+      for (PreparedLayer applicationLayer : applicationLayers) {
         imageBuilder
-            .addLayer(applicationLayer.getCachedLayer())
+            .addLayer(applicationLayer)
             .addHistory(
                 HistoryEntry.builder()
                     .setCreationTimestamp(layerCreationTime)
                     .setAuthor("Jib")
                     .setCreatedBy(buildConfiguration.getToolName() + ":" + ProjectInfo.VERSION)
-                    .setComment(Verify.verifyNotNull(applicationLayer.getName()))
+                    .setComment(applicationLayer.getName())
                     .build());
       }
       if (containerConfiguration != null) {

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java → jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java

@@ -17,6 +17,7 @@
 package com.google.cloud.tools.jib.builder.steps;
 
 import com.google.cloud.tools.jib.api.DescriptorDigest;
+import com.google.cloud.tools.jib.api.RegistryException;
 import com.google.cloud.tools.jib.builder.ProgressEventDispatcher;
 import com.google.cloud.tools.jib.builder.TimerEventDispatcher;
 import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization;
@@ -27,6 +28,7 @@
 import com.google.cloud.tools.jib.http.Authorization;
 import com.google.cloud.tools.jib.image.Layer;
 import com.google.cloud.tools.jib.registry.RegistryClient;
+import com.google.common.base.Verify;
 import com.google.common.collect.ImmutableList;
 import java.io.IOException;
 import java.util.ArrayList;
@@ -36,14 +38,50 @@
 import javax.annotation.Nullable;
 
 /** Pulls and caches a single base image layer. */
-class PullAndCacheBaseImageLayerStep implements Callable<CachedLayerAndName> {
+class ObtainBaseImageLayerStep implements Callable<PreparedLayer> {
 
   private static final String DESCRIPTION = "Pulling base image layer %s";
 
-  static ImmutableList<PullAndCacheBaseImageLayerStep> makeList(
+  @FunctionalInterface
+  private interface BlobExistenceChecker {
+
+    Optional<Boolean> exists(DescriptorDigest digest) throws IOException, RegistryException;
+  }
+
+  static ImmutableList<ObtainBaseImageLayerStep> makeListForForcedDownload(
       BuildConfiguration buildConfiguration,
       ProgressEventDispatcher.Factory progressEventDispatcherFactory,
       ImageAndAuthorization baseImageAndAuth) {
+    BlobExistenceChecker noOpBlobChecker = ignored -> Optional.empty();
+    return makeList(
+        buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, noOpBlobChecker);
+  }
+
+  static ImmutableList<ObtainBaseImageLayerStep> makeListForSelectiveDownload(
+      BuildConfiguration buildConfiguration,
+      ProgressEventDispatcher.Factory progressEventDispatcherFactory,
+      ImageAndAuthorization baseImageAndAuth,
+      Authorization pushAuthorization) {
+    Verify.verify(!buildConfiguration.isOffline());
+
+    RegistryClient targetRegistryClient =
+        buildConfiguration
+            .newTargetImageRegistryClientFactory()
+            .setAuthorization(pushAuthorization)
+            .newRegistryClient();
+    // TODO: also check if cross-repo blob mount is possible.
+    BlobExistenceChecker blobExistenceChecker =
+        digest -> Optional.of(targetRegistryClient.checkBlob(digest).isPresent());
+
+    return makeList(
+        buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobExistenceChecker);
+  }
+
+  private static ImmutableList<ObtainBaseImageLayerStep> makeList(
+      BuildConfiguration buildConfiguration,
+      ProgressEventDispatcher.Factory progressEventDispatcherFactory,
+      ImageAndAuthorization baseImageAndAuth,
+      BlobExistenceChecker blobExistenceChecker) {
     ImmutableList<Layer> baseImageLayers = baseImageAndAuth.getImage().getLayers();
 
     try (ProgressEventDispatcher progressEventDispatcher =
@@ -53,14 +91,15 @@ static ImmutableList<PullAndCacheBaseImageLayerStep> makeList(
             new TimerEventDispatcher(
                 buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) {
 
-      List<PullAndCacheBaseImageLayerStep> layerPullers = new ArrayList<>();
+      List<ObtainBaseImageLayerStep> layerPullers = new ArrayList<>();
       for (Layer layer : baseImageLayers) {
         layerPullers.add(
-            new PullAndCacheBaseImageLayerStep(
+            new ObtainBaseImageLayerStep(
                 buildConfiguration,
                 progressEventDispatcher.newChildProducer(),
-                layer.getBlobDescriptor().getDigest(),
-                baseImageAndAuth.getAuthorization()));
+                layer,
+                baseImageAndAuth.getAuthorization(),
+                blobExistenceChecker));
       }
       return ImmutableList.copyOf(layerPullers);
     }
@@ -69,38 +108,50 @@ static ImmutableList<PullAndCacheBaseImageLayerStep> makeList(
   private final BuildConfiguration buildConfiguration;
   private final ProgressEventDispatcher.Factory progressEventDispatcherFactory;
 
-  private final DescriptorDigest layerDigest;
+  private final Layer layer;
   private final @Nullable Authorization pullAuthorization;
+  private final BlobExistenceChecker blobExistenceChecker;
 
-  PullAndCacheBaseImageLayerStep(
+  ObtainBaseImageLayerStep(
       BuildConfiguration buildConfiguration,
       ProgressEventDispatcher.Factory progressEventDispatcherFactory,
-      DescriptorDigest layerDigest,
-      @Nullable Authorization pullAuthorization) {
+      Layer layer,
+      @Nullable Authorization pullAuthorization,
+      BlobExistenceChecker blobExistenceChecker) {
     this.buildConfiguration = buildConfiguration;
     this.progressEventDispatcherFactory = progressEventDispatcherFactory;
-    this.layerDigest = layerDigest;
+    this.layer = layer;
     this.pullAuthorization = pullAuthorization;
+    this.blobExistenceChecker = blobExistenceChecker;
   }
 
   @Override
-  public CachedLayerAndName call() throws IOException, CacheCorruptedException {
+  public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException {
+    DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest();
     try (ProgressEventDispatcher progressEventDispatcher =
             progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1);
         TimerEventDispatcher ignored =
             new TimerEventDispatcher(
                 buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) {
+
+      Optional<Boolean> layerExists = blobExistenceChecker.exists(layerDigest);
+      if (layerExists.orElse(false)) {
+        return new PreparedLayer.Builder(layer).setStateInTarget(layerExists).build();
+      }
+
       Cache cache = buildConfiguration.getBaseImageLayersCache();
 
       // Checks if the layer already exists in the cache.
       Optional<CachedLayer> optionalCachedLayer = cache.retrieve(layerDigest);
       if (optionalCachedLayer.isPresent()) {
-        return new CachedLayerAndName(optionalCachedLayer.get(), null);
+        CachedLayer cachedLayer = optionalCachedLayer.get();
+        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build();
       } else if (buildConfiguration.isOffline()) {
         throw new IOException(
             "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer "
                 + layerDigest
-                + ". You may need to rerun Jib in online mode to re-download the base image layers.");
+                + ". Rerun Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to "
+                + "re-download the base image layers.");
       }
 
       RegistryClient registryClient =
@@ -119,7 +170,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException {
                     layerDigest,
                     progressEventDispatcherWrapper::setProgressTarget,
                     progressEventDispatcherWrapper::dispatchProgress));
-        return new CachedLayerAndName(cachedLayer, null);
+        return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build();
       }
     }
   }

jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java

@@ -0,0 +1,90 @@
+/*
+ * Copyright 2019 Google LLC.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package com.google.cloud.tools.jib.builder.steps;
+
+import com.google.cloud.tools.jib.api.DescriptorDigest;
+import com.google.cloud.tools.jib.blob.Blob;
+import com.google.cloud.tools.jib.blob.BlobDescriptor;
+import com.google.cloud.tools.jib.image.Layer;
+import java.util.Optional;
+
+/**
+ * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link
+ * ObtainBaseImageLayerStep} to hold information about either a base image layer or an application
+ * layer.
+ */
+class PreparedLayer implements Layer {
+
+  static class Builder {
+
+    private Layer layer;
+    private String name = "unnamed layer";
+    private Optional<Boolean> stateInTarget = Optional.empty();
+
+    Builder(Layer layer) {
+      this.layer = layer;
+    }
+
+    Builder setName(String name) {
+      this.name = name;
+      return this;
+    }
+
+    /** Sets whether the layer exists in a target destination. Empty (absence) means unknown. */
+    Builder setStateInTarget(Optional<Boolean> stateInTarget) {
+      this.stateInTarget = stateInTarget;
+      return this;
+    }
+
+    PreparedLayer build() {
+      return new PreparedLayer(layer, name, stateInTarget);
+    }
+  }
+
+  private final Layer layer;
+  private final String name;
+  private final Optional<Boolean> stateInTarget;
+
+  private PreparedLayer(Layer layer, String name, Optional<Boolean> stateInTarget) {
+    this.layer = layer;
+    this.name = name;
+    this.stateInTarget = stateInTarget;
+  }
+
+  String getName() {
+    return name;
+  }
+
+  Optional<Boolean> existsInTarget() {
+    return stateInTarget;
+  }
+
+  @Override
+  public Blob getBlob() {
+    return layer.getBlob();
+  }
+
+  @Override
+  public BlobDescriptor getBlobDescriptor() {
+    return layer.getBlobDescriptor();
+  }
+
+  @Override
+  public DescriptorDigest getDiffId() {
+    return layer.getDiffId();
+  }
+}