diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java index 96a3d3cafd..10dfc33153 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStep.java @@ -34,7 +34,7 @@ import java.util.concurrent.Callable; /** Builds and caches application layers. */ -class BuildAndCacheApplicationLayerStep implements Callable { +class BuildAndCacheApplicationLayerStep implements Callable { private static final String DESCRIPTION = "Building %s layer"; @@ -86,7 +86,7 @@ private BuildAndCacheApplicationLayerStep( } @Override - public CachedLayerAndName call() throws IOException, CacheCorruptedException { + public PreparedLayer call() throws IOException, CacheCorruptedException { String description = String.format(DESCRIPTION, layerName); EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); @@ -101,7 +101,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { Optional optionalCachedLayer = cache.retrieve(layerConfiguration.getLayerEntries()); if (optionalCachedLayer.isPresent()) { - return new CachedLayerAndName(optionalCachedLayer.get(), layerName); + return new PreparedLayer.Builder(optionalCachedLayer.get()).setName(layerName).build(); } Blob layerBlob = new ReproducibleLayerBuilder(layerConfiguration.getLayerEntries()).build(); @@ -110,7 +110,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { eventHandlers.dispatch(LogEvent.debug(description + " built " + cachedLayer.getDigest())); - return new CachedLayerAndName(cachedLayer, layerName); + return new PreparedLayer.Builder(cachedLayer).setName(layerName).build(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java index bb3783eb75..e221317557 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/BuildImageStep.java @@ -25,7 +25,6 @@ import com.google.cloud.tools.jib.image.Image; import com.google.cloud.tools.jib.image.LayerPropertyNotFoundException; import com.google.cloud.tools.jib.image.json.HistoryEntry; -import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import java.time.Instant; import java.util.List; @@ -40,15 +39,15 @@ class BuildImageStep implements Callable { private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; private final Image baseImage; - private final List baseImageLayers; - private final List applicationLayers; + private final List baseImageLayers; + private final List applicationLayers; BuildImageStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, Image baseImage, - List baseImageLayers, - List applicationLayers) { + List baseImageLayers, + List applicationLayers) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.baseImage = baseImage; @@ -68,9 +67,7 @@ public Image call() throws LayerPropertyNotFoundException { buildConfiguration.getContainerConfiguration(); // Base image layers - for (CachedLayerAndName baseImageLayer : baseImageLayers) { - imageBuilder.addLayer(baseImageLayer.getCachedLayer()); - } + baseImageLayers.stream().forEach(imageBuilder::addLayer); // Passthrough config and count non-empty history entries int nonEmptyLayerCount = 0; @@ -104,15 +101,15 @@ public Image call() throws LayerPropertyNotFoundException { } // Add built layers/configuration - for (CachedLayerAndName applicationLayer : applicationLayers) { + for (PreparedLayer applicationLayer : applicationLayers) { imageBuilder - .addLayer(applicationLayer.getCachedLayer()) + .addLayer(applicationLayer) .addHistory( HistoryEntry.builder() .setCreationTimestamp(layerCreationTime) .setAuthor("Jib") .setCreatedBy(buildConfiguration.getToolName() + ":" + ProjectInfo.VERSION) - .setComment(Verify.verifyNotNull(applicationLayer.getName())) + .setComment(applicationLayer.getName()) .build()); } if (containerConfiguration != null) { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java deleted file mode 100644 index eed59fc14e..0000000000 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/CachedLayerAndName.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2019 Google LLC. - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - -package com.google.cloud.tools.jib.builder.steps; - -import com.google.cloud.tools.jib.cache.CachedLayer; -import javax.annotation.Nullable; - -/** Simple structure to hold a pair of {#link CachedLayer} and its string name. */ -class CachedLayerAndName { - - private CachedLayer cachedLayer; - @Nullable private String name; - - CachedLayerAndName(CachedLayer cachedLayer, @Nullable String name) { - this.cachedLayer = cachedLayer; - this.name = name; - } - - CachedLayer getCachedLayer() { - return cachedLayer; - } - - @Nullable - String getName() { - return name; - } -} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java similarity index 60% rename from jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java rename to jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java index 0868aef0ac..735be6a676 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PullAndCacheBaseImageLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStep.java @@ -17,6 +17,7 @@ package com.google.cloud.tools.jib.builder.steps; import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; @@ -27,6 +28,7 @@ import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.image.Layer; import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.base.Verify; import com.google.common.collect.ImmutableList; import java.io.IOException; import java.util.ArrayList; @@ -36,14 +38,50 @@ import javax.annotation.Nullable; /** Pulls and caches a single base image layer. */ -class PullAndCacheBaseImageLayerStep implements Callable { +class ObtainBaseImageLayerStep implements Callable { private static final String DESCRIPTION = "Pulling base image layer %s"; - static ImmutableList makeList( + @FunctionalInterface + private interface BlobExistenceChecker { + + Optional exists(DescriptorDigest digest) throws IOException, RegistryException; + } + + static ImmutableList makeListForForcedDownload( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, ImageAndAuthorization baseImageAndAuth) { + BlobExistenceChecker noOpBlobChecker = ignored -> Optional.empty(); + return makeList( + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, noOpBlobChecker); + } + + static ImmutableList makeListForSelectiveDownload( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + ImageAndAuthorization baseImageAndAuth, + Authorization pushAuthorization) { + Verify.verify(!buildConfiguration.isOffline()); + + RegistryClient targetRegistryClient = + buildConfiguration + .newTargetImageRegistryClientFactory() + .setAuthorization(pushAuthorization) + .newRegistryClient(); + // TODO: also check if cross-repo blob mount is possible. + BlobExistenceChecker blobExistenceChecker = + digest -> Optional.of(targetRegistryClient.checkBlob(digest).isPresent()); + + return makeList( + buildConfiguration, progressEventDispatcherFactory, baseImageAndAuth, blobExistenceChecker); + } + + private static ImmutableList makeList( + BuildConfiguration buildConfiguration, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, + ImageAndAuthorization baseImageAndAuth, + BlobExistenceChecker blobExistenceChecker) { ImmutableList baseImageLayers = baseImageAndAuth.getImage().getLayers(); try (ProgressEventDispatcher progressEventDispatcher = @@ -53,14 +91,15 @@ static ImmutableList makeList( new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing base image layer pullers")) { - List layerPullers = new ArrayList<>(); + List layerPullers = new ArrayList<>(); for (Layer layer : baseImageLayers) { layerPullers.add( - new PullAndCacheBaseImageLayerStep( + new ObtainBaseImageLayerStep( buildConfiguration, progressEventDispatcher.newChildProducer(), - layer.getBlobDescriptor().getDigest(), - baseImageAndAuth.getAuthorization())); + layer, + baseImageAndAuth.getAuthorization(), + blobExistenceChecker)); } return ImmutableList.copyOf(layerPullers); } @@ -69,38 +108,50 @@ static ImmutableList makeList( private final BuildConfiguration buildConfiguration; private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; - private final DescriptorDigest layerDigest; + private final Layer layer; private final @Nullable Authorization pullAuthorization; + private final BlobExistenceChecker blobExistenceChecker; - PullAndCacheBaseImageLayerStep( + ObtainBaseImageLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, - DescriptorDigest layerDigest, - @Nullable Authorization pullAuthorization) { + Layer layer, + @Nullable Authorization pullAuthorization, + BlobExistenceChecker blobExistenceChecker) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; - this.layerDigest = layerDigest; + this.layer = layer; this.pullAuthorization = pullAuthorization; + this.blobExistenceChecker = blobExistenceChecker; } @Override - public CachedLayerAndName call() throws IOException, CacheCorruptedException { + public PreparedLayer call() throws IOException, CacheCorruptedException, RegistryException { + DescriptorDigest layerDigest = layer.getBlobDescriptor().getDigest(); try (ProgressEventDispatcher progressEventDispatcher = progressEventDispatcherFactory.create("checking base image layer " + layerDigest, 1); TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), String.format(DESCRIPTION, layerDigest))) { + + Optional layerExists = blobExistenceChecker.exists(layerDigest); + if (layerExists.orElse(false)) { + return new PreparedLayer.Builder(layer).setStateInTarget(layerExists).build(); + } + Cache cache = buildConfiguration.getBaseImageLayersCache(); // Checks if the layer already exists in the cache. Optional optionalCachedLayer = cache.retrieve(layerDigest); if (optionalCachedLayer.isPresent()) { - return new CachedLayerAndName(optionalCachedLayer.get(), null); + CachedLayer cachedLayer = optionalCachedLayer.get(); + return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build(); } else if (buildConfiguration.isOffline()) { throw new IOException( "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + layerDigest - + ". You may need to rerun Jib in online mode to re-download the base image layers."); + + ". Rerun Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to " + + "re-download the base image layers."); } RegistryClient registryClient = @@ -119,7 +170,7 @@ public CachedLayerAndName call() throws IOException, CacheCorruptedException { layerDigest, progressEventDispatcherWrapper::setProgressTarget, progressEventDispatcherWrapper::dispatchProgress)); - return new CachedLayerAndName(cachedLayer, null); + return new PreparedLayer.Builder(cachedLayer).setStateInTarget(layerExists).build(); } } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java new file mode 100644 index 0000000000..dd02e8a2cd --- /dev/null +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PreparedLayer.java @@ -0,0 +1,90 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.image.Layer; +import java.util.Optional; + +/** + * Layer prepared from {@link BuildAndCacheApplicationLayerStep} and {@link + * ObtainBaseImageLayerStep} to hold information about either a base image layer or an application + * layer. + */ +class PreparedLayer implements Layer { + + static class Builder { + + private Layer layer; + private String name = "unnamed layer"; + private Optional stateInTarget = Optional.empty(); + + Builder(Layer layer) { + this.layer = layer; + } + + Builder setName(String name) { + this.name = name; + return this; + } + + /** Sets whether the layer exists in a target destination. Empty (absence) means unknown. */ + Builder setStateInTarget(Optional stateInTarget) { + this.stateInTarget = stateInTarget; + return this; + } + + PreparedLayer build() { + return new PreparedLayer(layer, name, stateInTarget); + } + } + + private final Layer layer; + private final String name; + private final Optional stateInTarget; + + private PreparedLayer(Layer layer, String name, Optional stateInTarget) { + this.layer = layer; + this.name = name; + this.stateInTarget = stateInTarget; + } + + String getName() { + return name; + } + + Optional existsInTarget() { + return stateInTarget; + } + + @Override + public Blob getBlob() { + return layer.getBlob(); + } + + @Override + public BlobDescriptor getBlobDescriptor() { + return layer.getBlobDescriptor(); + } + + @Override + public DescriptorDigest getDiffId() { + return layer.getDiffId(); + } +} diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java index c3711d8ef0..12bba2e04e 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushBlobStep.java @@ -16,6 +16,7 @@ package com.google.cloud.tools.jib.builder.steps; +import com.google.cloud.tools.jib.api.DescriptorDigest; import com.google.cloud.tools.jib.api.LogEvent; import com.google.cloud.tools.jib.api.RegistryException; import com.google.cloud.tools.jib.blob.Blob; @@ -23,6 +24,7 @@ import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.event.EventHandlers; import com.google.cloud.tools.jib.event.progress.ThrottledAccumulatingConsumer; import com.google.cloud.tools.jib.http.Authorization; import com.google.cloud.tools.jib.registry.RegistryClient; @@ -36,33 +38,37 @@ class PushBlobStep implements Callable { private static final String DESCRIPTION = "Pushing BLOB "; private final BuildConfiguration buildConfiguration; - private final ProgressEventDispatcher.Factory progressEventDipatcherFactory; + private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; @Nullable private final Authorization authorization; private final BlobDescriptor blobDescriptor; private final Blob blob; + private final boolean forcePush; PushBlobStep( BuildConfiguration buildConfiguration, - ProgressEventDispatcher.Factory progressEventDipatcherFactory, + ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization authorization, BlobDescriptor blobDescriptor, - Blob blob) { + Blob blob, + boolean forcePush) { this.buildConfiguration = buildConfiguration; - this.progressEventDipatcherFactory = progressEventDipatcherFactory; + this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.authorization = authorization; this.blobDescriptor = blobDescriptor; this.blob = blob; + this.forcePush = forcePush; } @Override public BlobDescriptor call() throws IOException, RegistryException { + EventHandlers eventHandlers = buildConfiguration.getEventHandlers(); + DescriptorDigest blobDigest = blobDescriptor.getDigest(); try (ProgressEventDispatcher progressEventDispatcher = - progressEventDipatcherFactory.create( - "pushing blob " + blobDescriptor.getDigest(), blobDescriptor.getSize()); + progressEventDispatcherFactory.create( + "pushing blob " + blobDigest, blobDescriptor.getSize()); TimerEventDispatcher ignored = - new TimerEventDispatcher( - buildConfiguration.getEventHandlers(), DESCRIPTION + blobDescriptor); + new TimerEventDispatcher(eventHandlers, DESCRIPTION + blobDescriptor); ThrottledAccumulatingConsumer throttledProgressReporter = new ThrottledAccumulatingConsumer(progressEventDispatcher::dispatchProgress)) { RegistryClient registryClient = @@ -72,10 +78,9 @@ public BlobDescriptor call() throws IOException, RegistryException { .newRegistryClient(); // check if the BLOB is available - if (registryClient.checkBlob(blobDescriptor.getDigest()).isPresent()) { - buildConfiguration - .getEventHandlers() - .dispatch(LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); + if (!forcePush && registryClient.checkBlob(blobDigest).isPresent()) { + eventHandlers.dispatch( + LogEvent.info("BLOB : " + blobDescriptor + " already exists on registry")); return blobDescriptor; } @@ -87,8 +92,7 @@ public BlobDescriptor call() throws IOException, RegistryException { String baseRepository = buildConfiguration.getBaseImageConfiguration().getImageRepository(); String targetRegistry = buildConfiguration.getTargetImageConfiguration().getImageRegistry(); String sourceRepository = targetRegistry.equals(baseRegistry) ? baseRepository : null; - registryClient.pushBlob( - blobDescriptor.getDigest(), blob, sourceRepository, throttledProgressReporter); + registryClient.pushBlob(blobDigest, blob, sourceRepository, throttledProgressReporter); return blobDescriptor; } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java index 87e12d064b..54049feaa7 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushContainerConfigurationStep.java @@ -67,7 +67,8 @@ public BlobDescriptor call() throws IOException, RegistryException { progressEventDispatcher.newChildProducer(), pushAuthorization, Digests.computeDigest(containerConfiguration), - Blobs.from(containerConfiguration)) + Blobs.from(containerConfiguration), + false) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java index 05fb34b8f9..de654b26f1 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/PushLayerStep.java @@ -20,12 +20,10 @@ import com.google.cloud.tools.jib.blob.BlobDescriptor; import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; import com.google.cloud.tools.jib.builder.TimerEventDispatcher; -import com.google.cloud.tools.jib.cache.CachedLayer; import com.google.cloud.tools.jib.configuration.BuildConfiguration; import com.google.cloud.tools.jib.http.Authorization; import com.google.common.collect.ImmutableList; import java.io.IOException; -import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; @@ -38,7 +36,7 @@ static ImmutableList makeList( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - List> cachedLayers) { + List> cachedLayers) { try (TimerEventDispatcher ignored = new TimerEventDispatcher( buildConfiguration.getEventHandlers(), "Preparing application layer pushers"); @@ -47,15 +45,16 @@ static ImmutableList makeList( "preparing application layer pushers", cachedLayers.size())) { // Constructs a PushBlobStep for each layer. - List blobPushers = new ArrayList<>(); - for (Future layer : cachedLayers) { - ProgressEventDispatcher.Factory childProgressProducer = - progressEventDispatcher.newChildProducer(); - blobPushers.add( - new PushLayerStep(buildConfiguration, childProgressProducer, pushAuthorization, layer)); - } - - return ImmutableList.copyOf(blobPushers); + return cachedLayers + .stream() + .map( + layer -> + new PushLayerStep( + buildConfiguration, + progressEventDispatcher.newChildProducer(), + pushAuthorization, + layer)) + .collect(ImmutableList.toImmutableList()); } } @@ -63,29 +62,36 @@ static ImmutableList makeList( private final ProgressEventDispatcher.Factory progressEventDispatcherFactory; @Nullable private final Authorization pushAuthorization; - private final Future cachedLayerAndName; + private final Future preparedLayer; PushLayerStep( BuildConfiguration buildConfiguration, ProgressEventDispatcher.Factory progressEventDispatcherFactory, @Nullable Authorization pushAuthorization, - Future cachedLayerAndName) { + Future preparedLayer) { this.buildConfiguration = buildConfiguration; this.progressEventDispatcherFactory = progressEventDispatcherFactory; this.pushAuthorization = pushAuthorization; - this.cachedLayerAndName = cachedLayerAndName; + this.preparedLayer = preparedLayer; } @Override public BlobDescriptor call() throws IOException, RegistryException, ExecutionException, InterruptedException { - CachedLayer layer = cachedLayerAndName.get().getCachedLayer(); + PreparedLayer layer = preparedLayer.get(); + + boolean queriedExistence = layer.existsInTarget().isPresent(); + if (queriedExistence && layer.existsInTarget().get()) { + return layer.getBlobDescriptor(); // skip pushing if known to exist in registry + } + return new PushBlobStep( buildConfiguration, progressEventDispatcherFactory, pushAuthorization, - new BlobDescriptor(layer.getSize(), layer.getDigest()), - layer.getBlob()) + layer.getBlobDescriptor(), + layer.getBlob(), + queriedExistence) .call(); } } diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java index cf0e838f31..1a73e959c2 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/builder/steps/StepsRunner.java @@ -60,8 +60,8 @@ private static Future failedFuture() { } private Future baseImageAndAuth = failedFuture(); - private Future>> baseImageLayers = failedFuture(); - @Nullable private List> applicationLayers; + private Future>> baseImageLayers = failedFuture(); + @Nullable private List> applicationLayers; private Future builtImage = failedFuture(); private Future> targetRegistryCredentials = failedFuture(); private Future> pushAuthorization = failedFuture(); @@ -120,9 +120,10 @@ private StepsRunner( public StepsRunner dockerLoadSteps(DockerClient dockerClient) { rootProgressDescription = "building image to Docker daemon"; + // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> obtainBaseImageLayers(true)); // always pull layers for docker builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // load to Docker @@ -132,9 +133,10 @@ public StepsRunner dockerLoadSteps(DockerClient dockerClient) { public StepsRunner tarBuildSteps(Path outputPath) { rootProgressDescription = "building image to tar file"; + // build and cache stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> obtainBaseImageLayers(true)); // always pull layers for tar builds stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); // create a tar @@ -144,14 +146,17 @@ public StepsRunner tarBuildSteps(Path outputPath) { public StepsRunner registryPushSteps() { rootProgressDescription = "building image to registry"; - // build and cache + boolean layersRequiredLocally = JibSystemProperties.alwaysCacheBaseImage(); + + stepsToRun.add(this::retrieveTargetRegistryCredentials); + stepsToRun.add(this::authenticatePush); + stepsToRun.add(this::pullBaseImage); - stepsToRun.add(this::pullAndCacheBaseImageLayers); + stepsToRun.add(() -> obtainBaseImageLayers(layersRequiredLocally)); stepsToRun.add(this::buildAndCacheApplicationLayers); stepsToRun.add(this::buildImage); + // push to registry - stepsToRun.add(this::retrieveTargetRegistryCredentials); - stepsToRun.add(this::authenticatePush); stepsToRun.add(this::pushBaseImageLayers); stepsToRun.add(this::pushApplicationLayers); stepsToRun.add(this::pushContainerConfiguration); @@ -212,7 +217,7 @@ private void pullBaseImage() { new PullBaseImageStep(buildConfiguration, childProgressDispatcherFactory)); } - private void pullAndCacheBaseImageLayers() { + private void obtainBaseImageLayers(boolean layersRequiredLocally) { ProgressEventDispatcher.Factory childProgressDispatcherFactory = Verify.verifyNotNull(rootProgressDispatcher).newChildProducer(); @@ -220,10 +225,16 @@ private void pullAndCacheBaseImageLayers() { executorService.submit( () -> scheduleCallables( - PullAndCacheBaseImageLayerStep.makeList( - buildConfiguration, - childProgressDispatcherFactory, - results.baseImageAndAuth.get()))); + layersRequiredLocally + ? ObtainBaseImageLayerStep.makeListForForcedDownload( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get()) + : ObtainBaseImageLayerStep.makeListForSelectiveDownload( + buildConfiguration, + childProgressDispatcherFactory, + results.baseImageAndAuth.get(), + results.pushAuthorization.get().orElse(null)))); } private void pushBaseImageLayers() { diff --git a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java index 2d11ac09f7..4a97dcb82b 100644 --- a/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java +++ b/jib-core/src/main/java/com/google/cloud/tools/jib/global/JibSystemProperties.java @@ -27,6 +27,8 @@ public class JibSystemProperties { @VisibleForTesting static final String CROSS_REPOSITORY_BLOB_MOUNTS = "jib.blobMounts"; + @VisibleForTesting static final String ALWAYS_CACHE_BASE_IMAGE = "jib.alwaysCacheBaseImage"; + @VisibleForTesting public static final String SEND_CREDENTIALS_OVER_HTTP = "sendCredentialsOverHttp"; @@ -91,6 +93,16 @@ public static boolean isUserAgentEnabled() { return Strings.isNullOrEmpty(System.getProperty(DISABLE_USER_AGENT)); } + /** + * Gets whether to always cache base image layers. Determined from the {@code + * jib.alwaysCacheBaseImage} system property. + * + * @return true if the property is set to {@code always}; false otherwise + */ + public static boolean alwaysCacheBaseImage() { + return Boolean.getBoolean(ALWAYS_CACHE_BASE_IMAGE); + } + /** * Checks the {@code jib.httpTimeout} system property for invalid (non-integer or negative) * values. diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java index 38a45bbf07..cc7d3c9bf9 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildAndCacheApplicationLayerStepTest.java @@ -135,7 +135,7 @@ private List buildFakeLayersToCache() for (BuildAndCacheApplicationLayerStep buildAndCacheApplicationLayerStep : buildAndCacheApplicationLayerSteps) { - applicationLayers.add(buildAndCacheApplicationLayerStep.call().getCachedLayer()); + applicationLayers.add(buildAndCacheApplicationLayerStep.call()); } return applicationLayers; diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java index 5fe9044450..c0d7040c5e 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/BuildImageStepTest.java @@ -55,8 +55,8 @@ public class BuildImageStepTest { @Mock private CachedLayer mockCachedLayer; private Image baseImage; - private List baseImageLayers; - private List applicationLayers; + private List baseImageLayers; + private List applicationLayers; private DescriptorDigest testDescriptorDigest; private HistoryEntry nonEmptyLayerHistory; @@ -122,15 +122,15 @@ public void setUp() throws DigestException { .build(); baseImageLayers = Arrays.asList( - new CachedLayerAndName(mockCachedLayer, null), - new CachedLayerAndName(mockCachedLayer, null), - new CachedLayerAndName(mockCachedLayer, null)); + new PreparedLayer.Builder(mockCachedLayer).build(), + new PreparedLayer.Builder(mockCachedLayer).build(), + new PreparedLayer.Builder(mockCachedLayer).build()); applicationLayers = Arrays.asList( - new CachedLayerAndName(mockCachedLayer, "dependencies"), - new CachedLayerAndName(mockCachedLayer, "resources"), - new CachedLayerAndName(mockCachedLayer, "classes"), - new CachedLayerAndName(mockCachedLayer, "extra files")); + new PreparedLayer.Builder(mockCachedLayer).setName("dependencies").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("resources").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("classes").build(), + new PreparedLayer.Builder(mockCachedLayer).setName("extra files").build()); } @Test diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java new file mode 100644 index 0000000000..ab7025d47d --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/ObtainBaseImageLayerStepTest.java @@ -0,0 +1,176 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.DescriptorDigest; +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.blob.Blob; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.builder.steps.PullBaseImageStep.ImageAndAuthorization; +import com.google.cloud.tools.jib.cache.CacheCorruptedException; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.image.Image; +import com.google.cloud.tools.jib.image.Layer; +import com.google.cloud.tools.jib.image.ReferenceLayer; +import com.google.cloud.tools.jib.registry.RegistryClient; +import com.google.common.collect.ImmutableList; +import java.io.IOException; +import java.security.DigestException; +import java.util.Optional; +import java.util.function.Consumer; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.AdditionalAnswers; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; +import org.mockito.stubbing.Answer3; + +/** Tests for {@link ObtainBaseImageLayerStep}. */ +@RunWith(MockitoJUnitRunner.class) +public class ObtainBaseImageLayerStepTest { + + private ImageAndAuthorization baseImageAndAuth; + + private DescriptorDigest existingLayerDigest; + private DescriptorDigest freshLayerDigest; + + @Mock private Image image; + @Mock private RegistryClient registryClient; + + @Mock(answer = Answers.RETURNS_MOCKS) + private BuildConfiguration buildConfiguration; + + @Mock(answer = Answers.RETURNS_MOCKS) + private ProgressEventDispatcher.Factory progressDispatcherFactory; + + @Before + public void setUp() throws IOException, RegistryException, DigestException { + baseImageAndAuth = new ImageAndAuthorization(image, null); + + existingLayerDigest = + DescriptorDigest.fromHash( + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"); + freshLayerDigest = + DescriptorDigest.fromHash( + "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); + + DescriptorDigest diffId = Mockito.mock(DescriptorDigest.class); + Layer existingLayer = new ReferenceLayer(new BlobDescriptor(existingLayerDigest), diffId); + Layer freshLayer = new ReferenceLayer(new BlobDescriptor(freshLayerDigest), diffId); + Mockito.when(image.getLayers()).thenReturn(ImmutableList.of(existingLayer, freshLayer)); + + Mockito.when(registryClient.checkBlob(existingLayerDigest)) + .thenReturn(Optional.of(Mockito.mock(BlobDescriptor.class))); + Mockito.when(registryClient.checkBlob(freshLayerDigest)).thenReturn(Optional.empty()); + + RegistryClient.Factory registryClientFactory = + Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); + Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); + + Mockito.lenient() + .when(buildConfiguration.newBaseImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + Mockito.when(buildConfiguration.newTargetImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + + // necessary to prevent error from classes dealing with progress report + Answer3, Consumer> progressSizeSetter = + (ignored1, progressSizeConsumer, ignored2) -> { + progressSizeConsumer.accept(Long.valueOf(12345)); + return null; + }; + Mockito.when(registryClient.pullBlob(Mockito.any(), Mockito.any(), Mockito.any())) + .thenAnswer(AdditionalAnswers.answer(progressSizeSetter)); + } + + @Test + public void testMakeListForSelectiveDownload() + throws IOException, CacheCorruptedException, RegistryException { + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForSelectiveDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth, null); + + Assert.assertEquals(2, pullers.size()); + PreparedLayer preparedExistingLayer = pullers.get(0).call(); + PreparedLayer preparedFreshLayer = pullers.get(1).call(); + + Assert.assertTrue(preparedExistingLayer.existsInTarget().get()); + Assert.assertFalse(preparedFreshLayer.existsInTarget().get()); + + // Should have queried all blobs. + Mockito.verify(registryClient).checkBlob(existingLayerDigest); + Mockito.verify(registryClient).checkBlob(freshLayerDigest); + + // Only the missing layer should be pulled. + Mockito.verify(registryClient, Mockito.never()) + .pullBlob(Mockito.eq(existingLayerDigest), Mockito.any(), Mockito.any()); + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(freshLayerDigest), Mockito.any(), Mockito.any()); + } + + @Test + public void testMakeListForForcedDownload() + throws IOException, CacheCorruptedException, RegistryException { + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForForcedDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth); + + Assert.assertEquals(2, pullers.size()); + PreparedLayer preparedExistingLayer = pullers.get(0).call(); + PreparedLayer preparedFreshLayer = pullers.get(1).call(); + + // Unknown if layers exist in target registry. + Assert.assertEquals(Optional.empty(), preparedExistingLayer.existsInTarget()); + Assert.assertEquals(Optional.empty(), preparedFreshLayer.existsInTarget()); + + // No blob checking should happen. + Mockito.verify(registryClient, Mockito.never()).checkBlob(existingLayerDigest); + Mockito.verify(registryClient, Mockito.never()).checkBlob(freshLayerDigest); + + // All layers should be pulled. + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(existingLayerDigest), Mockito.any(), Mockito.any()); + Mockito.verify(registryClient) + .pullBlob(Mockito.eq(freshLayerDigest), Mockito.any(), Mockito.any()); + } + + @Test + public void testLayerMissingInCacheInOfflineMode() + throws CacheCorruptedException, RegistryException { + Mockito.when(buildConfiguration.isOffline()).thenReturn(true); + + ImmutableList pullers = + ObtainBaseImageLayerStep.makeListForForcedDownload( + buildConfiguration, progressDispatcherFactory, baseImageAndAuth); + try { + pullers.get(1).call(); + Assert.fail(); + } catch (IOException ex) { + Assert.assertEquals( + "Cannot run Jib in offline mode; local Jib cache for base image is missing image layer " + + "sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb. Rerun " + + "Jib in online mode with \"-Djib.alwaysCacheBaseImage=true\" to re-download the " + + "base image layers.", + ex.getMessage()); + } + } +} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java new file mode 100644 index 0000000000..514cc4b674 --- /dev/null +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/builder/steps/PushBlobStepTest.java @@ -0,0 +1,97 @@ +/* + * Copyright 2019 Google LLC. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package com.google.cloud.tools.jib.builder.steps; + +import com.google.cloud.tools.jib.api.ImageReference; +import com.google.cloud.tools.jib.api.RegistryException; +import com.google.cloud.tools.jib.blob.BlobDescriptor; +import com.google.cloud.tools.jib.builder.ProgressEventDispatcher; +import com.google.cloud.tools.jib.configuration.BuildConfiguration; +import com.google.cloud.tools.jib.configuration.ImageConfiguration; +import com.google.cloud.tools.jib.registry.RegistryClient; +import java.io.IOException; +import java.util.Optional; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.Answers; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.junit.MockitoJUnitRunner; + +/** Tests for {@link PushBlobStep}. */ +@RunWith(MockitoJUnitRunner.class) +public class PushBlobStepTest { + + @Mock private BlobDescriptor blobDescriptor; + @Mock private RegistryClient registryClient; + + @Mock(answer = Answers.RETURNS_MOCKS) + private ProgressEventDispatcher.Factory progressDispatcherFactory; + + @Mock(answer = Answers.RETURNS_MOCKS) + private BuildConfiguration buildConfiguration; + + @Before + public void setUp() { + RegistryClient.Factory registryClientFactory = + Mockito.mock(RegistryClient.Factory.class, Answers.RETURNS_SELF); + Mockito.when(registryClientFactory.newRegistryClient()).thenReturn(registryClient); + + Mockito.when(buildConfiguration.newTargetImageRegistryClientFactory()) + .thenReturn(registryClientFactory); + Mockito.when(buildConfiguration.getTargetImageConfiguration()) + .thenReturn(ImageConfiguration.builder(ImageReference.scratch()).build()); + } + + @Test + public void testCall_doBlobCheckAndBlobExists() throws IOException, RegistryException { + Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.of(blobDescriptor)); + + call(false); + + Mockito.verify(registryClient).checkBlob(Mockito.any()); + Mockito.verify(registryClient, Mockito.never()) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testCall_doBlobCheckAndBlobDoesNotExist() throws IOException, RegistryException { + Mockito.when(registryClient.checkBlob(Mockito.any())).thenReturn(Optional.empty()); + + call(false); + + Mockito.verify(registryClient).checkBlob(Mockito.any()); + Mockito.verify(registryClient) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + @Test + public void testCall_forcePushWithNoBlobCheck() throws IOException, RegistryException { + call(true); + + Mockito.verify(registryClient, Mockito.never()).checkBlob(Mockito.any()); + Mockito.verify(registryClient) + .pushBlob(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.any()); + } + + private void call(boolean forcePush) throws IOException, RegistryException { + new PushBlobStep( + buildConfiguration, progressDispatcherFactory, null, blobDescriptor, null, forcePush) + .call(); + } +} diff --git a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java index 4fca87bef0..85996ac10b 100644 --- a/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java +++ b/jib-core/src/test/java/com/google/cloud/tools/jib/global/JibSystemPropertiesTest.java @@ -37,6 +37,8 @@ public void setUp() { @After public void tearDown() { System.clearProperty(JibSystemProperties.HTTP_TIMEOUT); + System.clearProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS); + System.clearProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE); System.clearProperty("http.proxyPort"); System.clearProperty("https.proxyPort"); if (httpProxyPortSaved != null) { @@ -45,7 +47,6 @@ public void tearDown() { if (httpsProxyPortSaved != null) { System.setProperty("https.proxyPort", httpsProxyPortSaved); } - System.clearProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS); } @Test @@ -182,4 +183,28 @@ public void testUseBlobMounts_other() { System.setProperty(JibSystemProperties.CROSS_REPOSITORY_BLOB_MOUNTS, "nonbool"); Assert.assertFalse(JibSystemProperties.useCrossRepositoryBlobMounts()); } + + @Test + public void testAlwaysCacheBaseImage_undefined() { + System.clearProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_true() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "true"); + Assert.assertTrue(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_false() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "false"); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + } + + @Test + public void testAlwaysCacheBaseImage_other() { + System.setProperty(JibSystemProperties.ALWAYS_CACHE_BASE_IMAGE, "nonbool"); + Assert.assertFalse(JibSystemProperties.alwaysCacheBaseImage()); + } } diff --git a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java index 38e9ffa85e..6ed601e0ab 100644 --- a/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java +++ b/jib-gradle-plugin/src/integration-test/java/com/google/cloud/tools/jib/gradle/SingleProjectIntegrationTest.java @@ -21,6 +21,8 @@ import com.google.cloud.tools.jib.registry.LocalRegistry; import com.google.common.base.Splitter; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.DigestException; import java.time.Instant; import org.gradle.testkit.runner.BuildResult; @@ -146,14 +148,15 @@ private static void assertExtraDirectoryDeprecationWarning(String pomXml) + "'jib.extraDirectories.permissions'")); } - private static String buildAndRunComplex( + private static void buildAndRunComplex( String imageReference, String username, String password, LocalRegistry targetRegistry) throws IOException, InterruptedException { + Path baseCache = simpleTestProject.getProjectRoot().resolve("build/jib-base-cache"); BuildResult buildResult = simpleTestProject.build( "clean", "jib", - "-Djib.useOnlyProjectCache=true", + "-Djib.baseImageCache=" + baseCache, "-Djib.console=plain", "-D_TARGET_IMAGE=" + imageReference, "-D_TARGET_USERNAME=" + username, @@ -168,7 +171,11 @@ private static String buildAndRunComplex( assertDockerInspect(imageReference); String history = new Command("docker", "history", imageReference).run(); Assert.assertThat(history, CoreMatchers.containsString("jib-gradle-plugin")); - return new Command("docker", "run", "--rm", imageReference).run(); + Assert.assertEquals( + "Hello, world. An argument.\n1970-01-01T00:00:01Z\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n" + + "1970-01-01T00:00:01Z\n1970-01-01T00:00:01Z\n" + + "-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", + new Command("docker", "run", "--rm", imageReference).run()); } @Before @@ -318,11 +325,7 @@ public void testDockerDaemon_simple_multipleExtraDirectoriesWithAlternativeConfi public void testBuild_complex() throws IOException, InterruptedException { String targetImage = "localhost:6000/compleximage:gradle" + System.nanoTime(); Instant beforeBuild = Instant.now(); - Assert.assertEquals( - "Hello, world. An argument.\n1970-01-01T00:00:01Z\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n" - + "1970-01-01T00:00:01Z\n1970-01-01T00:00:01Z\n" - + "-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", - buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2)); + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); assertSimpleCreationTimeIsAfter(beforeBuild, targetImage); assertWorkingDirectory("", targetImage); } @@ -331,11 +334,7 @@ public void testBuild_complex() throws IOException, InterruptedException { public void testBuild_complex_sameFromAndToRegistry() throws IOException, InterruptedException { String targetImage = "localhost:5000/compleximage:gradle" + System.nanoTime(); Instant beforeBuild = Instant.now(); - Assert.assertEquals( - "Hello, world. An argument.\n1970-01-01T00:00:01Z\nrwxr-xr-x\nrwxrwxrwx\nfoo\ncat\n" - + "1970-01-01T00:00:01Z\n1970-01-01T00:00:01Z\n" - + "-Xms512m\n-Xdebug\nenvvalue1\nenvvalue2\n", - buildAndRunComplex(targetImage, "testuser", "testpassword", localRegistry1)); + buildAndRunComplex(targetImage, "testuser", "testpassword", localRegistry1); assertSimpleCreationTimeIsAfter(beforeBuild, targetImage); assertWorkingDirectory("", targetImage); } @@ -363,6 +362,22 @@ public void testDockerDaemon_jarContainerization() simpleTestProject, targetImage, "build-jar-containerization.gradle")); } + @Test + public void testBuild_skipDownloadingBaseImageLayers() throws IOException, InterruptedException { + Path baseLayersCacheDirectory = + simpleTestProject.getProjectRoot().resolve("build/jib-base-cache/layers"); + String targetImage = "localhost:6000/simpleimage:gradle" + System.nanoTime(); + + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); + // Base image layer tarballs exist. + Assert.assertTrue(Files.exists(baseLayersCacheDirectory)); + Assert.assertTrue(baseLayersCacheDirectory.toFile().list().length >= 2); + + buildAndRunComplex(targetImage, "testuser2", "testpassword2", localRegistry2); + // no base layers downloaded after "gradle clean jib ..." + Assert.assertFalse(Files.exists(baseLayersCacheDirectory)); + } + @Test public void testDockerDaemon_filesModificationTimeCustom() throws DigestException, IOException, InterruptedException { diff --git a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java index e2bf745290..84851ac8a9 100644 --- a/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java +++ b/jib-maven-plugin/src/test/java/com/google/cloud/tools/jib/maven/BuildImageMojoIntegrationTest.java @@ -112,19 +112,22 @@ private static Verifier build( verifier.addCliOption("--file=" + pomXml); verifier.executeGoals(Arrays.asList("clean", "compile")); + if (!buildTwice) { + verifier.executeGoal("jib:build"); + return verifier; + } + // Builds twice, and checks if the second build took less time. + verifier.addCliOption("-Djib.alwaysCacheBaseImage=true"); verifier.executeGoal("jib:build"); float timeOne = getBuildTimeFromVerifierLog(verifier); - if (buildTwice) { - verifier.resetStreams(); - verifier.executeGoal("jib:build"); - float timeTwo = getBuildTimeFromVerifierLog(verifier); - - String failMessage = "First build time (%s) is not greater than second build time (%s)"; - Assert.assertTrue(String.format(failMessage, timeOne, timeTwo), timeOne > timeTwo); - } + verifier.resetStreams(); + verifier.executeGoal("jib:build"); + float timeTwo = getBuildTimeFromVerifierLog(verifier); + String failMessage = "First build time (%s) is not greater than second build time (%s)"; + Assert.assertTrue(String.format(failMessage, timeOne, timeTwo), timeOne > timeTwo); return verifier; }