From fdb2402b1ea90fc1cfc4631005563c08887fdfd3 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Mon, 26 Oct 2015 17:45:29 +0100 Subject: [PATCH 1/6] Add support for blob rewrite - Add rewrite method to StorageRpc and DefaultStorageRpc - Add rewriter method to Storage and StorageImpl - Add unit and integration tests --- .../google/gcloud/spi/DefaultStorageRpc.java | 28 +++ .../com/google/gcloud/spi/StorageRpc.java | 5 + .../google/gcloud/storage/BlobRewriter.java | 203 ++++++++++++++++++ .../com/google/gcloud/storage/Storage.java | 170 +++++++++++++++ .../google/gcloud/storage/StorageImpl.java | 11 + .../gcloud/storage/BlobRewriterTest.java | 149 +++++++++++++ .../google/gcloud/storage/ITStorageTest.java | 38 ++++ 7 files changed, 604 insertions(+) create mode 100644 gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java create mode 100644 gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 90f2c98ad4b8..f28e3dba8a65 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -55,6 +55,7 @@ import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; +import com.google.api.services.storage.model.RewriteResponse; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; @@ -521,4 +522,31 @@ public String open(StorageObject object, Map options) throw translate(ex); } } + + @Override + public RewriteResponse rewrite(StorageObject source, Map sourceOptions, + StorageObject target, Map targetOptions, String token, Long maxByteRewrittenPerCall) + throws StorageException { + try { + return storage + .objects() + .rewrite(source.getBucket(), source.getName(), target.getBucket(), target.getName(), + target) + .setRewriteToken(token) + .setMaxBytesRewrittenPerCall(maxByteRewrittenPerCall) + .setProjection(DEFAULT_PROJECTION) + .setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) + .setIfSourceMetagenerationNotMatch( + IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) + .setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) + .setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) + .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) + .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) + .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) + .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) + .execute(); + } catch (IOException ex) { + throw translate(ex); + } + } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index b7ac99bf909e..a463583a563c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -19,6 +19,7 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.api.services.storage.model.Bucket; +import com.google.api.services.storage.model.RewriteResponse; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -174,4 +175,8 @@ byte[] read(StorageObject from, Map options, long position, int bytes void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageException; + + RewriteResponse rewrite(StorageObject source, Map sourceOptions, + StorageObject target, Map targetOptions, String token, Long maxByteRewrittenPerCall) + throws StorageException; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java new file mode 100644 index 000000000000..e144fe013261 --- /dev/null +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java @@ -0,0 +1,203 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import static com.google.common.base.MoreObjects.firstNonNull; +import static com.google.gcloud.RetryHelper.runWithRetries; + +import com.google.api.services.storage.model.RewriteResponse; +import com.google.gcloud.RetryHelper; +import com.google.gcloud.spi.StorageRpc; + +import java.math.BigInteger; +import java.util.Map; +import java.util.concurrent.Callable; + +/** + * Google Storage blob rewriter. + */ +public final class BlobRewriter { + + private final StorageOptions serviceOptions; + private final BlobId source; + private final Map sourceOptions; + private final Map targetOptions; + private final Long maxBytesRewrittenPerCall; + private BigInteger blobSize; + private BlobInfo target; + private Boolean isDone; + private String rewriteToken; + private BigInteger totalBytesRewritten; + + private final StorageRpc storageRpc; + + private BlobRewriter(Builder builder) { + this.serviceOptions = builder.serviceOptions; + this.source = builder.source; + this.sourceOptions = builder.sourceOptions; + this.target = builder.target; + this.targetOptions = builder.targetOptions; + this.blobSize = builder.blobSize; + this.isDone = builder.isDone; + this.rewriteToken = builder.rewriteToken; + this.totalBytesRewritten = firstNonNull(builder.totalBytesRewritten, BigInteger.ZERO); + this.maxBytesRewrittenPerCall = builder.maxBytesRewrittenPerCall; + this.storageRpc = serviceOptions.storageRpc(); + } + + static class Builder { + + private final StorageOptions serviceOptions; + private final BlobId source; + private final Map sourceOptions; + private final BlobInfo target; + private final Map targetOptions; + private BigInteger blobSize; + private Boolean isDone; + private String rewriteToken; + private BigInteger totalBytesRewritten; + private Long maxBytesRewrittenPerCall; + + Builder(StorageOptions serviceOptions, BlobId source, Map sourceOptions, + BlobInfo target, Map targetOptions) { + this.serviceOptions = serviceOptions; + this.source = source; + this.sourceOptions = sourceOptions; + this.target = target; + this.targetOptions = targetOptions; + } + + Builder blobSize(BigInteger blobSize) { + this.blobSize = blobSize; + return this; + } + + Builder isDone(Boolean isDone) { + this.isDone = isDone; + return this; + } + + Builder rewriteToken(String rewriteToken) { + this.rewriteToken = rewriteToken; + return this; + } + + Builder totalBytesRewritten(BigInteger totalBytesRewritten) { + this.totalBytesRewritten = totalBytesRewritten; + return this; + } + + Builder maxBytesRewrittenPerCall(Long maxBytesRewrittenPerCall) { + this.maxBytesRewrittenPerCall = maxBytesRewrittenPerCall; + return this; + } + + BlobRewriter build() { + return new BlobRewriter(this); + } + } + + static Builder builder(StorageOptions options, BlobId source, + Map sourceOpt, + BlobInfo target, Map targetOpt) { + return new Builder(options, source, sourceOpt, target, targetOpt); + } + + /** + * Returns the id of the source blob. + */ + public BlobId source() { + return source; + } + + /** + * Returns the info for the target blob. When {@link #isDone} is {@code true} this method returns + * the updated information for the just written blob. + */ + public BlobInfo target() { + return target; + } + + /** + * Size of the blob being copied. Returns {@code null} until the first copy request returns. + */ + public BigInteger blobSize() { + return blobSize; + } + + /** + * Returns {@code true} of blob rewrite finished, {@code false} otherwise. + */ + public Boolean isDone() { + return isDone; + } + + /** + * Returns the token to be used to rewrite the next chunk of the blob. + */ + public String rewriteToken() { + return rewriteToken; + } + + /** + * Returns the number of bytes written. + */ + public BigInteger totalBytesRewritten() { + return totalBytesRewritten; + } + + /** + * Returns the maximum number of bytes to be copied with each {@link #copyChunk()} call. This + * parameter is ignored if source and target blob share the same location and storage class as + * rewrite is made with one single RPC. + */ + public Long maxBytesRewrittenPerCall() { + return maxBytesRewrittenPerCall; + } + + /** + * Rewrite the next chunk of the blob. An RPC is issued only if rewrite has not finished yet + * ({@link #isDone} returns {@code false}). + * + * @throws StorageException upon failure + */ + public void copyChunk() { + if (!isDone) { + try { + RewriteResponse response = runWithRetries(new Callable() { + @Override + public RewriteResponse call() { + return storageRpc.rewrite( + source.toPb(), + sourceOptions, + target.toPb(), + targetOptions, + rewriteToken, + maxBytesRewrittenPerCall); + } + }, serviceOptions.retryParams(), StorageImpl.EXCEPTION_HANDLER); + rewriteToken = response.getRewriteToken(); + isDone = response.getDone(); + blobSize = response.getObjectSize(); + totalBytesRewritten = response.getTotalBytesRewritten(); + target = response.getResource() != null ? BlobInfo.fromPb(response.getResource()) : target; + } catch (RetryHelper.RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } + } + } +} diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 7f96ba90fc1f..f5dfee6f2f70 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -596,6 +596,157 @@ public static Builder builder() { } } + class RewriteRequest implements Serializable { + + private static final long serialVersionUID = -4498650529476219937L; + + private final BlobId source; + private final List sourceOptions; + private final BlobInfo target; + private final List targetOptions; + private final Long maxBytesRewrittenPerCall; + + public static class Builder { + + private final Set sourceOptions = new LinkedHashSet<>(); + private final Set targetOptions = new LinkedHashSet<>(); + private BlobId source; + private BlobInfo target; + private Long maxBytesRewrittenPerCall; + + /** + * Sets the blob to rewrite given bucket and blob name. + * + * @return the builder. + */ + public Builder source(String bucket, String blob) { + this.source = BlobId.of(bucket, blob); + return this; + } + + /** + * Sets the blob to rewrite given a {@link BlobId}. + * + * @return the builder. + */ + public Builder source(BlobId source) { + this.source = source; + return this; + } + + /** + * Sets blob's source options. + * + * @return the builder. + */ + public Builder sourceOptions(BlobSourceOption... options) { + Collections.addAll(sourceOptions, options); + return this; + } + + /** + * Sets blob's source options. + * + * @return the builder. + */ + public Builder sourceOptions(Iterable options) { + Iterables.addAll(sourceOptions, options); + return this; + } + + /** + * Sets the rewrite target. + * + * @return the builder. + */ + public Builder target(BlobInfo target) { + this.target = target; + return this; + } + + /** + * Sets blob's target options. + * + * @return the builder. + */ + public Builder targetOptions(BlobTargetOption... options) { + Collections.addAll(targetOptions, options); + return this; + } + + /** + * Sets blob's target options. + * + * @return the builder. + */ + public Builder targetOptions(Iterable options) { + Iterables.addAll(targetOptions, options); + return this; + } + + /** + * Sets the maximum number of bytes to copy for each RPC call. This parameter is ignored if + * source and target blob share the same location and storage class as rewrite is made with + * one single RPC. + * + * @return the builder. + */ + public Builder maxBytesRewrittenPerCall(Long maxBytesRewrittenPerCall) { + this.maxBytesRewrittenPerCall = maxBytesRewrittenPerCall; + return this; + } + + /** + * Creates a {@code RewriteRequest}. + */ + public RewriteRequest build() { + checkNotNull(source); + checkNotNull(target); + return new RewriteRequest(this); + } + } + + private RewriteRequest(Builder builder) { + source = checkNotNull(builder.source); + sourceOptions = ImmutableList.copyOf(builder.sourceOptions); + target = checkNotNull(builder.target); + targetOptions = ImmutableList.copyOf(builder.targetOptions); + maxBytesRewrittenPerCall = builder.maxBytesRewrittenPerCall; + } + + public BlobId source() { + return source; + } + + public List sourceOptions() { + return sourceOptions; + } + + public BlobInfo target() { + return target; + } + + public List targetOptions() { + return targetOptions; + } + + public Long maxBytesRewrittenPerCall() { + return maxBytesRewrittenPerCall; + } + + public static RewriteRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { + return builder().source(sourceBucket, sourceBlob).target(target).build(); + } + + public static RewriteRequest of(BlobId sourceBlobId, BlobInfo target) { + return builder().source(sourceBlobId).target(target).build(); + } + + public static Builder builder() { + return new Builder(); + } + } + /** * Create a new bucket. * @@ -865,4 +1016,23 @@ public static Builder builder() { * @throws StorageException upon failure */ List delete(BlobId... blobIds); + + /** + * Returns a {@link BlobRewriter} object for the provided {@code rewriteRequest}. If source and + * destination objects share the same location and storage class the source blob is copied with a + * single call of {@link BlobRewriter#copyChunk()}, regardless of the {@link + * RewriteRequest#maxBytesRewrittenPerCall} parameter. If source and destination have different + * location or storage class multiple RPC calls might be needed depending on blob's size. + *

+ * Example usage of blob rewriter: + * 

    {@code BlobRewriter rewriter = service.rewriter(rewriteRequest);}
+   *    {@code while(!rewriter.isDone()) {
+   *       rewriter.copyChunk();
+   *   }}
+   *
+ * + * @throws StorageException upon failure + * @see Rewrite + */ + BlobRewriter rewriter(RewriteRequest rewriteRequest); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 21cd8b726753..bfa760288d6f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -663,6 +663,17 @@ public List delete(BlobId... blobIds) { return Collections.unmodifiableList(transformResultList(response.deletes(), Boolean.FALSE)); } + @Override + public BlobRewriter rewriter(final RewriteRequest req) { + final Map sourceOpts = optionMap(null, null, req.sourceOptions(), true); + final Map targetOpts = optionMap(req.target().generation(), + req.target().metageneration(), req.targetOptions()); + return BlobRewriter.builder(options(), req.source(), sourceOpts, req.target(), targetOpts) + .isDone(false) + .maxBytesRewrittenPerCall(req.maxBytesRewrittenPerCall()) + .build(); + } + private static List transformResultList( List> results, final T errorValue) { return Lists.transform(results, new Function, T>() { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java new file mode 100644 index 000000000000..0f5e009fd95e --- /dev/null +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java @@ -0,0 +1,149 @@ +/* + * Copyright 2015 Google Inc. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.gcloud.storage; + +import com.google.api.services.storage.model.RewriteResponse; +import static org.easymock.EasyMock.verify; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.assertEquals; + +import com.google.common.collect.ImmutableMap; +import com.google.gcloud.RetryParams; +import com.google.gcloud.spi.StorageRpc; + +import org.easymock.EasyMock; +import org.junit.Test; +import org.junit.Before; + +import java.io.IOException; +import java.math.BigInteger; +import java.util.Map; +import org.junit.After; + +public class BlobRewriterTest { + + private static final String SOURCE_BUCKET_NAME = "b"; + private static final String SOURCE_BLOB_NAME = "n"; + private static final String DESTINATION_BUCKET_NAME = "b1"; + private static final String DESTINATION_BLOB_NAME = "n1"; + private static final BlobId BLOB_ID = BlobId.of(SOURCE_BUCKET_NAME, SOURCE_BLOB_NAME); + private static final BlobInfo BLOB_INFO = + BlobInfo.builder(DESTINATION_BUCKET_NAME, DESTINATION_BLOB_NAME).build(); + private static final Map EMPTY_OPTIONS = ImmutableMap.of(); + + private StorageOptions optionsMock; + private StorageRpc storageRpcMock; + private BlobRewriter rewriter; + + @Before + public void setUp() throws IOException, InterruptedException { + optionsMock = EasyMock.createMock(StorageOptions.class); + storageRpcMock = EasyMock.createMock(StorageRpc.class); + } + + @After + public void tearDown() throws Exception { + verify(optionsMock, storageRpcMock); + } + + @Test + public void testRewrite() { + RewriteResponse response = new RewriteResponse() + .setDone(true) + .setResource(BLOB_INFO.toPb()) + .setObjectSize(BigInteger.valueOf(42L)) + .setTotalBytesRewritten(BigInteger.valueOf(42L)); + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.expect(storageRpcMock.rewrite( + BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, null)) + .andReturn(response); + EasyMock.replay(optionsMock, storageRpcMock); + rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) + .isDone(false) + .build(); + rewriter.copyChunk(); + assertTrue(rewriter.isDone()); + assertEquals(BLOB_INFO, rewriter.target()); + assertEquals(BLOB_ID, rewriter.source()); + assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); + assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); + } + + @Test + public void testRewriteCustumSize() { + RewriteResponse response = new RewriteResponse() + .setDone(true) + .setResource(BLOB_INFO.toPb()) + .setObjectSize(BigInteger.valueOf(42L)) + .setTotalBytesRewritten(BigInteger.valueOf(42L)); + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); + EasyMock.expect(storageRpcMock.rewrite( + BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, 1048576L)) + .andReturn(response); + EasyMock.replay(optionsMock, storageRpcMock); + rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) + .isDone(false) + .maxBytesRewrittenPerCall(1048576L) + .build(); + rewriter.copyChunk(); + assertTrue(rewriter.isDone()); + assertEquals(BLOB_INFO, rewriter.target()); + assertEquals(BLOB_ID, rewriter.source()); + assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); + assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); + } + + @Test + public void testRewriteMultipleRequests() { + RewriteResponse firstResponse = new RewriteResponse() + .setDone(false) + .setResource(BLOB_INFO.toPb()) + .setObjectSize(BigInteger.valueOf(42L)) + .setRewriteToken("token") + .setTotalBytesRewritten(BigInteger.valueOf(21L)); + RewriteResponse secondResponse = new RewriteResponse() + .setDone(true) + .setResource(BLOB_INFO.toPb()) + .setObjectSize(BigInteger.valueOf(42L)) + .setTotalBytesRewritten(BigInteger.valueOf(42L)); + EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); + EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()).times(2); + EasyMock.expect(storageRpcMock.rewrite( + BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, null)) + .andReturn(firstResponse); + EasyMock.expect(storageRpcMock.rewrite( + BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, "token", null)) + .andReturn(secondResponse); + EasyMock.replay(optionsMock, storageRpcMock); + rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) + .isDone(false) + .build(); + int loopCount = 0; + while (!rewriter.isDone()) { + rewriter.copyChunk(); + loopCount++; + } + assertTrue(rewriter.isDone()); + assertEquals(BLOB_INFO, rewriter.target()); + assertEquals(BLOB_ID, rewriter.source()); + assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); + assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); + assertEquals(2, loopCount); + } +} diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 7b29086ecbe5..b7e7d4b415d0 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -660,4 +660,42 @@ public void testUpdateBlobsFail() { assertNull(updatedBlobs.get(1)); assertTrue(storage.delete(BUCKET, sourceBlobName1)); } + + @Test + public void testRewriteBlob() { + String sourceBlobName = "test-rewrite-blob-source"; + BlobId source = BlobId.of(bucket, sourceBlobName); + assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); + String targetBlobName = "test-rewrite-blob-target"; + BlobInfo target = BlobInfo.builder(bucket, targetBlobName).contentType(CONTENT_TYPE).build(); + Storage.RewriteRequest req = Storage.RewriteRequest.of(source, target); + BlobRewriter rewriter = storage.rewriter(req); + rewriter.copyChunk(); + assertTrue(rewriter.isDone()); + assertEquals(rewriter.target(), storage.get(bucket, targetBlobName)); + assertTrue(storage.delete(bucket, sourceBlobName)); + assertTrue(storage.delete(bucket, targetBlobName)); + } + + @Test + public void testRewriteBlobFail() { + String sourceBlobName = "test-rewrite-blob-source-fail"; + BlobId source = BlobId.of(bucket, sourceBlobName); + assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); + String targetBlobName = "test-rewrite-blob-target-fail"; + BlobInfo target = BlobInfo.builder(bucket, targetBlobName).contentType(CONTENT_TYPE).build(); + Storage.RewriteRequest req = Storage.RewriteRequest.builder() + .source(source) + .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) + .target(target) + .build(); + BlobRewriter rewriter = storage.rewriter(req); + try { + rewriter.copyChunk(); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } + assertTrue(storage.delete(bucket, sourceBlobName)); + } } From 15372f1920ef32ba0f161ff293f35e0e37664637 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Tue, 27 Oct 2015 16:41:49 +0100 Subject: [PATCH 2/6] Refactor StorageRpc and Storage rewrite - Split StorageRpc.rewrite into openRewrite and continueRewrite - Add StorageRpc.RewriteResponse and RewriteRequest classe - Move first rewrite request from BlobRewriter to StorageImpl - Change maxBytesRewrittenPerCall to megabytesRewrittenPerCall - Change BlobWriter.blobSize for BigInteger to Long - BlobRewriter extends Restorable - Refactor tests and add StorageImpl unit tests --- .../google/gcloud/spi/DefaultStorageRpc.java | 49 ++- .../com/google/gcloud/spi/StorageRpc.java | 91 ++++- .../google/gcloud/storage/BlobRewriter.java | 318 ++++++++++-------- .../com/google/gcloud/storage/Storage.java | 37 +- .../google/gcloud/storage/StorageImpl.java | 28 +- .../gcloud/storage/BlobRewriterTest.java | 140 ++++---- .../google/gcloud/storage/ITStorageTest.java | 19 +- .../gcloud/storage/StorageImplTest.java | 65 ++++ 8 files changed, 489 insertions(+), 258 deletions(-) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index f28e3dba8a65..1098ca4c538d 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -55,7 +55,6 @@ import com.google.api.services.storage.model.ComposeRequest; import com.google.api.services.storage.model.ComposeRequest.SourceObjects.ObjectPreconditions; import com.google.api.services.storage.model.Objects; -import com.google.api.services.storage.model.RewriteResponse; import com.google.api.services.storage.model.StorageObject; import com.google.common.base.MoreObjects; import com.google.common.collect.ImmutableSet; @@ -79,6 +78,7 @@ public class DefaultStorageRpc implements StorageRpc { // see: https://cloud.google.com/storage/docs/concepts-techniques#practices private static final Set RETRYABLE_CODES = ImmutableSet.of(504, 503, 502, 500, 429, 408); + private static final long MEGABYTE = 1024L * 1024L; public DefaultStorageRpc(StorageOptions options) { HttpTransport transport = options.httpTransportFactory().create(); @@ -524,27 +524,42 @@ public String open(StorageObject object, Map options) } @Override - public RewriteResponse rewrite(StorageObject source, Map sourceOptions, - StorageObject target, Map targetOptions, String token, Long maxByteRewrittenPerCall) - throws StorageException { + public RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException { + return rewrite(rewriteRequest, null); + } + + @Override + public RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException { + return rewrite(previousResponse.rewriteRequest, previousResponse.rewriteToken); + } + + private RewriteResponse rewrite(RewriteRequest req, String token) throws StorageException { try { - return storage - .objects() - .rewrite(source.getBucket(), source.getName(), target.getBucket(), target.getName(), - target) + Long maxBytesRewrittenPerCall = req.megabytesRewrittenPerCall != null + ? req.megabytesRewrittenPerCall * MEGABYTE : null; + com.google.api.services.storage.model.RewriteResponse rewriteReponse = storage.objects() + .rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(), + req.target.getName(), req.target) .setRewriteToken(token) - .setMaxBytesRewrittenPerCall(maxByteRewrittenPerCall) + .setMaxBytesRewrittenPerCall(maxBytesRewrittenPerCall) .setProjection(DEFAULT_PROJECTION) - .setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) + .setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(req.sourceOptions)) .setIfSourceMetagenerationNotMatch( - IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) - .setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) - .setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) - .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) - .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) - .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) - .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) + IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(req.sourceOptions)) + .setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(req.sourceOptions)) + .setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(req.sourceOptions)) + .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(req.targetOptions)) + .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(req.targetOptions)) + .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(req.targetOptions)) + .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(req.targetOptions)) .execute(); + return new RewriteResponse( + req, + rewriteReponse.getResource(), + rewriteReponse.getObjectSize().longValue(), + rewriteReponse.getDone(), + rewriteReponse.getRewriteToken(), + rewriteReponse.getTotalBytesRewritten().longValue()); } catch (IOException ex) { throw translate(ex); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index a463583a563c..1a4afa0524e0 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -19,7 +19,6 @@ import static com.google.common.base.MoreObjects.firstNonNull; import com.google.api.services.storage.model.Bucket; -import com.google.api.services.storage.model.RewriteResponse; import com.google.api.services.storage.model.StorageObject; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -28,6 +27,7 @@ import java.io.InputStream; import java.util.List; import java.util.Map; +import java.util.Objects; public interface StorageRpc { @@ -133,6 +133,89 @@ public BatchResponse(Map> delete } } + class RewriteRequest { + + public final StorageObject source; + public final Map sourceOptions; + public final StorageObject target; + public final Map targetOptions; + public final Long megabytesRewrittenPerCall; + + public RewriteRequest(StorageObject source, Map sourceOptions, + StorageObject target, Map targetOptions, + Long megabytesRewrittenPerCall) { + this.source = source; + this.sourceOptions = sourceOptions; + this.target = target; + this.targetOptions = targetOptions; + this.megabytesRewrittenPerCall = megabytesRewrittenPerCall; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof RewriteRequest)) { + return false; + } + final RewriteRequest other = (RewriteRequest) obj; + return Objects.equals(this.source, other.source) + && Objects.equals(this.sourceOptions, other.sourceOptions) + && Objects.equals(this.target, other.target) + && Objects.equals(this.targetOptions, other.targetOptions) + && Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall); + } + + @Override + public int hashCode() { + return Objects.hash(source, sourceOptions, target, targetOptions, megabytesRewrittenPerCall); + } + } + + class RewriteResponse { + + public final RewriteRequest rewriteRequest; + public final StorageObject result; + public final Long blobSize; + public final Boolean isDone; + public final String rewriteToken; + public final Long totalBytesRewritten; + + public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, Long blobSize, + Boolean isDone, String rewriteToken, Long totalBytesRewritten) { + this.rewriteRequest = rewriteRequest; + this.result = result; + this.blobSize = blobSize; + this.isDone = isDone; + this.rewriteToken = rewriteToken; + this.totalBytesRewritten = totalBytesRewritten; + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof RewriteResponse)) { + return false; + } + final RewriteResponse other = (RewriteResponse) obj; + return Objects.equals(this.rewriteRequest, other.rewriteRequest) + && Objects.equals(this.result, other.result) + && Objects.equals(this.rewriteToken, other.rewriteToken) + && Objects.equals(this.blobSize, other.blobSize) + && Objects.equals(this.isDone, other.isDone) + && Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten); + } + + @Override + public int hashCode() { + return Objects.hash(rewriteRequest, result, blobSize, isDone, rewriteToken, + totalBytesRewritten); + } + } + Bucket create(Bucket bucket, Map options) throws StorageException; StorageObject create(StorageObject object, InputStream content, Map options) @@ -176,7 +259,7 @@ byte[] read(StorageObject from, Map options, long position, int bytes void write(String uploadId, byte[] toWrite, int toWriteOffset, StorageObject dest, long destOffset, int length, boolean last) throws StorageException; - RewriteResponse rewrite(StorageObject source, Map sourceOptions, - StorageObject target, Map targetOptions, String token, Long maxByteRewrittenPerCall) - throws StorageException; + RewriteResponse openRewrite(RewriteRequest rewriteRequest) throws StorageException; + + RewriteResponse continueRewrite(RewriteResponse previousResponse) throws StorageException; } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java index e144fe013261..75b0516f0acf 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java @@ -16,157 +16,63 @@ package com.google.gcloud.storage; -import static com.google.common.base.MoreObjects.firstNonNull; import static com.google.gcloud.RetryHelper.runWithRetries; -import com.google.api.services.storage.model.RewriteResponse; +import com.google.common.base.MoreObjects; +import com.google.gcloud.Restorable; +import com.google.gcloud.RestorableState; import com.google.gcloud.RetryHelper; import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.spi.StorageRpc.RewriteRequest; +import com.google.gcloud.spi.StorageRpc.RewriteResponse; -import java.math.BigInteger; +import java.io.Serializable; import java.util.Map; +import java.util.Objects; import java.util.concurrent.Callable; /** * Google Storage blob rewriter. */ -public final class BlobRewriter { +public final class BlobRewriter implements Restorable { private final StorageOptions serviceOptions; - private final BlobId source; - private final Map sourceOptions; - private final Map targetOptions; - private final Long maxBytesRewrittenPerCall; - private BigInteger blobSize; - private BlobInfo target; - private Boolean isDone; - private String rewriteToken; - private BigInteger totalBytesRewritten; - private final StorageRpc storageRpc; + private RewriteResponse rewriteResponse; - private BlobRewriter(Builder builder) { - this.serviceOptions = builder.serviceOptions; - this.source = builder.source; - this.sourceOptions = builder.sourceOptions; - this.target = builder.target; - this.targetOptions = builder.targetOptions; - this.blobSize = builder.blobSize; - this.isDone = builder.isDone; - this.rewriteToken = builder.rewriteToken; - this.totalBytesRewritten = firstNonNull(builder.totalBytesRewritten, BigInteger.ZERO); - this.maxBytesRewrittenPerCall = builder.maxBytesRewrittenPerCall; - this.storageRpc = serviceOptions.storageRpc(); - } - - static class Builder { - - private final StorageOptions serviceOptions; - private final BlobId source; - private final Map sourceOptions; - private final BlobInfo target; - private final Map targetOptions; - private BigInteger blobSize; - private Boolean isDone; - private String rewriteToken; - private BigInteger totalBytesRewritten; - private Long maxBytesRewrittenPerCall; - - Builder(StorageOptions serviceOptions, BlobId source, Map sourceOptions, - BlobInfo target, Map targetOptions) { - this.serviceOptions = serviceOptions; - this.source = source; - this.sourceOptions = sourceOptions; - this.target = target; - this.targetOptions = targetOptions; - } - - Builder blobSize(BigInteger blobSize) { - this.blobSize = blobSize; - return this; - } - - Builder isDone(Boolean isDone) { - this.isDone = isDone; - return this; - } - - Builder rewriteToken(String rewriteToken) { - this.rewriteToken = rewriteToken; - return this; - } - - Builder totalBytesRewritten(BigInteger totalBytesRewritten) { - this.totalBytesRewritten = totalBytesRewritten; - return this; - } - - Builder maxBytesRewrittenPerCall(Long maxBytesRewrittenPerCall) { - this.maxBytesRewrittenPerCall = maxBytesRewrittenPerCall; - return this; - } - - BlobRewriter build() { - return new BlobRewriter(this); - } - } - - static Builder builder(StorageOptions options, BlobId source, - Map sourceOpt, - BlobInfo target, Map targetOpt) { - return new Builder(options, source, sourceOpt, target, targetOpt); - } - - /** - * Returns the id of the source blob. - */ - public BlobId source() { - return source; + BlobRewriter(StorageOptions serviceOptions, RewriteResponse rewriteResponse) { + this.serviceOptions = serviceOptions; + this.rewriteResponse = rewriteResponse; + this.storageRpc = serviceOptions.rpc(); } /** - * Returns the info for the target blob. When {@link #isDone} is {@code true} this method returns - * the updated information for the just written blob. + * Returns the updated information for the just written blob when {@link #isDone} is {@code true}. + * Returns {@code null} otherwise. */ - public BlobInfo target() { - return target; + public BlobInfo result() { + return rewriteResponse.result != null ? BlobInfo.fromPb(rewriteResponse.result) : null; } /** - * Size of the blob being copied. Returns {@code null} until the first copy request returns. + * Size of the blob being copied. */ - public BigInteger blobSize() { - return blobSize; + public Long blobSize() { + return rewriteResponse.blobSize; } /** * Returns {@code true} of blob rewrite finished, {@code false} otherwise. */ public Boolean isDone() { - return isDone; - } - - /** - * Returns the token to be used to rewrite the next chunk of the blob. - */ - public String rewriteToken() { - return rewriteToken; + return rewriteResponse.isDone; } /** * Returns the number of bytes written. */ - public BigInteger totalBytesRewritten() { - return totalBytesRewritten; - } - - /** - * Returns the maximum number of bytes to be copied with each {@link #copyChunk()} call. This - * parameter is ignored if source and target blob share the same location and storage class as - * rewrite is made with one single RPC. - */ - public Long maxBytesRewrittenPerCall() { - return maxBytesRewrittenPerCall; + public Long totalBytesRewritten() { + return rewriteResponse.totalBytesRewritten; } /** @@ -176,28 +82,178 @@ public Long maxBytesRewrittenPerCall() { * @throws StorageException upon failure */ public void copyChunk() { - if (!isDone) { + if (!isDone()) { try { - RewriteResponse response = runWithRetries(new Callable() { + this.rewriteResponse = runWithRetries(new Callable() { @Override public RewriteResponse call() { - return storageRpc.rewrite( - source.toPb(), - sourceOptions, - target.toPb(), - targetOptions, - rewriteToken, - maxBytesRewrittenPerCall); + return storageRpc.continueRewrite(rewriteResponse); } }, serviceOptions.retryParams(), StorageImpl.EXCEPTION_HANDLER); - rewriteToken = response.getRewriteToken(); - isDone = response.getDone(); - blobSize = response.getObjectSize(); - totalBytesRewritten = response.getTotalBytesRewritten(); - target = response.getResource() != null ? BlobInfo.fromPb(response.getResource()) : target; } catch (RetryHelper.RetryHelperException e) { throw StorageException.translateAndThrow(e); } } } + + @Override + public RestorableState capture() { + return StateImpl.builder( + serviceOptions, + BlobId.fromPb(rewriteResponse.rewriteRequest.source), + rewriteResponse.rewriteRequest.sourceOptions, + BlobInfo.fromPb(rewriteResponse.rewriteRequest.target), + rewriteResponse.rewriteRequest.targetOptions) + .blobSize(blobSize()) + .isDone(isDone()) + .megabytesRewrittenPerCall(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall) + .rewriteToken(rewriteResponse.rewriteToken) + .totalBytesRewritten(totalBytesRewritten()) + .build(); + } + + static class StateImpl implements RestorableState, Serializable { + + private static final long serialVersionUID = 8279287678903181701L; + + private final StorageOptions serviceOptions; + private final BlobId source; + private final Map sourceOptions; + private final BlobInfo target; + private final Map targetOptions; + private final BlobInfo result; + private final Long blobSize; + private final Boolean isDone; + private final String rewriteToken; + private final Long totalBytesRewritten; + private final Long megabytesRewrittenPerCall; + + StateImpl(Builder builder) { + this.serviceOptions = builder.serviceOptions; + this.source = builder.source; + this.sourceOptions = builder.sourceOptions; + this.target = builder.target; + this.targetOptions = builder.targetOptions; + this.result = builder.result; + this.blobSize = builder.blobSize; + this.isDone = builder.isDone; + this.rewriteToken = builder.rewriteToken; + this.totalBytesRewritten = builder.totalBytesRewritten; + this.megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall; + } + + static class Builder { + + private final StorageOptions serviceOptions; + private final BlobId source; + private final Map sourceOptions; + private final BlobInfo target; + private final Map targetOptions; + private BlobInfo result; + private Long blobSize; + private Boolean isDone; + private String rewriteToken; + private Long totalBytesRewritten; + private Long megabytesRewrittenPerCall; + + private Builder(StorageOptions options, BlobId source, + Map sourceOptions, + BlobInfo target, Map targetOptions) { + this.serviceOptions = options; + this.source = source; + this.sourceOptions = sourceOptions; + this.target = target; + this.targetOptions = targetOptions; + } + + Builder result(BlobInfo result) { + this.result = result; + return this; + } + + Builder blobSize(Long blobSize) { + this.blobSize = blobSize; + return this; + } + + Builder isDone(Boolean isDone) { + this.isDone = isDone; + return this; + } + + Builder rewriteToken(String rewriteToken) { + this.rewriteToken = rewriteToken; + return this; + } + + Builder totalBytesRewritten(Long totalBytesRewritten) { + this.totalBytesRewritten = totalBytesRewritten; + return this; + } + + Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { + this.megabytesRewrittenPerCall = megabytesRewrittenPerCall; + return this; + } + + RestorableState build() { + return new StateImpl(this); + } + } + + static Builder builder(StorageOptions options, BlobId source, + Map sourceOptions, BlobInfo target, + Map targetOptions) { + return new Builder(options, source, sourceOptions, target, targetOptions); + } + + @Override + public BlobRewriter restore() { + RewriteRequest rewriteRequest = new RewriteRequest( + source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesRewrittenPerCall); + RewriteResponse rewriteResponse = new RewriteResponse(rewriteRequest, + result != null ? result.toPb() : null, blobSize, isDone, rewriteToken, + totalBytesRewritten); + return new BlobRewriter(serviceOptions, rewriteResponse); + } + + @Override + public int hashCode() { + return Objects.hash(serviceOptions, source, sourceOptions, target, targetOptions, result, + blobSize, isDone, megabytesRewrittenPerCall, rewriteToken, totalBytesRewritten); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof StateImpl)) { + return false; + } + final StateImpl other = (StateImpl) obj; + return Objects.equals(this.serviceOptions, other.serviceOptions) + && Objects.equals(this.source, other.source) + && Objects.equals(this.sourceOptions, other.sourceOptions) + && Objects.equals(this.target, other.target) + && Objects.equals(this.targetOptions, other.targetOptions) + && Objects.equals(this.result, other.result) + && Objects.equals(this.rewriteToken, other.rewriteToken) + && Objects.equals(this.blobSize, other.blobSize) + && Objects.equals(this.isDone, other.isDone) + && Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall) + && Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("source", source) + .add("target", target) + .add("isDone", isDone) + .add("totalBytesRewritten", totalBytesRewritten) + .add("blobSize", blobSize) + .toString(); + } + } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index f5dfee6f2f70..88f2a80f8b9b 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -604,7 +604,7 @@ class RewriteRequest implements Serializable { private final List sourceOptions; private final BlobInfo target; private final List targetOptions; - private final Long maxBytesRewrittenPerCall; + private final Long megabytesRewrittenPerCall; public static class Builder { @@ -612,7 +612,7 @@ public static class Builder { private final Set targetOptions = new LinkedHashSet<>(); private BlobId source; private BlobInfo target; - private Long maxBytesRewrittenPerCall; + private Long megabytesRewrittenPerCall; /** * Sets the blob to rewrite given bucket and blob name. @@ -685,14 +685,14 @@ public Builder targetOptions(Iterable options) { } /** - * Sets the maximum number of bytes to copy for each RPC call. This parameter is ignored if - * source and target blob share the same location and storage class as rewrite is made with + * Sets the maximum number of megabytes to copy for each RPC call. This parameter is ignored + * if source and target blob share the same location and storage class as rewrite is made with * one single RPC. * * @return the builder. */ - public Builder maxBytesRewrittenPerCall(Long maxBytesRewrittenPerCall) { - this.maxBytesRewrittenPerCall = maxBytesRewrittenPerCall; + public Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { + this.megabytesRewrittenPerCall = megabytesRewrittenPerCall; return this; } @@ -711,27 +711,44 @@ private RewriteRequest(Builder builder) { sourceOptions = ImmutableList.copyOf(builder.sourceOptions); target = checkNotNull(builder.target); targetOptions = ImmutableList.copyOf(builder.targetOptions); - maxBytesRewrittenPerCall = builder.maxBytesRewrittenPerCall; + megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall; } + /** + * Returns the blob to rewrite, as a {@link BlobId}. + */ public BlobId source() { return source; } + /** + * Returns blob's source options. + */ public List sourceOptions() { return sourceOptions; } + /** + * Returns the rewrite target. + */ public BlobInfo target() { return target; } + /** + * Returns blob's target options. + */ public List targetOptions() { return targetOptions; } - public Long maxBytesRewrittenPerCall() { - return maxBytesRewrittenPerCall; + /** + * Returns the maximum number of megabytes to copy for each RPC call. This parameter is ignored + * if source and target blob share the same location and storage class as rewrite is made with + * one single RPC. + */ + public Long megabytesRewrittenPerCall() { + return megabytesRewrittenPerCall; } public static RewriteRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { @@ -1018,7 +1035,7 @@ public static Builder builder() { List delete(BlobId... blobIds); /** - * Returns a {@link BlobRewriter} object for the provided {@code rewriteRequest}. If source and + * Returns a {@link BlobRewriter} object for the provided {@code RewriteRequest}. If source and * destination objects share the same location and storage class the source blob is copied with a * single call of {@link BlobRewriter#copyChunk()}, regardless of the {@link * RewriteRequest#maxBytesRewrittenPerCall} parameter. If source and destination have different diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index bfa760288d6f..7bc39481b8a0 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -49,6 +49,7 @@ import com.google.gcloud.ExceptionHandler.Interceptor; import com.google.gcloud.RetryHelper.RetryHelperException; import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.spi.StorageRpc.RewriteResponse; import com.google.gcloud.spi.StorageRpc.Tuple; import java.io.ByteArrayInputStream; @@ -664,14 +665,25 @@ public List delete(BlobId... blobIds) { } @Override - public BlobRewriter rewriter(final RewriteRequest req) { - final Map sourceOpts = optionMap(null, null, req.sourceOptions(), true); - final Map targetOpts = optionMap(req.target().generation(), - req.target().metageneration(), req.targetOptions()); - return BlobRewriter.builder(options(), req.source(), sourceOpts, req.target(), targetOpts) - .isDone(false) - .maxBytesRewrittenPerCall(req.maxBytesRewrittenPerCall()) - .build(); + public BlobRewriter rewriter(final RewriteRequest rewriteRequest) { + final StorageObject source = rewriteRequest.source().toPb(); + final Map sourceOptions = + optionMap(null, null, rewriteRequest.sourceOptions(), true); + final StorageObject target = rewriteRequest.target().toPb(); + final Map targetOptions = optionMap(rewriteRequest.target().generation(), + rewriteRequest.target().metageneration(), rewriteRequest.targetOptions()); + try { + RewriteResponse rewriteResponse = runWithRetries(new Callable() { + @Override + public RewriteResponse call() { + return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, target, + targetOptions, rewriteRequest.megabytesRewrittenPerCall())); + } + }, options().retryParams(), EXCEPTION_HANDLER); + return new BlobRewriter(options(), rewriteResponse); + } catch (RetryHelperException e) { + throw StorageException.translateAndThrow(e); + } } private static List transformResultList( diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java index 0f5e009fd95e..83cda633fd6a 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java @@ -16,23 +16,28 @@ package com.google.gcloud.storage; -import com.google.api.services.storage.model.RewriteResponse; +import static org.easymock.EasyMock.anyObject; +import static org.easymock.EasyMock.createMock; +import static org.easymock.EasyMock.expect; +import static org.easymock.EasyMock.replay; import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertEquals; import com.google.common.collect.ImmutableMap; -import com.google.gcloud.RetryParams; +import com.google.gcloud.RestorableState; import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.spi.StorageRpc.RewriteRequest; +import com.google.gcloud.spi.StorageRpc.RewriteResponse; +import com.google.gcloud.spi.StorageRpcFactory; import org.easymock.EasyMock; +import org.junit.After; import org.junit.Test; import org.junit.Before; import java.io.IOException; -import java.math.BigInteger; import java.util.Map; -import org.junit.After; public class BlobRewriterTest { @@ -43,107 +48,86 @@ public class BlobRewriterTest { private static final BlobId BLOB_ID = BlobId.of(SOURCE_BUCKET_NAME, SOURCE_BLOB_NAME); private static final BlobInfo BLOB_INFO = BlobInfo.builder(DESTINATION_BUCKET_NAME, DESTINATION_BLOB_NAME).build(); + private static final BlobInfo RESULT = + BlobInfo.builder(DESTINATION_BUCKET_NAME, DESTINATION_BLOB_NAME).contentType("type").build(); private static final Map EMPTY_OPTIONS = ImmutableMap.of(); + private static final RewriteRequest REQUEST = new StorageRpc.RewriteRequest(BLOB_ID.toPb(), + EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null); + private static final RewriteResponse RESPONSE = new StorageRpc.RewriteResponse(REQUEST, + null, 42L, false, "token", 21L); + private static final RewriteResponse RESPONSE_DONE = new StorageRpc.RewriteResponse(REQUEST, + RESULT.toPb(), 42L, true, "token", 42L); - private StorageOptions optionsMock; + private StorageOptions options; + private StorageRpcFactory rpcFactoryMock; private StorageRpc storageRpcMock; private BlobRewriter rewriter; @Before public void setUp() throws IOException, InterruptedException { - optionsMock = EasyMock.createMock(StorageOptions.class); - storageRpcMock = EasyMock.createMock(StorageRpc.class); + rpcFactoryMock = createMock(StorageRpcFactory.class); + storageRpcMock = createMock(StorageRpc.class); + expect(rpcFactoryMock.create(anyObject(StorageOptions.class))) + .andReturn(storageRpcMock); + replay(rpcFactoryMock); + options = StorageOptions.builder() + .projectId("projectid") + .serviceRpcFactory(rpcFactoryMock) + .build(); } @After public void tearDown() throws Exception { - verify(optionsMock, storageRpcMock); + verify(rpcFactoryMock, storageRpcMock); } @Test public void testRewrite() { - RewriteResponse response = new RewriteResponse() - .setDone(true) - .setResource(BLOB_INFO.toPb()) - .setObjectSize(BigInteger.valueOf(42L)) - .setTotalBytesRewritten(BigInteger.valueOf(42L)); - EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); - EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); - EasyMock.expect(storageRpcMock.rewrite( - BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, null)) - .andReturn(response); - EasyMock.replay(optionsMock, storageRpcMock); - rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) - .isDone(false) - .build(); + EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); + EasyMock.replay(storageRpcMock); + rewriter = new BlobRewriter(options, RESPONSE); rewriter.copyChunk(); assertTrue(rewriter.isDone()); - assertEquals(BLOB_INFO, rewriter.target()); - assertEquals(BLOB_ID, rewriter.source()); - assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); - assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); - } - - @Test - public void testRewriteCustumSize() { - RewriteResponse response = new RewriteResponse() - .setDone(true) - .setResource(BLOB_INFO.toPb()) - .setObjectSize(BigInteger.valueOf(42L)) - .setTotalBytesRewritten(BigInteger.valueOf(42L)); - EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); - EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); - EasyMock.expect(storageRpcMock.rewrite( - BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, 1048576L)) - .andReturn(response); - EasyMock.replay(optionsMock, storageRpcMock); - rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) - .isDone(false) - .maxBytesRewrittenPerCall(1048576L) - .build(); - rewriter.copyChunk(); - assertTrue(rewriter.isDone()); - assertEquals(BLOB_INFO, rewriter.target()); - assertEquals(BLOB_ID, rewriter.source()); - assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); - assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); + assertEquals(RESULT, rewriter.result()); + assertEquals(new Long(42L), rewriter.totalBytesRewritten()); + assertEquals(new Long(42L), rewriter.blobSize()); } @Test public void testRewriteMultipleRequests() { - RewriteResponse firstResponse = new RewriteResponse() - .setDone(false) - .setResource(BLOB_INFO.toPb()) - .setObjectSize(BigInteger.valueOf(42L)) - .setRewriteToken("token") - .setTotalBytesRewritten(BigInteger.valueOf(21L)); - RewriteResponse secondResponse = new RewriteResponse() - .setDone(true) - .setResource(BLOB_INFO.toPb()) - .setObjectSize(BigInteger.valueOf(42L)) - .setTotalBytesRewritten(BigInteger.valueOf(42L)); - EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); - EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()).times(2); - EasyMock.expect(storageRpcMock.rewrite( - BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, null, null)) - .andReturn(firstResponse); - EasyMock.expect(storageRpcMock.rewrite( - BLOB_ID.toPb(), EMPTY_OPTIONS, BLOB_INFO.toPb(), EMPTY_OPTIONS, "token", null)) - .andReturn(secondResponse); - EasyMock.replay(optionsMock, storageRpcMock); - rewriter = BlobRewriter.builder(optionsMock, BLOB_ID, EMPTY_OPTIONS, BLOB_INFO, EMPTY_OPTIONS) - .isDone(false) - .build(); + EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE); + EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); + EasyMock.replay(storageRpcMock); + rewriter = new BlobRewriter(options, RESPONSE); int loopCount = 0; while (!rewriter.isDone()) { rewriter.copyChunk(); loopCount++; } assertTrue(rewriter.isDone()); - assertEquals(BLOB_INFO, rewriter.target()); - assertEquals(BLOB_ID, rewriter.source()); - assertEquals(BigInteger.valueOf(42L), rewriter.totalBytesRewritten()); - assertEquals(BigInteger.valueOf(42L), rewriter.blobSize()); + assertEquals(RESULT, rewriter.result()); + assertEquals(new Long(42L), rewriter.totalBytesRewritten()); + assertEquals(new Long(42L), rewriter.blobSize()); assertEquals(2, loopCount); } + + @Test + public void testSaveAndRestore() throws IOException { + EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE); + EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); + EasyMock.replay(storageRpcMock); + rewriter = new BlobRewriter(options, RESPONSE); + rewriter.copyChunk(); + assertTrue(!rewriter.isDone()); + assertEquals(null, rewriter.result()); + assertEquals(new Long(21L), rewriter.totalBytesRewritten()); + assertEquals(new Long(42L), rewriter.blobSize()); + RestorableState rewriterState = rewriter.capture(); + BlobRewriter restoredRewriter = rewriterState.restore(); + restoredRewriter.copyChunk(); + assertTrue(restoredRewriter.isDone()); + assertEquals(RESULT, restoredRewriter.result()); + assertEquals(new Long(42L), restoredRewriter.totalBytesRewritten()); + assertEquals(new Long(42L), restoredRewriter.blobSize()); + } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index b7e7d4b415d0..7025227a27f3 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -664,38 +664,37 @@ public void testUpdateBlobsFail() { @Test public void testRewriteBlob() { String sourceBlobName = "test-rewrite-blob-source"; - BlobId source = BlobId.of(bucket, sourceBlobName); + BlobId source = BlobId.of(BUCKET, sourceBlobName); assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); String targetBlobName = "test-rewrite-blob-target"; - BlobInfo target = BlobInfo.builder(bucket, targetBlobName).contentType(CONTENT_TYPE).build(); + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); Storage.RewriteRequest req = Storage.RewriteRequest.of(source, target); BlobRewriter rewriter = storage.rewriter(req); rewriter.copyChunk(); assertTrue(rewriter.isDone()); - assertEquals(rewriter.target(), storage.get(bucket, targetBlobName)); - assertTrue(storage.delete(bucket, sourceBlobName)); - assertTrue(storage.delete(bucket, targetBlobName)); + assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName)); + assertTrue(storage.delete(BUCKET, sourceBlobName)); + assertTrue(storage.delete(BUCKET, targetBlobName)); } @Test public void testRewriteBlobFail() { String sourceBlobName = "test-rewrite-blob-source-fail"; - BlobId source = BlobId.of(bucket, sourceBlobName); + BlobId source = BlobId.of(BUCKET, sourceBlobName); assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); String targetBlobName = "test-rewrite-blob-target-fail"; - BlobInfo target = BlobInfo.builder(bucket, targetBlobName).contentType(CONTENT_TYPE).build(); + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); Storage.RewriteRequest req = Storage.RewriteRequest.builder() .source(source) .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) .target(target) .build(); - BlobRewriter rewriter = storage.rewriter(req); try { - rewriter.copyChunk(); + storage.rewriter(req); fail("StorageException was expected"); } catch (StorageException ex) { // expected } - assertTrue(storage.delete(bucket, sourceBlobName)); + assertTrue(storage.delete(BUCKET, sourceBlobName)); } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index 9c80d43396c0..6248c9720511 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -38,6 +38,7 @@ import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.spi.StorageRpcFactory; +import com.google.gcloud.storage.Storage.RewriteRequest; import org.easymock.Capture; import org.easymock.EasyMock; @@ -960,6 +961,70 @@ public Tuple apply(StorageObject f) { } } + @Test + public void testRewriter() { + RewriteRequest request = Storage.RewriteRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobRewriter writer = storage.rewriter(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesRewritten()); + assertTrue(!writer.isDone()); + } + + @Test + public void testRewriterWithOptions() { + RewriteRequest request = Storage.RewriteRequest.builder() + .source(BLOB_INFO2.blobId()) + .sourceOptions(BLOB_SOURCE_GENERATION, BLOB_SOURCE_METAGENERATION) + .target(BLOB_INFO1) + .targetOptions(BLOB_TARGET_GENERATION, BLOB_TARGET_METAGENERATION) + .build(); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + BLOB_SOURCE_OPTIONS_COPY, request.target().toPb(), BLOB_TARGET_OPTIONS_COMPOSE, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobRewriter writer = storage.rewriter(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesRewritten()); + assertTrue(!writer.isDone()); + } + + @Test + public void testRewriterMultipleRequests() { + RewriteRequest request = Storage.RewriteRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); + StorageRpc.RewriteResponse rpcResponse1 = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + StorageRpc.RewriteResponse rpcResponse2 = new StorageRpc.RewriteResponse(rpcRequest, + BLOB_INFO1.toPb(), 42L, true, "token", 42L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse1); + EasyMock.expect(storageRpcMock.continueRewrite(rpcResponse1)).andReturn(rpcResponse2); + EasyMock.replay(storageRpcMock); + storage = options.service(); + BlobRewriter writer = storage.rewriter(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesRewritten()); + assertTrue(!writer.isDone()); + writer.copyChunk(); + assertTrue(writer.isDone()); + assertEquals(BLOB_INFO1, writer.result()); + assertEquals(new Long(42L), writer.totalBytesRewritten()); + assertEquals(new Long(42L), writer.blobSize()); + } + @Test public void testRetryableException() { BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); From 43460b9d9e01c52368d670119bf3a357e1d607aa Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 28 Oct 2015 15:31:54 +0100 Subject: [PATCH 3/6] Use StorageRpc.rewrite to implement Storage.copy - rename BlobRewriter to CopyWriter - update Storage.copy and Blob.copyTo methods to use StorageRpc.openRewrite - update Blob and Storage unit and intergration tests - update StorageExample --- .../gcloud/examples/StorageExample.java | 15 +- .../java/com/google/gcloud/storage/Blob.java | 31 ++-- .../{BlobRewriter.java => CopyWriter.java} | 45 +++--- .../com/google/gcloud/storage/Storage.java | 153 +++--------------- .../google/gcloud/storage/StorageImpl.java | 35 +--- .../com/google/gcloud/storage/BlobTest.java | 32 ++-- ...bRewriterTest.java => CopyWriterTest.java} | 48 +++--- .../google/gcloud/storage/ITStorageTest.java | 95 ++++------- .../gcloud/storage/StorageImplTest.java | 129 ++++++--------- 9 files changed, 204 insertions(+), 379 deletions(-) rename gcloud-java-storage/src/main/java/com/google/gcloud/storage/{BlobRewriter.java => CopyWriter.java} (84%) rename gcloud-java-storage/src/test/java/com/google/gcloud/storage/{BlobRewriterTest.java => CopyWriterTest.java} (78%) diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index 274710f10e93..a13e057d2555 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -24,6 +24,7 @@ import com.google.gcloud.storage.BlobId; import com.google.gcloud.storage.BlobInfo; import com.google.gcloud.storage.BlobReadChannel; +import com.google.gcloud.storage.CopyWriter; import com.google.gcloud.storage.BlobWriteChannel; import com.google.gcloud.storage.Bucket; import com.google.gcloud.storage.BucketInfo; @@ -366,21 +367,25 @@ public String params() { private static class CopyAction extends StorageAction { @Override public void run(Storage storage, CopyRequest request) { - BlobInfo copiedBlobInfo = storage.copy(request); - System.out.println("Copied " + copiedBlobInfo); + CopyWriter copyWriter = storage.copy(request); + while (!copyWriter.isDone()) { + copyWriter.copyChunk(); + } + System.out.println("Copied " + copyWriter.result()); } @Override CopyRequest parse(String... args) { - if (args.length != 4) { + if (args.length != 5) { throw new IllegalArgumentException(); } - return CopyRequest.of(args[0], args[1], BlobInfo.builder(args[2], args[3]).build()); + return CopyRequest.of(args[0], args[1], + BlobInfo.builder(args[2], args[3]).contentType(args[4]).build()); } @Override public String params() { - return " "; + return " "; } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index bc75408997f4..dfd112d39c55 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -213,19 +213,20 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { } /** - * Copies this blob to the specified target. Possibly copying also some of the metadata - * (e.g. content-type). + * Sends a copy request for the current blob to the target blob. Possibly also some of the + * metadata are copied (e.g. content-type). * * @param targetBlob target blob's id * @param options source blob options - * @return the copied blob + * @return a {@link CopyWriter} object that can be used to get information on the newly created + * blob or to complete the copy if more than one RPC request is needed * @throws StorageException upon failure */ - public Blob copyTo(BlobId targetBlob, BlobSourceOption... options) { + public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) { BlobInfo updatedInfo = info.toBuilder().blobId(targetBlob).build(); CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) .sourceOptions(convert(info, options)).target(updatedInfo).build(); - return new Blob(storage, storage.copy(copyRequest)); + return storage.copy(copyRequest); } /** @@ -240,33 +241,35 @@ public boolean delete(BlobSourceOption... options) { } /** - * Copies this blob to the target bucket, preserving its name. Possibly copying also some of the - * metadata (e.g. content-type). + * Sends a copy request for the current blob to the target bucket, preserving its name. Possibly + * copying also some of the metadata (e.g. content-type). * * @param targetBucket target bucket's name * @param options source blob options - * @return the copied blob + * @return a {@link CopyWriter} object that can be used to get information on the newly created + * blob or to complete the copy if more than one RPC request is needed * @throws StorageException upon failure */ - public Blob copyTo(String targetBucket, BlobSourceOption... options) { + public CopyWriter copyTo(String targetBucket, BlobSourceOption... options) { return copyTo(targetBucket, info.name(), options); } /** - * Copies this blob to the target bucket with a new name. Possibly copying also some of the - * metadata (e.g. content-type). + * Sends a copy request for the current blob to the target blob. Possibly also some of the + * metadata are copied (e.g. content-type). * * @param targetBucket target bucket's name * @param targetBlob target blob's name * @param options source blob options - * @return the copied blob + * @return a {@link CopyWriter} object that can be used to get information on the newly created + * blob or to complete the copy if more than one RPC request is needed * @throws StorageException upon failure */ - public Blob copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) { + public CopyWriter copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) { BlobInfo updatedInfo = info.toBuilder().blobId(BlobId.of(targetBucket, targetBlob)).build(); CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) .sourceOptions(convert(info, options)).target(updatedInfo).build(); - return new Blob(storage, storage.copy(copyRequest)); + return storage.copy(copyRequest); } /** diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java similarity index 84% rename from gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java rename to gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java index 75b0516f0acf..916d3c18865f 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/BlobRewriter.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java @@ -32,15 +32,20 @@ import java.util.concurrent.Callable; /** - * Google Storage blob rewriter. + * Google Storage blob copy writer. This class holds the result of a copy request. + * If source and destination blobs do not share the same location or storage class more than one + * RPC request is needed to copy the blob. When this is the case {@link #copyChunk()} can be used + * to copy to destination other chunks of the source blob. + * + * @see Rewrite */ -public final class BlobRewriter implements Restorable { +public class CopyWriter implements Restorable { private final StorageOptions serviceOptions; private final StorageRpc storageRpc; private RewriteResponse rewriteResponse; - BlobRewriter(StorageOptions serviceOptions, RewriteResponse rewriteResponse) { + CopyWriter(StorageOptions serviceOptions, RewriteResponse rewriteResponse) { this.serviceOptions = serviceOptions; this.rewriteResponse = rewriteResponse; this.storageRpc = serviceOptions.rpc(); @@ -69,14 +74,14 @@ public Boolean isDone() { } /** - * Returns the number of bytes written. + * Returns the number of bytes copied. */ - public Long totalBytesRewritten() { + public Long totalBytesCopied() { return rewriteResponse.totalBytesRewritten; } /** - * Rewrite the next chunk of the blob. An RPC is issued only if rewrite has not finished yet + * Copies the next chunk of the blob. An RPC is issued only if copy has not finished yet * ({@link #isDone} returns {@code false}). * * @throws StorageException upon failure @@ -97,7 +102,7 @@ public RewriteResponse call() { } @Override - public RestorableState capture() { + public RestorableState capture() { return StateImpl.builder( serviceOptions, BlobId.fromPb(rewriteResponse.rewriteRequest.source), @@ -108,11 +113,11 @@ public RestorableState capture() { .isDone(isDone()) .megabytesRewrittenPerCall(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall) .rewriteToken(rewriteResponse.rewriteToken) - .totalBytesRewritten(totalBytesRewritten()) + .totalBytesRewritten(totalBytesCopied()) .build(); } - static class StateImpl implements RestorableState, Serializable { + static class StateImpl implements RestorableState, Serializable { private static final long serialVersionUID = 8279287678903181701L; @@ -125,7 +130,7 @@ static class StateImpl implements RestorableState, Serializable { private final Long blobSize; private final Boolean isDone; private final String rewriteToken; - private final Long totalBytesRewritten; + private final Long totalBytesCopied; private final Long megabytesRewrittenPerCall; StateImpl(Builder builder) { @@ -138,7 +143,7 @@ static class StateImpl implements RestorableState, Serializable { this.blobSize = builder.blobSize; this.isDone = builder.isDone; this.rewriteToken = builder.rewriteToken; - this.totalBytesRewritten = builder.totalBytesRewritten; + this.totalBytesCopied = builder.totalBytesCopied; this.megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall; } @@ -153,7 +158,7 @@ static class Builder { private Long blobSize; private Boolean isDone; private String rewriteToken; - private Long totalBytesRewritten; + private Long totalBytesCopied; private Long megabytesRewrittenPerCall; private Builder(StorageOptions options, BlobId source, @@ -187,7 +192,7 @@ Builder rewriteToken(String rewriteToken) { } Builder totalBytesRewritten(Long totalBytesRewritten) { - this.totalBytesRewritten = totalBytesRewritten; + this.totalBytesCopied = totalBytesRewritten; return this; } @@ -196,7 +201,7 @@ Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { return this; } - RestorableState build() { + RestorableState build() { return new StateImpl(this); } } @@ -208,19 +213,19 @@ static Builder builder(StorageOptions options, BlobId source, } @Override - public BlobRewriter restore() { + public CopyWriter restore() { RewriteRequest rewriteRequest = new RewriteRequest( source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesRewrittenPerCall); RewriteResponse rewriteResponse = new RewriteResponse(rewriteRequest, result != null ? result.toPb() : null, blobSize, isDone, rewriteToken, - totalBytesRewritten); - return new BlobRewriter(serviceOptions, rewriteResponse); + totalBytesCopied); + return new CopyWriter(serviceOptions, rewriteResponse); } @Override public int hashCode() { return Objects.hash(serviceOptions, source, sourceOptions, target, targetOptions, result, - blobSize, isDone, megabytesRewrittenPerCall, rewriteToken, totalBytesRewritten); + blobSize, isDone, megabytesRewrittenPerCall, rewriteToken, totalBytesCopied); } @Override @@ -242,7 +247,7 @@ public boolean equals(Object obj) { && Objects.equals(this.blobSize, other.blobSize) && Objects.equals(this.isDone, other.isDone) && Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall) - && Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten); + && Objects.equals(this.totalBytesCopied, other.totalBytesCopied); } @Override @@ -251,7 +256,7 @@ public String toString() { .add("source", source) .add("target", target) .add("isDone", isDone) - .add("totalBytesRewritten", totalBytesRewritten) + .add("totalBytesRewritten", totalBytesCopied) .add("blobSize", blobSize) .toString(); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 88f2a80f8b9b..cc9c3770e790 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -494,110 +494,6 @@ public static Builder builder() { class CopyRequest implements Serializable { - private static final long serialVersionUID = -2606508373751748775L; - - private final BlobId source; - private final List sourceOptions; - private final BlobInfo target; - private final List targetOptions; - - public static class Builder { - - private final Set sourceOptions = new LinkedHashSet<>(); - private final Set targetOptions = new LinkedHashSet<>(); - private BlobId source; - private BlobInfo target; - - public Builder source(String bucket, String blob) { - this.source = BlobId.of(bucket, blob); - return this; - } - - public Builder source(BlobId source) { - this.source = source; - return this; - } - - public Builder sourceOptions(BlobSourceOption... options) { - Collections.addAll(sourceOptions, options); - return this; - } - - public Builder sourceOptions(Iterable options) { - Iterables.addAll(sourceOptions, options); - return this; - } - - public Builder target(BlobInfo target) { - this.target = target; - return this; - } - - public Builder targetOptions(BlobTargetOption... options) { - Collections.addAll(targetOptions, options); - return this; - } - - public Builder targetOptions(Iterable options) { - Iterables.addAll(targetOptions, options); - return this; - } - - public CopyRequest build() { - checkNotNull(source); - checkNotNull(target); - return new CopyRequest(this); - } - } - - private CopyRequest(Builder builder) { - source = checkNotNull(builder.source); - sourceOptions = ImmutableList.copyOf(builder.sourceOptions); - target = checkNotNull(builder.target); - targetOptions = ImmutableList.copyOf(builder.targetOptions); - } - - public BlobId source() { - return source; - } - - public List sourceOptions() { - return sourceOptions; - } - - public BlobInfo target() { - return target; - } - - public List targetOptions() { - return targetOptions; - } - - public static CopyRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { - return builder().source(sourceBucket, sourceBlob).target(target).build(); - } - - public static CopyRequest of(BlobId sourceBlobId, BlobInfo target) { - return builder().source(sourceBlobId).target(target).build(); - } - - public static CopyRequest of(String sourceBucket, String sourceBlob, String targetBlob) { - return of(sourceBucket, sourceBlob, - BlobInfo.builder(BlobId.of(sourceBucket, targetBlob)).build()); - } - - public static CopyRequest of(BlobId sourceBlobId, String targetBlob) { - return of(sourceBlobId, - BlobInfo.builder(BlobId.of(sourceBlobId.bucket(), targetBlob)).build()); - } - - public static Builder builder() { - return new Builder(); - } - } - - class RewriteRequest implements Serializable { - private static final long serialVersionUID = -4498650529476219937L; private final BlobId source; @@ -699,14 +595,14 @@ public Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { /** * Creates a {@code RewriteRequest}. */ - public RewriteRequest build() { + public CopyRequest build() { checkNotNull(source); checkNotNull(target); - return new RewriteRequest(this); + return new CopyRequest(this); } } - private RewriteRequest(Builder builder) { + private CopyRequest(Builder builder) { source = checkNotNull(builder.source); sourceOptions = ImmutableList.copyOf(builder.sourceOptions); target = checkNotNull(builder.target); @@ -751,11 +647,11 @@ public Long megabytesRewrittenPerCall() { return megabytesRewrittenPerCall; } - public static RewriteRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { + public static CopyRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { return builder().source(sourceBucket, sourceBlob).target(target).build(); } - public static RewriteRequest of(BlobId sourceBlobId, BlobInfo target) { + public static CopyRequest of(BlobId sourceBlobId, BlobInfo target) { return builder().source(sourceBlobId).target(target).build(); } @@ -923,12 +819,26 @@ public static Builder builder() { BlobInfo compose(ComposeRequest composeRequest); /** - * Send a copy request. + * Sends a copy request. Returns a {@link CopyWriter} object for the provided + * {@code CopyRequest}. If source and destination objects share the same location and storage + * class the source blob is copied and its information can be accessed with + * {@link CopyWriter#result()}, regardless of the {@link CopyRequest#megabytesRewrittenPerCall} + * parameter. If source and destination have different location or storage class multiple RPC + * calls might be needed depending on blob's size. + *

+ * Example usage of copy: + * 

    {@code CopyWriter copyWriter = service.copy(copyRequest);}
+   *    {@code while(!copyWriter.isDone()) {
+   *       copyWriter.copyChunk();
+   *   }}
+   *
* - * @return the copied blob. + * @return a {@link CopyWriter} object that can be used to get information on the newly created + * blob or to complete the copy if more than one RPC request is needed * @throws StorageException upon failure + * @see Rewrite */ - BlobInfo copy(CopyRequest copyRequest); + CopyWriter copy(CopyRequest copyRequest); /** * Reads all the bytes from a blob. @@ -1033,23 +943,4 @@ public static Builder builder() { * @throws StorageException upon failure */ List delete(BlobId... blobIds); - - /** - * Returns a {@link BlobRewriter} object for the provided {@code RewriteRequest}. If source and - * destination objects share the same location and storage class the source blob is copied with a - * single call of {@link BlobRewriter#copyChunk()}, regardless of the {@link - * RewriteRequest#maxBytesRewrittenPerCall} parameter. If source and destination have different - * location or storage class multiple RPC calls might be needed depending on blob's size. - *

- * Example usage of blob rewriter: - * 

    {@code BlobRewriter rewriter = service.rewriter(rewriteRequest);}
-   *    {@code while(!rewriter.isDone()) {
-   *       rewriter.copyChunk();
-   *   }}
-   *
- * - * @throws StorageException upon failure - * @see Rewrite - */ - BlobRewriter rewriter(RewriteRequest rewriteRequest); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index 7bc39481b8a0..a9fdee6705a0 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -445,21 +445,22 @@ public StorageObject call() { } @Override - public BlobInfo copy(CopyRequest copyRequest) { + public CopyWriter copy(final CopyRequest copyRequest) { final StorageObject source = copyRequest.source().toPb(); - copyRequest.sourceOptions(); final Map sourceOptions = optionMap(null, null, copyRequest.sourceOptions(), true); final StorageObject target = copyRequest.target().toPb(); final Map targetOptions = optionMap(copyRequest.target().generation(), copyRequest.target().metageneration(), copyRequest.targetOptions()); try { - return BlobInfo.fromPb(runWithRetries(new Callable() { + RewriteResponse rewriteResponse = runWithRetries(new Callable() { @Override - public StorageObject call() { - return storageRpc.copy(source, sourceOptions, target, targetOptions); + public RewriteResponse call() { + return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, target, + targetOptions, copyRequest.megabytesRewrittenPerCall())); } - }, options().retryParams(), EXCEPTION_HANDLER)); + }, options().retryParams(), EXCEPTION_HANDLER); + return new CopyWriter(options(), rewriteResponse); } catch (RetryHelperException e) { throw StorageException.translateAndThrow(e); } @@ -664,28 +665,6 @@ public List delete(BlobId... blobIds) { return Collections.unmodifiableList(transformResultList(response.deletes(), Boolean.FALSE)); } - @Override - public BlobRewriter rewriter(final RewriteRequest rewriteRequest) { - final StorageObject source = rewriteRequest.source().toPb(); - final Map sourceOptions = - optionMap(null, null, rewriteRequest.sourceOptions(), true); - final StorageObject target = rewriteRequest.target().toPb(); - final Map targetOptions = optionMap(rewriteRequest.target().generation(), - rewriteRequest.target().metageneration(), rewriteRequest.targetOptions()); - try { - RewriteResponse rewriteResponse = runWithRetries(new Callable() { - @Override - public RewriteResponse call() { - return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, target, - targetOptions, rewriteRequest.megabytesRewrittenPerCall())); - } - }, options().retryParams(), EXCEPTION_HANDLER); - return new BlobRewriter(options(), rewriteResponse); - } catch (RetryHelperException e) { - throw StorageException.translateAndThrow(e); - } - } - private static List transformResultList( List> results, final T errorValue) { return Lists.transform(results, new Function, T>() { diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java index dddbb763f04c..095937981b7f 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java @@ -31,10 +31,12 @@ import com.google.api.client.util.Lists; import com.google.gcloud.storage.Storage.CopyRequest; + import org.easymock.Capture; import org.junit.After; import org.junit.Before; import org.junit.Test; + import java.net.URL; import java.util.Arrays; import java.util.List; @@ -120,41 +122,47 @@ public void testDelete() throws Exception { @Test public void testCopyToBucket() throws Exception { BlobInfo target = BLOB_INFO.toBuilder().blobId(BlobId.of("bt", "n")).build(); + CopyWriter copyWriter = createMock(CopyWriter.class); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); + expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); - Blob targetBlob = blob.copyTo("bt"); - assertEquals(target, targetBlob.info()); + CopyWriter returnedCopyWriter = blob.copyTo("bt"); + assertEquals(copyWriter, returnedCopyWriter); assertEquals(capturedCopyRequest.getValue().source(), blob.id()); assertEquals(capturedCopyRequest.getValue().target(), target); - assertSame(storage, targetBlob.storage()); + assertTrue(capturedCopyRequest.getValue().sourceOptions().isEmpty()); + assertTrue(capturedCopyRequest.getValue().targetOptions().isEmpty()); } @Test public void testCopyTo() throws Exception { BlobInfo target = BLOB_INFO.toBuilder().blobId(BlobId.of("bt", "nt")).build(); + CopyWriter copyWriter = createMock(CopyWriter.class); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); + expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); - Blob targetBlob = blob.copyTo("bt", "nt"); - assertEquals(target, targetBlob.info()); + CopyWriter returnedCopyWriter = blob.copyTo("bt", "nt"); + assertEquals(copyWriter, returnedCopyWriter); assertEquals(capturedCopyRequest.getValue().source(), blob.id()); assertEquals(capturedCopyRequest.getValue().target(), target); - assertSame(storage, targetBlob.storage()); + assertTrue(capturedCopyRequest.getValue().sourceOptions().isEmpty()); + assertTrue(capturedCopyRequest.getValue().targetOptions().isEmpty()); } @Test public void testCopyToBlobId() throws Exception { BlobId targetId = BlobId.of("bt", "nt"); + CopyWriter copyWriter = createMock(CopyWriter.class); BlobInfo target = BLOB_INFO.toBuilder().blobId(targetId).build(); Capture capturedCopyRequest = Capture.newInstance(); - expect(storage.copy(capture(capturedCopyRequest))).andReturn(target); + expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter); replay(storage); - Blob targetBlob = blob.copyTo(targetId); - assertEquals(target, targetBlob.info()); + CopyWriter returnedCopyWriter = blob.copyTo(targetId); + assertEquals(copyWriter, returnedCopyWriter); assertEquals(capturedCopyRequest.getValue().source(), blob.id()); assertEquals(capturedCopyRequest.getValue().target(), target); - assertSame(storage, targetBlob.storage()); + assertTrue(capturedCopyRequest.getValue().sourceOptions().isEmpty()); + assertTrue(capturedCopyRequest.getValue().targetOptions().isEmpty()); } @Test diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java similarity index 78% rename from gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java rename to gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java index 83cda633fd6a..feba88d57118 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobRewriterTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java @@ -39,7 +39,7 @@ import java.io.IOException; import java.util.Map; -public class BlobRewriterTest { +public class CopyWriterTest { private static final String SOURCE_BUCKET_NAME = "b"; private static final String SOURCE_BLOB_NAME = "n"; @@ -61,7 +61,7 @@ public class BlobRewriterTest { private StorageOptions options; private StorageRpcFactory rpcFactoryMock; private StorageRpc storageRpcMock; - private BlobRewriter rewriter; + private CopyWriter copyWriter; @Before public void setUp() throws IOException, InterruptedException { @@ -85,12 +85,12 @@ public void tearDown() throws Exception { public void testRewrite() { EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); EasyMock.replay(storageRpcMock); - rewriter = new BlobRewriter(options, RESPONSE); - rewriter.copyChunk(); - assertTrue(rewriter.isDone()); - assertEquals(RESULT, rewriter.result()); - assertEquals(new Long(42L), rewriter.totalBytesRewritten()); - assertEquals(new Long(42L), rewriter.blobSize()); + copyWriter = new CopyWriter(options, RESPONSE); + copyWriter.copyChunk(); + assertTrue(copyWriter.isDone()); + assertEquals(RESULT, copyWriter.result()); + assertEquals(new Long(42L), copyWriter.totalBytesCopied()); + assertEquals(new Long(42L), copyWriter.blobSize()); } @Test @@ -98,16 +98,16 @@ public void testRewriteMultipleRequests() { EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE); EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); EasyMock.replay(storageRpcMock); - rewriter = new BlobRewriter(options, RESPONSE); + copyWriter = new CopyWriter(options, RESPONSE); int loopCount = 0; - while (!rewriter.isDone()) { - rewriter.copyChunk(); + while (!copyWriter.isDone()) { + copyWriter.copyChunk(); loopCount++; } - assertTrue(rewriter.isDone()); - assertEquals(RESULT, rewriter.result()); - assertEquals(new Long(42L), rewriter.totalBytesRewritten()); - assertEquals(new Long(42L), rewriter.blobSize()); + assertTrue(copyWriter.isDone()); + assertEquals(RESULT, copyWriter.result()); + assertEquals(new Long(42L), copyWriter.totalBytesCopied()); + assertEquals(new Long(42L), copyWriter.blobSize()); assertEquals(2, loopCount); } @@ -116,18 +116,18 @@ public void testSaveAndRestore() throws IOException { EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE); EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE); EasyMock.replay(storageRpcMock); - rewriter = new BlobRewriter(options, RESPONSE); - rewriter.copyChunk(); - assertTrue(!rewriter.isDone()); - assertEquals(null, rewriter.result()); - assertEquals(new Long(21L), rewriter.totalBytesRewritten()); - assertEquals(new Long(42L), rewriter.blobSize()); - RestorableState rewriterState = rewriter.capture(); - BlobRewriter restoredRewriter = rewriterState.restore(); + copyWriter = new CopyWriter(options, RESPONSE); + copyWriter.copyChunk(); + assertTrue(!copyWriter.isDone()); + assertEquals(null, copyWriter.result()); + assertEquals(new Long(21L), copyWriter.totalBytesCopied()); + assertEquals(new Long(42L), copyWriter.blobSize()); + RestorableState rewriterState = copyWriter.capture(); + CopyWriter restoredRewriter = rewriterState.restore(); restoredRewriter.copyChunk(); assertTrue(restoredRewriter.isDone()); assertEquals(RESULT, restoredRewriter.result()); - assertEquals(new Long(42L), restoredRewriter.totalBytesRewritten()); + assertEquals(new Long(42L), restoredRewriter.totalBytesCopied()); assertEquals(new Long(42L), restoredRewriter.blobSize()); } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 7025227a27f3..612d4c37aa07 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -317,16 +317,15 @@ public void testComposeBlobFail() { @Test public void testCopyBlob() { String sourceBlobName = "test-copy-blob-source"; - BlobInfo blob = BlobInfo.builder(BUCKET, sourceBlobName).build(); - assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT)); - String targetBlobName = "test-copy-blob-target"; - Storage.CopyRequest req = Storage.CopyRequest.of(blob.blobId(), targetBlobName); - BlobInfo remoteBlob = storage.copy(req); - assertNotNull(remoteBlob); - assertEquals(BUCKET, remoteBlob.bucket()); - assertEquals(targetBlobName, remoteBlob.name()); - byte[] readBytes = storage.readAllBytes(BUCKET, targetBlobName); - assertArrayEquals(BLOB_BYTE_CONTENT, readBytes); + BlobId source = BlobId.of(BUCKET, sourceBlobName); + assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); + String targetBlobName = "test-rewrite-blob-target"; + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); + Storage.CopyRequest req = Storage.CopyRequest.of(source, target); + CopyWriter rewriter = storage.copy(req); + rewriter.copyChunk(); + assertTrue(rewriter.isDone()); + assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName)); assertTrue(storage.delete(BUCKET, sourceBlobName)); assertTrue(storage.delete(BUCKET, targetBlobName)); } @@ -334,30 +333,33 @@ public void testCopyBlob() { @Test public void testCopyBlobUpdateMetadata() { String sourceBlobName = "test-copy-blob-update-metadata-source"; - BlobInfo sourceBlob = BlobInfo.builder(BUCKET, sourceBlobName).build(); - assertNotNull(storage.create(sourceBlob)); - String targetBlobName = "test-copy-blob-update-metadata-target"; - BlobInfo targetBlob = - BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); - Storage.CopyRequest req = Storage.CopyRequest.of(BUCKET, sourceBlobName, targetBlob); - BlobInfo remoteBlob = storage.copy(req); - assertNotNull(remoteBlob); - assertEquals(targetBlob.blobId(), remoteBlob.blobId()); - assertEquals(CONTENT_TYPE, remoteBlob.contentType()); + BlobId source = BlobId.of(BUCKET, sourceBlobName); + assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); + String targetBlobName = "test-rewrite-blob-target"; + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName) + .contentType(CONTENT_TYPE) + .metadata(ImmutableMap.of("k", "v")) + .build(); + Storage.CopyRequest req = Storage.CopyRequest.of(source, target); + CopyWriter rewriter = storage.copy(req); + rewriter.copyChunk(); + assertTrue(rewriter.isDone()); + assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName)); assertTrue(storage.delete(BUCKET, sourceBlobName)); assertTrue(storage.delete(BUCKET, targetBlobName)); } @Test public void testCopyBlobFail() { - String sourceBlobName = "test-copy-blob-fail-source"; - BlobInfo blob = BlobInfo.builder(BUCKET, sourceBlobName).build(); - assertNotNull(storage.create(blob)); - String targetBlobName = "test-copy-blob-fail-target"; - Storage.CopyRequest req = new Storage.CopyRequest.Builder() - .source(BUCKET, sourceBlobName) - .target(BlobInfo.builder(BUCKET, targetBlobName).build()) - .sourceOptions(Storage.BlobSourceOption.metagenerationMatch(-1L)) + String sourceBlobName = "test-copy-blob-source-fail"; + BlobId source = BlobId.of(BUCKET, sourceBlobName); + assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); + String targetBlobName = "test-rewrite-blob-target-fail"; + BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); + Storage.CopyRequest req = Storage.CopyRequest.builder() + .source(source) + .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) + .target(target) .build(); try { storage.copy(req); @@ -660,41 +662,4 @@ public void testUpdateBlobsFail() { assertNull(updatedBlobs.get(1)); assertTrue(storage.delete(BUCKET, sourceBlobName1)); } - - @Test - public void testRewriteBlob() { - String sourceBlobName = "test-rewrite-blob-source"; - BlobId source = BlobId.of(BUCKET, sourceBlobName); - assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); - String targetBlobName = "test-rewrite-blob-target"; - BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); - Storage.RewriteRequest req = Storage.RewriteRequest.of(source, target); - BlobRewriter rewriter = storage.rewriter(req); - rewriter.copyChunk(); - assertTrue(rewriter.isDone()); - assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName)); - assertTrue(storage.delete(BUCKET, sourceBlobName)); - assertTrue(storage.delete(BUCKET, targetBlobName)); - } - - @Test - public void testRewriteBlobFail() { - String sourceBlobName = "test-rewrite-blob-source-fail"; - BlobId source = BlobId.of(BUCKET, sourceBlobName); - assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT)); - String targetBlobName = "test-rewrite-blob-target-fail"; - BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build(); - Storage.RewriteRequest req = Storage.RewriteRequest.builder() - .source(source) - .sourceOptions(Storage.BlobSourceOption.generationMatch(-1L)) - .target(target) - .build(); - try { - storage.rewriter(req); - fail("StorageException was expected"); - } catch (StorageException ex) { - // expected - } - assertTrue(storage.delete(BUCKET, sourceBlobName)); - } } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index 6248c9720511..1a99741fa4d0 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -38,7 +38,7 @@ import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.spi.StorageRpc.Tuple; import com.google.gcloud.spi.StorageRpcFactory; -import com.google.gcloud.storage.Storage.RewriteRequest; +import com.google.gcloud.storage.Storage.CopyRequest; import org.easymock.Capture; import org.easymock.EasyMock; @@ -591,33 +591,66 @@ public void testComposeWithOptions() { @Test public void testCopy() { - Storage.CopyRequest req = Storage.CopyRequest.builder() - .source(BUCKET_NAME1, BLOB_NAME2) - .target(BLOB_INFO1) - .build(); - EasyMock.expect(storageRpcMock.copy(BLOB_INFO2.toPb(), EMPTY_RPC_OPTIONS, BLOB_INFO1.toPb(), - EMPTY_RPC_OPTIONS)).andReturn(BLOB_INFO1.toPb()); + CopyRequest request = Storage.CopyRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); EasyMock.replay(storageRpcMock); storage = options.service(); - BlobInfo blob = storage.copy(req); - assertEquals(BLOB_INFO1, blob); + CopyWriter writer = storage.copy(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesCopied()); + assertTrue(!writer.isDone()); } @Test public void testCopyWithOptions() { - Storage.CopyRequest req = Storage.CopyRequest.builder() - .source(BUCKET_NAME1, BLOB_NAME2) + CopyRequest request = Storage.CopyRequest.builder() + .source(BLOB_INFO2.blobId()) .sourceOptions(BLOB_SOURCE_GENERATION, BLOB_SOURCE_METAGENERATION) .target(BLOB_INFO1) .targetOptions(BLOB_TARGET_GENERATION, BLOB_TARGET_METAGENERATION) .build(); - EasyMock.expect( - storageRpcMock.copy(BLOB_INFO2.toPb(), BLOB_SOURCE_OPTIONS_COPY, BLOB_INFO1.toPb(), - BLOB_TARGET_OPTIONS_COMPOSE)).andReturn(BLOB_INFO1.toPb()); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + BLOB_SOURCE_OPTIONS_COPY, request.target().toPb(), BLOB_TARGET_OPTIONS_COMPOSE, null); + StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); EasyMock.replay(storageRpcMock); storage = options.service(); - BlobInfo blob = storage.copy(req); - assertEquals(BLOB_INFO1, blob); + CopyWriter writer = storage.copy(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesCopied()); + assertTrue(!writer.isDone()); + } + + @Test + public void testCopyMultipleRequests() { + CopyRequest request = Storage.CopyRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); + StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), + EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); + StorageRpc.RewriteResponse rpcResponse1 = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, + false, "token", 21L); + StorageRpc.RewriteResponse rpcResponse2 = new StorageRpc.RewriteResponse(rpcRequest, + BLOB_INFO1.toPb(), 42L, true, "token", 42L); + EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse1); + EasyMock.expect(storageRpcMock.continueRewrite(rpcResponse1)).andReturn(rpcResponse2); + EasyMock.replay(storageRpcMock); + storage = options.service(); + CopyWriter writer = storage.copy(request); + assertNull(writer.result()); + assertEquals(new Long(42L), writer.blobSize()); + assertEquals(new Long(21L), writer.totalBytesCopied()); + assertTrue(!writer.isDone()); + writer.copyChunk(); + assertTrue(writer.isDone()); + assertEquals(BLOB_INFO1, writer.result()); + assertEquals(new Long(42L), writer.totalBytesCopied()); + assertEquals(new Long(42L), writer.blobSize()); } @Test @@ -961,70 +994,6 @@ public Tuple apply(StorageObject f) { } } - @Test - public void testRewriter() { - RewriteRequest request = Storage.RewriteRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); - StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), - EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); - StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, - false, "token", 21L); - EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); - EasyMock.replay(storageRpcMock); - storage = options.service(); - BlobRewriter writer = storage.rewriter(request); - assertNull(writer.result()); - assertEquals(new Long(42L), writer.blobSize()); - assertEquals(new Long(21L), writer.totalBytesRewritten()); - assertTrue(!writer.isDone()); - } - - @Test - public void testRewriterWithOptions() { - RewriteRequest request = Storage.RewriteRequest.builder() - .source(BLOB_INFO2.blobId()) - .sourceOptions(BLOB_SOURCE_GENERATION, BLOB_SOURCE_METAGENERATION) - .target(BLOB_INFO1) - .targetOptions(BLOB_TARGET_GENERATION, BLOB_TARGET_METAGENERATION) - .build(); - StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), - BLOB_SOURCE_OPTIONS_COPY, request.target().toPb(), BLOB_TARGET_OPTIONS_COMPOSE, null); - StorageRpc.RewriteResponse rpcResponse = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, - false, "token", 21L); - EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse); - EasyMock.replay(storageRpcMock); - storage = options.service(); - BlobRewriter writer = storage.rewriter(request); - assertNull(writer.result()); - assertEquals(new Long(42L), writer.blobSize()); - assertEquals(new Long(21L), writer.totalBytesRewritten()); - assertTrue(!writer.isDone()); - } - - @Test - public void testRewriterMultipleRequests() { - RewriteRequest request = Storage.RewriteRequest.of(BLOB_INFO1.blobId(), BLOB_INFO2); - StorageRpc.RewriteRequest rpcRequest = new StorageRpc.RewriteRequest(request.source().toPb(), - EMPTY_RPC_OPTIONS, request.target().toPb(), EMPTY_RPC_OPTIONS, null); - StorageRpc.RewriteResponse rpcResponse1 = new StorageRpc.RewriteResponse(rpcRequest, null, 42L, - false, "token", 21L); - StorageRpc.RewriteResponse rpcResponse2 = new StorageRpc.RewriteResponse(rpcRequest, - BLOB_INFO1.toPb(), 42L, true, "token", 42L); - EasyMock.expect(storageRpcMock.openRewrite(rpcRequest)).andReturn(rpcResponse1); - EasyMock.expect(storageRpcMock.continueRewrite(rpcResponse1)).andReturn(rpcResponse2); - EasyMock.replay(storageRpcMock); - storage = options.service(); - BlobRewriter writer = storage.rewriter(request); - assertNull(writer.result()); - assertEquals(new Long(42L), writer.blobSize()); - assertEquals(new Long(21L), writer.totalBytesRewritten()); - assertTrue(!writer.isDone()); - writer.copyChunk(); - assertTrue(writer.isDone()); - assertEquals(BLOB_INFO1, writer.result()); - assertEquals(new Long(42L), writer.totalBytesRewritten()); - assertEquals(new Long(42L), writer.blobSize()); - } - @Test public void testRetryableException() { BlobId blob = BlobId.of(BUCKET_NAME1, BLOB_NAME1); From f90e70c4e9784d1d99784d863a5ae2c5ec257985 Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 28 Oct 2015 17:13:44 +0100 Subject: [PATCH 4/6] Check contentType != null before adding object to rewrite --- .../com/google/gcloud/examples/StorageExample.java | 7 +++---- .../java/com/google/gcloud/spi/DefaultStorageRpc.java | 2 +- .../main/java/com/google/gcloud/storage/Storage.java | 10 ++++++++++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index a13e057d2555..184c4f3d5e0f 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -376,16 +376,15 @@ public void run(Storage storage, CopyRequest request) { @Override CopyRequest parse(String... args) { - if (args.length != 5) { + if (args.length != 4) { throw new IllegalArgumentException(); } - return CopyRequest.of(args[0], args[1], - BlobInfo.builder(args[2], args[3]).contentType(args[4]).build()); + return CopyRequest.of(args[0], args[1], BlobInfo.builder(args[2], args[3]).build()); } @Override public String params() { - return " "; + return " "; } } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index 1098ca4c538d..c627d65363f6 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -539,7 +539,7 @@ private RewriteResponse rewrite(RewriteRequest req, String token) throws Storage ? req.megabytesRewrittenPerCall * MEGABYTE : null; com.google.api.services.storage.model.RewriteResponse rewriteReponse = storage.objects() .rewrite(req.source.getBucket(), req.source.getName(), req.target.getBucket(), - req.target.getName(), req.target) + req.target.getName(), req.target.getContentType() != null ? req.target : null) .setRewriteToken(token) .setMaxBytesRewrittenPerCall(maxBytesRewrittenPerCall) .setProjection(DEFAULT_PROJECTION) diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index cc9c3770e790..8c505a7826aa 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -655,6 +655,16 @@ public static CopyRequest of(BlobId sourceBlobId, BlobInfo target) { return builder().source(sourceBlobId).target(target).build(); } + public static CopyRequest of(String sourceBucket, String sourceBlob, String targetBlob) { + return of(sourceBucket, sourceBlob, + BlobInfo.builder(BlobId.of(sourceBucket, targetBlob)).build()); + } + + public static CopyRequest of(BlobId sourceBlobId, String targetBlob) { + return of(sourceBlobId, + BlobInfo.builder(BlobId.of(sourceBlobId.bucket(), targetBlob)).build()); + } + public static Builder builder() { return new Builder(); } From 352e09756f24b339619ea5384db9020299766c6a Mon Sep 17 00:00:00 2001 From: Marco Ziccardi Date: Wed, 28 Oct 2015 22:25:42 +0100 Subject: [PATCH 5/6] Refactor CopyWriter, Storage and StorageRpc - remove StorageRpc.copy - make CopyWriter.result issue requests until copy completes - make RewriteResponse and CopyWriter blobSize isDone and totalBytesCopied primitive - rename Storage.RewriteRequest megabytesRewrittenPerCall to megabytesCopiedPerChunk - update tests - update StorageExample --- .../gcloud/examples/StorageExample.java | 3 - .../google/gcloud/spi/DefaultStorageRpc.java | 24 ------- .../com/google/gcloud/spi/StorageRpc.java | 17 +++-- .../java/com/google/gcloud/storage/Blob.java | 4 +- .../com/google/gcloud/storage/CopyWriter.java | 63 ++++++++++--------- .../com/google/gcloud/storage/Storage.java | 39 ++++++------ .../google/gcloud/storage/StorageImpl.java | 2 +- .../com/google/gcloud/storage/BlobTest.java | 6 +- .../google/gcloud/storage/CopyWriterTest.java | 31 ++++----- .../google/gcloud/storage/ITStorageTest.java | 20 +++--- .../gcloud/storage/StorageImplTest.java | 22 +++---- 11 files changed, 94 insertions(+), 137 deletions(-) diff --git a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java index 184c4f3d5e0f..b71a51fe6ac6 100644 --- a/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java +++ b/gcloud-java-examples/src/main/java/com/google/gcloud/examples/StorageExample.java @@ -368,9 +368,6 @@ private static class CopyAction extends StorageAction { @Override public void run(Storage storage, CopyRequest request) { CopyWriter copyWriter = storage.copy(request); - while (!copyWriter.isDone()) { - copyWriter.copyChunk(); - } System.out.println("Copied " + copyWriter.result()); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java index c627d65363f6..957fbc90095d 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/DefaultStorageRpc.java @@ -321,30 +321,6 @@ public StorageObject compose(Iterable sources, StorageObject targ } } - @Override - public StorageObject copy(StorageObject source, Map sourceOptions, - StorageObject target, Map targetOptions) throws StorageException { - try { - return storage - .objects() - .copy(source.getBucket(), source.getName(), target.getBucket(), target.getName(), - target.getContentType() != null ? target : null) - .setProjection(DEFAULT_PROJECTION) - .setIfSourceMetagenerationMatch(IF_SOURCE_METAGENERATION_MATCH.getLong(sourceOptions)) - .setIfSourceMetagenerationNotMatch( - IF_SOURCE_METAGENERATION_NOT_MATCH.getLong(sourceOptions)) - .setIfSourceGenerationMatch(IF_SOURCE_GENERATION_MATCH.getLong(sourceOptions)) - .setIfSourceGenerationNotMatch(IF_SOURCE_GENERATION_NOT_MATCH.getLong(sourceOptions)) - .setIfMetagenerationMatch(IF_METAGENERATION_MATCH.getLong(targetOptions)) - .setIfMetagenerationNotMatch(IF_METAGENERATION_NOT_MATCH.getLong(targetOptions)) - .setIfGenerationMatch(IF_GENERATION_MATCH.getLong(targetOptions)) - .setIfGenerationNotMatch(IF_GENERATION_NOT_MATCH.getLong(targetOptions)) - .execute(); - } catch (IOException ex) { - throw translate(ex); - } - } - @Override public byte[] load(StorageObject from, Map options) throws StorageException { diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java index 1a4afa0524e0..40382a857fca 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/spi/StorageRpc.java @@ -177,13 +177,13 @@ class RewriteResponse { public final RewriteRequest rewriteRequest; public final StorageObject result; - public final Long blobSize; - public final Boolean isDone; + public final long blobSize; + public final boolean isDone; public final String rewriteToken; - public final Long totalBytesRewritten; + public final long totalBytesRewritten; - public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, Long blobSize, - Boolean isDone, String rewriteToken, Long totalBytesRewritten) { + public RewriteResponse(RewriteRequest rewriteRequest, StorageObject result, long blobSize, + boolean isDone, String rewriteToken, long totalBytesRewritten) { this.rewriteRequest = rewriteRequest; this.result = result; this.blobSize = blobSize; @@ -204,9 +204,9 @@ public boolean equals(Object obj) { return Objects.equals(this.rewriteRequest, other.rewriteRequest) && Objects.equals(this.result, other.result) && Objects.equals(this.rewriteToken, other.rewriteToken) - && Objects.equals(this.blobSize, other.blobSize) + && this.blobSize == other.blobSize && Objects.equals(this.isDone, other.isDone) - && Objects.equals(this.totalBytesRewritten, other.totalBytesRewritten); + && this.totalBytesRewritten == other.totalBytesRewritten; } @Override @@ -245,9 +245,6 @@ StorageObject patch(StorageObject storageObject, Map options) StorageObject compose(Iterable sources, StorageObject target, Map targetOptions) throws StorageException; - StorageObject copy(StorageObject source, Map sourceOptions, - StorageObject target, Map targetOptions) throws StorageException; - byte[] load(StorageObject storageObject, Map options) throws StorageException; diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index dfd112d39c55..284a7818457c 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -223,7 +223,7 @@ public Blob update(BlobInfo blobInfo, BlobTargetOption... options) { * @throws StorageException upon failure */ public CopyWriter copyTo(BlobId targetBlob, BlobSourceOption... options) { - BlobInfo updatedInfo = info.toBuilder().blobId(targetBlob).build(); + BlobInfo updatedInfo = BlobInfo.builder(targetBlob).build(); CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) .sourceOptions(convert(info, options)).target(updatedInfo).build(); return storage.copy(copyRequest); @@ -266,7 +266,7 @@ public CopyWriter copyTo(String targetBucket, BlobSourceOption... options) { * @throws StorageException upon failure */ public CopyWriter copyTo(String targetBucket, String targetBlob, BlobSourceOption... options) { - BlobInfo updatedInfo = info.toBuilder().blobId(BlobId.of(targetBucket, targetBlob)).build(); + BlobInfo updatedInfo = BlobInfo.builder(targetBucket, targetBlob).build(); CopyRequest copyRequest = CopyRequest.builder().source(info.bucket(), info.name()) .sourceOptions(convert(info, options)).target(updatedInfo).build(); return storage.copy(copyRequest); diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java index 916d3c18865f..592cd6e0289a 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java @@ -34,8 +34,8 @@ /** * Google Storage blob copy writer. This class holds the result of a copy request. * If source and destination blobs do not share the same location or storage class more than one - * RPC request is needed to copy the blob. When this is the case {@link #copyChunk()} can be used - * to copy to destination other chunks of the source blob. + * RPC request is needed to copy the blob otherwise one or more {@link #copyChunk()} calls are + * necessary to complete the copy. * * @see Rewrite */ @@ -52,31 +52,36 @@ public class CopyWriter implements Restorable { } /** - * Returns the updated information for the just written blob when {@link #isDone} is {@code true}. - * Returns {@code null} otherwise. + * Returns the updated information for the just written blob. This method might block and issue + * several RPC requests to complete blob copy. + * + * @throws StorageException upon failure */ public BlobInfo result() { - return rewriteResponse.result != null ? BlobInfo.fromPb(rewriteResponse.result) : null; + while(!isDone()) { + copyChunk(); + } + return BlobInfo.fromPb(rewriteResponse.result); } /** * Size of the blob being copied. */ - public Long blobSize() { + public long blobSize() { return rewriteResponse.blobSize; } /** * Returns {@code true} of blob rewrite finished, {@code false} otherwise. */ - public Boolean isDone() { + public boolean isDone() { return rewriteResponse.isDone; } /** * Returns the number of bytes copied. */ - public Long totalBytesCopied() { + public long totalBytesCopied() { return rewriteResponse.totalBytesRewritten; } @@ -111,7 +116,7 @@ public RestorableState capture() { rewriteResponse.rewriteRequest.targetOptions) .blobSize(blobSize()) .isDone(isDone()) - .megabytesRewrittenPerCall(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall) + .megabytesCopiedPerChunk(rewriteResponse.rewriteRequest.megabytesRewrittenPerCall) .rewriteToken(rewriteResponse.rewriteToken) .totalBytesRewritten(totalBytesCopied()) .build(); @@ -127,11 +132,11 @@ static class StateImpl implements RestorableState, Serializable { private final BlobInfo target; private final Map targetOptions; private final BlobInfo result; - private final Long blobSize; - private final Boolean isDone; + private final long blobSize; + private final boolean isDone; private final String rewriteToken; - private final Long totalBytesCopied; - private final Long megabytesRewrittenPerCall; + private final long totalBytesCopied; + private final Long megabytesCopiedPerChunk; StateImpl(Builder builder) { this.serviceOptions = builder.serviceOptions; @@ -144,7 +149,7 @@ static class StateImpl implements RestorableState, Serializable { this.isDone = builder.isDone; this.rewriteToken = builder.rewriteToken; this.totalBytesCopied = builder.totalBytesCopied; - this.megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall; + this.megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk; } static class Builder { @@ -155,11 +160,11 @@ static class Builder { private final BlobInfo target; private final Map targetOptions; private BlobInfo result; - private Long blobSize; - private Boolean isDone; + private long blobSize; + private boolean isDone; private String rewriteToken; - private Long totalBytesCopied; - private Long megabytesRewrittenPerCall; + private long totalBytesCopied; + private Long megabytesCopiedPerChunk; private Builder(StorageOptions options, BlobId source, Map sourceOptions, @@ -176,12 +181,12 @@ Builder result(BlobInfo result) { return this; } - Builder blobSize(Long blobSize) { + Builder blobSize(long blobSize) { this.blobSize = blobSize; return this; } - Builder isDone(Boolean isDone) { + Builder isDone(boolean isDone) { this.isDone = isDone; return this; } @@ -191,13 +196,13 @@ Builder rewriteToken(String rewriteToken) { return this; } - Builder totalBytesRewritten(Long totalBytesRewritten) { + Builder totalBytesRewritten(long totalBytesRewritten) { this.totalBytesCopied = totalBytesRewritten; return this; } - Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { - this.megabytesRewrittenPerCall = megabytesRewrittenPerCall; + Builder megabytesCopiedPerChunk(Long megabytesCopiedPerChunk) { + this.megabytesCopiedPerChunk = megabytesCopiedPerChunk; return this; } @@ -215,7 +220,7 @@ static Builder builder(StorageOptions options, BlobId source, @Override public CopyWriter restore() { RewriteRequest rewriteRequest = new RewriteRequest( - source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesRewrittenPerCall); + source.toPb(), sourceOptions, target.toPb(), targetOptions, megabytesCopiedPerChunk); RewriteResponse rewriteResponse = new RewriteResponse(rewriteRequest, result != null ? result.toPb() : null, blobSize, isDone, rewriteToken, totalBytesCopied); @@ -225,7 +230,7 @@ public CopyWriter restore() { @Override public int hashCode() { return Objects.hash(serviceOptions, source, sourceOptions, target, targetOptions, result, - blobSize, isDone, megabytesRewrittenPerCall, rewriteToken, totalBytesCopied); + blobSize, isDone, megabytesCopiedPerChunk, rewriteToken, totalBytesCopied); } @Override @@ -244,10 +249,10 @@ public boolean equals(Object obj) { && Objects.equals(this.targetOptions, other.targetOptions) && Objects.equals(this.result, other.result) && Objects.equals(this.rewriteToken, other.rewriteToken) - && Objects.equals(this.blobSize, other.blobSize) - && Objects.equals(this.isDone, other.isDone) - && Objects.equals(this.megabytesRewrittenPerCall, other.megabytesRewrittenPerCall) - && Objects.equals(this.totalBytesCopied, other.totalBytesCopied); + && Objects.equals(this.megabytesCopiedPerChunk, other.megabytesCopiedPerChunk) + && this.blobSize == other.blobSize + && this.isDone == other.isDone + && this.totalBytesCopied == other.totalBytesCopied; } @Override diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 8c505a7826aa..1c65ff82dfaa 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -500,7 +500,7 @@ class CopyRequest implements Serializable { private final List sourceOptions; private final BlobInfo target; private final List targetOptions; - private final Long megabytesRewrittenPerCall; + private final Long megabytesCopiedPerChunk; public static class Builder { @@ -508,10 +508,10 @@ public static class Builder { private final Set targetOptions = new LinkedHashSet<>(); private BlobId source; private BlobInfo target; - private Long megabytesRewrittenPerCall; + private Long megabytesCopiedPerChunk; /** - * Sets the blob to rewrite given bucket and blob name. + * Sets the blob to copy given bucket and blob name. * * @return the builder. */ @@ -521,7 +521,7 @@ public Builder source(String bucket, String blob) { } /** - * Sets the blob to rewrite given a {@link BlobId}. + * Sets the blob to copy given a {@link BlobId}. * * @return the builder. */ @@ -551,7 +551,7 @@ public Builder sourceOptions(Iterable options) { } /** - * Sets the rewrite target. + * Sets the copy target. * * @return the builder. */ @@ -582,18 +582,18 @@ public Builder targetOptions(Iterable options) { /** * Sets the maximum number of megabytes to copy for each RPC call. This parameter is ignored - * if source and target blob share the same location and storage class as rewrite is made with + * if source and target blob share the same location and storage class as copy is made with * one single RPC. * * @return the builder. */ - public Builder megabytesRewrittenPerCall(Long megabytesRewrittenPerCall) { - this.megabytesRewrittenPerCall = megabytesRewrittenPerCall; + public Builder megabytesCopiedPerChunk(Long megabytesCopiedPerChunk) { + this.megabytesCopiedPerChunk = megabytesCopiedPerChunk; return this; } /** - * Creates a {@code RewriteRequest}. + * Creates a {@code CopyRequest}. */ public CopyRequest build() { checkNotNull(source); @@ -607,7 +607,7 @@ private CopyRequest(Builder builder) { sourceOptions = ImmutableList.copyOf(builder.sourceOptions); target = checkNotNull(builder.target); targetOptions = ImmutableList.copyOf(builder.targetOptions); - megabytesRewrittenPerCall = builder.megabytesRewrittenPerCall; + megabytesCopiedPerChunk = builder.megabytesCopiedPerChunk; } /** @@ -640,11 +640,11 @@ public List targetOptions() { /** * Returns the maximum number of megabytes to copy for each RPC call. This parameter is ignored - * if source and target blob share the same location and storage class as rewrite is made with + * if source and target blob share the same location and storage class as copy is made with * one single RPC. */ - public Long megabytesRewrittenPerCall() { - return megabytesRewrittenPerCall; + public Long megabytesCopiedPerChunk() { + return megabytesCopiedPerChunk; } public static CopyRequest of(String sourceBucket, String sourceBlob, BlobInfo target) { @@ -831,16 +831,13 @@ public static Builder builder() { /** * Sends a copy request. Returns a {@link CopyWriter} object for the provided * {@code CopyRequest}. If source and destination objects share the same location and storage - * class the source blob is copied and its information can be accessed with - * {@link CopyWriter#result()}, regardless of the {@link CopyRequest#megabytesRewrittenPerCall} - * parameter. If source and destination have different location or storage class multiple RPC - * calls might be needed depending on blob's size. + * class the source blob is copied with one request and {@link CopyWriter#result()} immediately + * returns, regardless of the {@link CopyRequest#megabytesCopiedPerChunk} parameter. + * If source and destination have different location or storage class {@link CopyWriter#result()} + * might issue multiple RPC calls depending on blob's size. *

* Example usage of copy: - * 

    {@code CopyWriter copyWriter = service.copy(copyRequest);}
-   *    {@code while(!copyWriter.isDone()) {
-   *       copyWriter.copyChunk();
-   *   }}
+   * 
    {@code BlobInfo blob = service.copy(copyRequest).result();}
    * 

    *
    * @return a {@link CopyWriter} object that can be used to get information on the newly created
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
index a9fdee6705a0..ab85dc8b4609 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java
@@ -457,7 +457,7 @@ public CopyWriter copy(final CopyRequest copyRequest) {
         @Override
         public RewriteResponse call() {
           return storageRpc.openRewrite(new StorageRpc.RewriteRequest(source, sourceOptions, target,
-              targetOptions, copyRequest.megabytesRewrittenPerCall()));
+              targetOptions, copyRequest.megabytesCopiedPerChunk()));
         }
       }, options().retryParams(), EXCEPTION_HANDLER);
       return new CopyWriter(options(), rewriteResponse);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
index 095937981b7f..defb1d35e3f4 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/BlobTest.java
@@ -121,7 +121,7 @@ public void testDelete() throws Exception {
 
   @Test
   public void testCopyToBucket() throws Exception {
-    BlobInfo target = BLOB_INFO.toBuilder().blobId(BlobId.of("bt", "n")).build();
+    BlobInfo target = BlobInfo.builder(BlobId.of("bt", "n")).build();
     CopyWriter copyWriter = createMock(CopyWriter.class);
     Capture capturedCopyRequest = Capture.newInstance();
     expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter);
@@ -136,7 +136,7 @@ public void testCopyToBucket() throws Exception {
 
   @Test
   public void testCopyTo() throws Exception {
-    BlobInfo target = BLOB_INFO.toBuilder().blobId(BlobId.of("bt", "nt")).build();
+    BlobInfo target = BlobInfo.builder(BlobId.of("bt", "nt")).build();
     CopyWriter copyWriter = createMock(CopyWriter.class);
     Capture capturedCopyRequest = Capture.newInstance();
     expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter);
@@ -153,7 +153,7 @@ public void testCopyTo() throws Exception {
   public void testCopyToBlobId() throws Exception {
     BlobId targetId = BlobId.of("bt", "nt");
     CopyWriter copyWriter = createMock(CopyWriter.class);
-    BlobInfo target = BLOB_INFO.toBuilder().blobId(targetId).build();
+    BlobInfo target = BLOB_INFO.builder(targetId).build();
     Capture capturedCopyRequest = Capture.newInstance();
     expect(storage.copy(capture(capturedCopyRequest))).andReturn(copyWriter);
     replay(storage);
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
index feba88d57118..0fcdb744c244 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/CopyWriterTest.java
@@ -86,11 +86,10 @@ public void testRewrite() {
     EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE);
     EasyMock.replay(storageRpcMock);
     copyWriter = new CopyWriter(options, RESPONSE);
-    copyWriter.copyChunk();
-    assertTrue(copyWriter.isDone());
     assertEquals(RESULT, copyWriter.result());
-    assertEquals(new Long(42L), copyWriter.totalBytesCopied());
-    assertEquals(new Long(42L), copyWriter.blobSize());
+    assertTrue(copyWriter.isDone());
+    assertEquals(42L, copyWriter.totalBytesCopied());
+    assertEquals(42L, copyWriter.blobSize());
   }
 
   @Test
@@ -99,16 +98,10 @@ public void testRewriteMultipleRequests() {
     EasyMock.expect(storageRpcMock.continueRewrite(RESPONSE)).andReturn(RESPONSE_DONE);
     EasyMock.replay(storageRpcMock);
     copyWriter = new CopyWriter(options, RESPONSE);
-    int loopCount = 0;
-    while (!copyWriter.isDone()) {
-      copyWriter.copyChunk();
-      loopCount++;
-    }
-    assertTrue(copyWriter.isDone());
     assertEquals(RESULT, copyWriter.result());
-    assertEquals(new Long(42L), copyWriter.totalBytesCopied());
-    assertEquals(new Long(42L), copyWriter.blobSize());
-    assertEquals(2, loopCount);
+    assertTrue(copyWriter.isDone());
+    assertEquals(42L, copyWriter.totalBytesCopied());
+    assertEquals(42L, copyWriter.blobSize());
   }
 
   @Test
@@ -119,15 +112,13 @@ public void testSaveAndRestore() throws IOException {
     copyWriter = new CopyWriter(options, RESPONSE);
     copyWriter.copyChunk();
     assertTrue(!copyWriter.isDone());
-    assertEquals(null, copyWriter.result());
-    assertEquals(new Long(21L), copyWriter.totalBytesCopied());
-    assertEquals(new Long(42L), copyWriter.blobSize());
+    assertEquals(21L, copyWriter.totalBytesCopied());
+    assertEquals(42L, copyWriter.blobSize());
     RestorableState rewriterState = copyWriter.capture();
     CopyWriter restoredRewriter = rewriterState.restore();
-    restoredRewriter.copyChunk();
-    assertTrue(restoredRewriter.isDone());
     assertEquals(RESULT, restoredRewriter.result());
-    assertEquals(new Long(42L), restoredRewriter.totalBytesCopied());
-    assertEquals(new Long(42L), restoredRewriter.blobSize());
+    assertTrue(restoredRewriter.isDone());
+    assertEquals(42L, restoredRewriter.totalBytesCopied());
+    assertEquals(42L, restoredRewriter.blobSize());
   }
 }
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index 612d4c37aa07..408c6e7ed6d6 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -319,13 +319,12 @@ public void testCopyBlob() {
     String sourceBlobName = "test-copy-blob-source";
     BlobId source = BlobId.of(BUCKET, sourceBlobName);
     assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
-    String targetBlobName = "test-rewrite-blob-target";
+    String targetBlobName = "test-copy-blob-target";
     BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build();
     Storage.CopyRequest req = Storage.CopyRequest.of(source, target);
-    CopyWriter rewriter = storage.copy(req);
-    rewriter.copyChunk();
-    assertTrue(rewriter.isDone());
-    assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName));
+    CopyWriter copyWriter = storage.copy(req);
+    assertEquals(copyWriter.result(), storage.get(BUCKET, targetBlobName));
+    assertTrue(copyWriter.isDone());
     assertTrue(storage.delete(BUCKET, sourceBlobName));
     assertTrue(storage.delete(BUCKET, targetBlobName));
   }
@@ -335,16 +334,15 @@ public void testCopyBlobUpdateMetadata() {
     String sourceBlobName = "test-copy-blob-update-metadata-source";
     BlobId source = BlobId.of(BUCKET, sourceBlobName);
     assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
-    String targetBlobName = "test-rewrite-blob-target";
+    String targetBlobName = "test-copy-blob-target";
     BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName)
         .contentType(CONTENT_TYPE)
         .metadata(ImmutableMap.of("k", "v"))
         .build();
     Storage.CopyRequest req = Storage.CopyRequest.of(source, target);
-    CopyWriter rewriter = storage.copy(req);
-    rewriter.copyChunk();
-    assertTrue(rewriter.isDone());
-    assertEquals(rewriter.result(), storage.get(BUCKET, targetBlobName));
+    CopyWriter copyWriter = storage.copy(req);
+    assertEquals(copyWriter.result(), storage.get(BUCKET, targetBlobName));
+    assertTrue(copyWriter.isDone());
     assertTrue(storage.delete(BUCKET, sourceBlobName));
     assertTrue(storage.delete(BUCKET, targetBlobName));
   }
@@ -354,7 +352,7 @@ public void testCopyBlobFail() {
     String sourceBlobName = "test-copy-blob-source-fail";
     BlobId source = BlobId.of(BUCKET, sourceBlobName);
     assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
-    String targetBlobName = "test-rewrite-blob-target-fail";
+    String targetBlobName = "test-copy-blob-target-fail";
     BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build();
     Storage.CopyRequest req = Storage.CopyRequest.builder()
         .source(source)
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
index 1a99741fa4d0..910d614c64d7 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java
@@ -600,9 +600,8 @@ public void testCopy() {
     EasyMock.replay(storageRpcMock);
     storage = options.service();
     CopyWriter writer = storage.copy(request);
-    assertNull(writer.result());
-    assertEquals(new Long(42L), writer.blobSize());
-    assertEquals(new Long(21L), writer.totalBytesCopied());
+    assertEquals(42L, writer.blobSize());
+    assertEquals(21L, writer.totalBytesCopied());
     assertTrue(!writer.isDone());
   }
 
@@ -622,9 +621,8 @@ public void testCopyWithOptions() {
     EasyMock.replay(storageRpcMock);
     storage = options.service();
     CopyWriter writer = storage.copy(request);
-    assertNull(writer.result());
-    assertEquals(new Long(42L), writer.blobSize());
-    assertEquals(new Long(21L), writer.totalBytesCopied());
+    assertEquals(42L, writer.blobSize());
+    assertEquals(21L, writer.totalBytesCopied());
     assertTrue(!writer.isDone());
   }
 
@@ -642,15 +640,13 @@ public void testCopyMultipleRequests() {
     EasyMock.replay(storageRpcMock);
     storage = options.service();
     CopyWriter writer = storage.copy(request);
-    assertNull(writer.result());
-    assertEquals(new Long(42L), writer.blobSize());
-    assertEquals(new Long(21L), writer.totalBytesCopied());
+    assertEquals(42L, writer.blobSize());
+    assertEquals(21L, writer.totalBytesCopied());
     assertTrue(!writer.isDone());
-    writer.copyChunk();
-    assertTrue(writer.isDone());
     assertEquals(BLOB_INFO1, writer.result());
-    assertEquals(new Long(42L), writer.totalBytesCopied());
-    assertEquals(new Long(42L), writer.blobSize());
+    assertTrue(writer.isDone());
+    assertEquals(42L, writer.totalBytesCopied());
+    assertEquals(42L, writer.blobSize());
   }
 
   @Test

From 23215a1547449f044b0148b7d37cc4ecb958b83e Mon Sep 17 00:00:00 2001
From: Marco Ziccardi 
Date: Thu, 29 Oct 2015 09:02:41 +0100
Subject: [PATCH 6/6] Better javadoc, check contentType and metadata in
 integrationt tests

---
 .../com/google/gcloud/storage/CopyWriter.java | 23 +++++++++++-------
 .../com/google/gcloud/storage/Storage.java    |  8 +++++++
 .../google/gcloud/storage/ITStorageTest.java  | 24 ++++++++++++++-----
 3 files changed, 41 insertions(+), 14 deletions(-)

diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java
index 592cd6e0289a..142f8d4b6de7 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/CopyWriter.java
@@ -32,10 +32,11 @@
 import java.util.concurrent.Callable;
 
 /**
- * Google Storage blob copy writer. This class holds the result of a copy request.
- * If source and destination blobs do not share the same location or storage class more than one
- * RPC request is needed to copy the blob otherwise one or more {@link #copyChunk()} calls are
- * necessary to complete the copy.
+ * Google Storage blob copy writer. This class holds the result of a copy request. If source and
+ * destination blobs share the same location and storage class the copy is completed in one RPC call
+ * otherwise one or more {@link #copyChunk} calls are necessary to complete the copy. In addition,
+ * {@link CopyWriter#result()} can be used to automatically complete the copy and return information
+ * on the newly created blob.
  *
  * @see Rewrite
  */
@@ -52,20 +53,26 @@ public class CopyWriter implements Restorable {
   }
 
   /**
-   * Returns the updated information for the just written blob. This method might block and issue
-   * several RPC requests to complete blob copy.
+   * Returns the updated information for the written blob. Calling this method when {@code isDone()}
+   * is {@code false} will block until all pending chunks are copied.
+   * 

+   * This method has the same effect of doing:
+   * 
    {@code while (!copyWriter.isDone()) {
+   *        copyWriter.copyChunk();
+   *    }}
+   * 

    *
    * @throws StorageException upon failure
    */
   public BlobInfo result() {
-    while(!isDone()) {
+    while (!isDone()) {
       copyChunk();
     }
     return BlobInfo.fromPb(rewriteResponse.result);
   }
 
   /**
-   * Size of the blob being copied.
+   * Returns the size of the blob being copied.
    */
   public long blobSize() {
     return rewriteResponse.blobSize;
diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
index 1c65ff82dfaa..98698107a205 100644
--- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
+++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java
@@ -839,6 +839,14 @@ public static Builder builder() {
    * Example usage of copy:
    * 
    {@code BlobInfo blob = service.copy(copyRequest).result();}
    * 

+   * To explicitly issue chunk copy requests use {@link CopyWriter#copyChunk()} instead:
+   * 
    {@code CopyWriter copyWriter = service.copy(copyRequest);
+   *    while (!copyWriter.isDone()) {
+   *        copyWriter.copyChunk();
+   *    }
+   *    BlobInfo blob = copyWriter.result();
+   * }
+   * 

    *
    * @return a {@link CopyWriter} object that can be used to get information on the newly created
    *     blob or to complete the copy if more than one RPC request is needed
diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
index 408c6e7ed6d6..8957ed2b8364 100644
--- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
+++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java
@@ -318,12 +318,20 @@ public void testComposeBlobFail() {
   public void testCopyBlob() {
     String sourceBlobName = "test-copy-blob-source";
     BlobId source = BlobId.of(BUCKET, sourceBlobName);
-    assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
+    ImmutableMap metadata = ImmutableMap.of("k", "v");
+    BlobInfo blob = BlobInfo.builder(source)
+        .contentType(CONTENT_TYPE)
+        .metadata(metadata)
+        .build();
+    assertNotNull(storage.create(blob, BLOB_BYTE_CONTENT));
     String targetBlobName = "test-copy-blob-target";
-    BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).contentType(CONTENT_TYPE).build();
+    BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName).build();
     Storage.CopyRequest req = Storage.CopyRequest.of(source, target);
     CopyWriter copyWriter = storage.copy(req);
-    assertEquals(copyWriter.result(), storage.get(BUCKET, targetBlobName));
+    assertEquals(BUCKET, copyWriter.result().bucket());
+    assertEquals(targetBlobName, copyWriter.result().name());
+    assertEquals(CONTENT_TYPE, copyWriter.result().contentType());
+    assertEquals(metadata, copyWriter.result().metadata());
     assertTrue(copyWriter.isDone());
     assertTrue(storage.delete(BUCKET, sourceBlobName));
     assertTrue(storage.delete(BUCKET, targetBlobName));
@@ -334,14 +342,18 @@ public void testCopyBlobUpdateMetadata() {
     String sourceBlobName = "test-copy-blob-update-metadata-source";
     BlobId source = BlobId.of(BUCKET, sourceBlobName);
     assertNotNull(storage.create(BlobInfo.builder(source).build(), BLOB_BYTE_CONTENT));
-    String targetBlobName = "test-copy-blob-target";
+    String targetBlobName = "test-copy-blob-update-metadata-target";
+    ImmutableMap metadata = ImmutableMap.of("k", "v");
     BlobInfo target = BlobInfo.builder(BUCKET, targetBlobName)
         .contentType(CONTENT_TYPE)
-        .metadata(ImmutableMap.of("k", "v"))
+        .metadata(metadata)
         .build();
     Storage.CopyRequest req = Storage.CopyRequest.of(source, target);
     CopyWriter copyWriter = storage.copy(req);
-    assertEquals(copyWriter.result(), storage.get(BUCKET, targetBlobName));
+    assertEquals(BUCKET, copyWriter.result().bucket());
+    assertEquals(targetBlobName, copyWriter.result().name());
+    assertEquals(CONTENT_TYPE, copyWriter.result().contentType());
+    assertEquals(metadata, copyWriter.result().metadata());
     assertTrue(copyWriter.isDone());
     assertTrue(storage.delete(BUCKET, sourceBlobName));
     assertTrue(storage.delete(BUCKET, targetBlobName));