diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java index e70e0f14370e..a4a817ead2df 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Blob.java @@ -24,6 +24,7 @@ import com.google.common.collect.Lists; import com.google.gcloud.spi.StorageRpc; import com.google.gcloud.storage.Storage.BlobTargetOption; +import com.google.gcloud.storage.Storage.BlobWriteOption; import com.google.gcloud.storage.Storage.CopyRequest; import com.google.gcloud.storage.Storage.SignUrlOption; @@ -269,12 +270,14 @@ public BlobReadChannel reader(BlobSourceOption... options) { } /** - * Returns a {@code BlobWriteChannel} object for writing to this blob. + * Returns a {@code BlobWriteChannel} object for writing to this blob. By default any md5 and + * crc32c values in the current blob are ignored unless requested via the + * {@code BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. * * @param options target blob options * @throws StorageException upon failure */ - public BlobWriteChannel writer(BlobTargetOption... options) { + public BlobWriteChannel writer(BlobWriteOption... options) { return storage.writer(info, options); } diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java index 790961a7a3d1..f9a1c00d4bec 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/Storage.java @@ -21,9 +21,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; import com.google.gcloud.Service; import com.google.gcloud.spi.StorageRpc; +import com.google.gcloud.spi.StorageRpc.Tuple; import java.io.InputStream; import java.io.Serializable; @@ -33,6 +35,7 @@ import java.util.LinkedHashSet; import java.util.LinkedList; import java.util.List; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; @@ -145,6 +148,105 @@ public static BlobTargetOption metagenerationMatch() { public static BlobTargetOption metagenerationNotMatch() { return new BlobTargetOption(StorageRpc.Option.IF_METAGENERATION_NOT_MATCH); } + + static Tuple convert(BlobInfo info, BlobWriteOption... options) { + BlobInfo.Builder infoBuilder = info.toBuilder().crc32c(null).md5(null); + List targetOptions = Lists.newArrayListWithCapacity(options.length); + for (BlobWriteOption option : options) { + switch (option.option) { + case IF_CRC32C_MATCH: + infoBuilder.crc32c(info.crc32c()); + break; + case IF_MD5_MATCH: + infoBuilder.md5(info.md5()); + break; + default: + targetOptions.add(option.toTargetOption()); + break; + } + } + return Tuple.of(infoBuilder.build(), + targetOptions.toArray(new BlobTargetOption[targetOptions.size()])); + } + } + + class BlobWriteOption implements Serializable { + + private static final long serialVersionUID = -3880421670966224580L; + + private final Option option; + private final Object value; + + enum Option { + PREDEFINED_ACL, IF_GENERATION_MATCH, IF_GENERATION_NOT_MATCH, IF_METAGENERATION_MATCH, + IF_METAGENERATION_NOT_MATCH, IF_MD5_MATCH, IF_CRC32C_MATCH; + + StorageRpc.Option toRpcOption() { + return StorageRpc.Option.valueOf(this.name()); + } + } + + BlobTargetOption toTargetOption() { + return new BlobTargetOption(this.option.toRpcOption(), this.value); + } + + private BlobWriteOption(Option option, Object value) { + this.option = option; + this.value = value; + } + + private BlobWriteOption(Option option) { + this(option, null); + } + + @Override + public int hashCode() { + return Objects.hash(option, value); + } + + @Override + public boolean equals(Object obj) { + if (obj == null) { + return false; + } + if (!(obj instanceof BlobWriteOption)) { + return false; + } + final BlobWriteOption other = (BlobWriteOption) obj; + return this.option == other.option && Objects.equals(this.value, other.value); + } + + public static BlobWriteOption predefinedAcl(PredefinedAcl acl) { + return new BlobWriteOption(Option.PREDEFINED_ACL, acl.entry()); + } + + public static BlobWriteOption doesNotExist() { + return new BlobWriteOption(Option.IF_GENERATION_MATCH, 0L); + } + + public static BlobWriteOption generationMatch() { + return new BlobWriteOption(Option.IF_GENERATION_MATCH); + } + + public static BlobWriteOption generationNotMatch() { + return new BlobWriteOption(Option.IF_GENERATION_NOT_MATCH); + } + + public static BlobWriteOption metagenerationMatch() { + return new BlobWriteOption(Option.IF_METAGENERATION_MATCH); + } + + public static BlobWriteOption metagenerationNotMatch() { + return new BlobWriteOption(Option.IF_METAGENERATION_NOT_MATCH); + } + + public static BlobWriteOption md5Match() { + return new BlobWriteOption(Option.IF_MD5_MATCH, true); + } + + public static BlobWriteOption crc32cMatch() { + return new BlobWriteOption(Option.IF_CRC32C_MATCH, true); + } } class BlobSourceOption extends Option { @@ -510,21 +612,25 @@ public static Builder builder() { /** * Create a new blob. Direct upload is used to upload {@code content}. For large content, - * {@link #writer} is recommended as it uses resumable upload. + * {@link #writer} is recommended as it uses resumable upload. MD5 and CRC32C hashes of + * {@code content} are computed and used for validating transferred data. * * @return a complete blob information. * @throws StorageException upon failure + * @see Hashes and ETags */ BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options); /** * Create a new blob. Direct upload is used to upload {@code content}. For large content, - * {@link #writer} is recommended as it uses resumable upload. + * {@link #writer} is recommended as it uses resumable upload. By default any md5 and crc32c + * values in the given {@code blobInfo} are ignored unless requested via the + * {@code BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. * * @return a complete blob information. * @throws StorageException upon failure */ - BlobInfo create(BlobInfo blobInfo, InputStream content, BlobTargetOption... options); + BlobInfo create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options); /** * Return the requested bucket or {@code null} if not found. @@ -679,11 +785,13 @@ public static Builder builder() { BlobReadChannel reader(BlobId blob, BlobSourceOption... options); /** - * Create a blob and return a channel for writing its content. + * Create a blob and return a channel for writing its content. By default any md5 and crc32c + * values in the given {@code blobInfo} are ignored unless requested via the + * {@code BlobWriteOption.md5Match} and {@code BlobWriteOption.crc32cMatch} options. * * @throws StorageException upon failure */ - BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options); + BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options); /** * Generates a signed URL for a blob. diff --git a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java index bb114eccd093..6edd2713fded 100644 --- a/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java +++ b/gcloud-java-storage/src/main/java/com/google/gcloud/storage/StorageImpl.java @@ -40,6 +40,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import com.google.common.hash.Hashing; import com.google.common.io.BaseEncoding; import com.google.common.primitives.Ints; import com.google.gcloud.AuthCredentials.ServiceAccountAuthCredentials; @@ -93,13 +94,14 @@ public RetryResult beforeEval(Exception exception) { static final ExceptionHandler EXCEPTION_HANDLER = ExceptionHandler.builder() .abortOn(RuntimeException.class).interceptor(EXCEPTION_HANDLER_INTERCEPTOR).build(); private static final byte[] EMPTY_BYTE_ARRAY = {}; + private static final String EMPTY_BYTE_ARRAY_MD5 = "1B2M2Y8AsgTpgAmY7PhCfg=="; + private static final String EMPTY_BYTE_ARRAY_CRC32C = "AAAAAA=="; private final StorageRpc storageRpc; StorageImpl(StorageOptions options) { super(options); storageRpc = options.storageRpc(); - // todo: configure timeouts - https://developers.google.com/api-client-library/java/google-api-java-client/errors // todo: provide rewrite - https://cloud.google.com/storage/docs/json_api/v1/objects/rewrite // todo: check if we need to expose https://cloud.google.com/storage/docs/json_api/v1/bucketAccessControls/insert vs using bucket update/patch } @@ -123,20 +125,33 @@ public com.google.api.services.storage.model.Bucket call() { @Override public BlobInfo create(BlobInfo blobInfo, BlobTargetOption... options) { - return create(blobInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options); + BlobInfo updatedInfo = blobInfo.toBuilder() + .md5(EMPTY_BYTE_ARRAY_MD5) + .crc32c(EMPTY_BYTE_ARRAY_CRC32C) + .build(); + return create(updatedInfo, new ByteArrayInputStream(EMPTY_BYTE_ARRAY), options); } @Override - public BlobInfo create(BlobInfo blobInfo, final byte[] content, BlobTargetOption... options) { - return create(blobInfo, - new ByteArrayInputStream(firstNonNull(content, EMPTY_BYTE_ARRAY)), options); + public BlobInfo create(BlobInfo blobInfo, byte[] content, BlobTargetOption... options) { + content = firstNonNull(content, EMPTY_BYTE_ARRAY); + BlobInfo updatedInfo = blobInfo.toBuilder() + .md5(BaseEncoding.base64().encode(Hashing.md5().hashBytes(content).asBytes())) + .crc32c(BaseEncoding.base64().encode( + Ints.toByteArray(Hashing.crc32c().hashBytes(content).asInt()))) + .build(); + return create(updatedInfo, new ByteArrayInputStream(content), options); } @Override - public BlobInfo create(BlobInfo blobInfo, final InputStream content, - BlobTargetOption... options) { - final StorageObject blobPb = blobInfo.toPb(); - final Map optionsMap = optionMap(blobInfo, options); + public BlobInfo create(BlobInfo blobInfo, InputStream content, BlobWriteOption... options) { + Tuple targetOptions = BlobTargetOption.convert(blobInfo, options); + return create(targetOptions.x(), content, targetOptions.y()); + } + + private BlobInfo create(BlobInfo info, final InputStream content, BlobTargetOption... options) { + final StorageObject blobPb = info.toPb(); + final Map optionsMap = optionMap(info, options); try { return BlobInfo.fromPb(runWithRetries(new Callable() { @Override @@ -544,7 +559,12 @@ public BlobReadChannel reader(BlobId blob, BlobSourceOption... options) { } @Override - public BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { + public BlobWriteChannel writer(BlobInfo blobInfo, BlobWriteOption... options) { + Tuple targetOptions = BlobTargetOption.convert(blobInfo, options); + return writer(targetOptions.x(), targetOptions.y()); + } + + private BlobWriteChannel writer(BlobInfo blobInfo, BlobTargetOption... options) { final Map optionsMap = optionMap(blobInfo, options); return new BlobWriteChannelImpl(options(), blobInfo, optionsMap); } diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java index 7469b0bb7fb8..2747444d1f27 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/ITStorageTest.java @@ -145,6 +145,22 @@ public void testCreateBlobFail() { assertTrue(storage.delete(bucket, blobName)); } + @Test + public void testCreateBlobMd5Fail() throws UnsupportedEncodingException { + String blobName = "test-create-blob-md5-fail"; + BlobInfo blob = BlobInfo.builder(bucket, blobName) + .contentType(CONTENT_TYPE) + .md5("O1R4G1HJSDUISJjoIYmVhQ==") + .build(); + ByteArrayInputStream stream = new ByteArrayInputStream(BLOB_STRING_CONTENT.getBytes(UTF_8)); + try { + storage.create(blob, stream, Storage.BlobWriteOption.md5Match()); + fail("StorageException was expected"); + } catch (StorageException ex) { + // expected + } + } + @Test public void testUpdateBlob() { String blobName = "test-update-blob"; @@ -449,7 +465,7 @@ public void testWriteChannelFail() throws UnsupportedEncodingException, IOExcept BlobInfo blob = BlobInfo.builder(bucket, blobName).generation(-1L).build(); try { try (BlobWriteChannel writer = - storage.writer(blob, Storage.BlobTargetOption.generationMatch())) { + storage.writer(blob, Storage.BlobWriteOption.generationMatch())) { writer.write(ByteBuffer.allocate(42)); } fail("StorageException was expected"); @@ -458,6 +474,20 @@ public void testWriteChannelFail() throws UnsupportedEncodingException, IOExcept } } + @Test + public void testWriteChannelExistingBlob() throws UnsupportedEncodingException, IOException { + String blobName = "test-write-channel-existing-blob"; + BlobInfo blob = BlobInfo.builder(bucket, blobName).build(); + BlobInfo remoteBlob = storage.create(blob); + byte[] stringBytes; + try (BlobWriteChannel writer = storage.writer(remoteBlob)) { + stringBytes = BLOB_STRING_CONTENT.getBytes(UTF_8); + writer.write(ByteBuffer.wrap(stringBytes)); + } + assertArrayEquals(stringBytes, storage.readAllBytes(blob.blobId())); + assertTrue(storage.delete(bucket, blobName)); + } + @Test public void testGetSignedUrl() throws IOException { String blobName = "test-get-signed-url-blob"; diff --git a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java index f7c1b539375f..b3a6fe36859e 100644 --- a/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java +++ b/gcloud-java-storage/src/test/java/com/google/gcloud/storage/StorageImplTest.java @@ -76,6 +76,8 @@ public class StorageImplTest { private static final String BLOB_NAME2 = "n2"; private static final String BLOB_NAME3 = "n3"; private static final byte[] BLOB_CONTENT = {0xD, 0xE, 0xA, 0xD}; + private static final String CONTENT_MD5 = "O1R4G1HJSDUISJjoIYmVhQ=="; + private static final String CONTENT_CRC32C = "9N3EPQ=="; private static final int DEFAULT_CHUNK_SIZE = 2 * 1024 * 1024; // BucketInfo objects @@ -121,6 +123,18 @@ public class StorageImplTest { StorageRpc.Option.IF_GENERATION_MATCH, BLOB_INFO1.generation(), StorageRpc.Option.IF_METAGENERATION_MATCH, BLOB_INFO1.metageneration()); + // Blob write options (create, writer) + private static final Storage.BlobWriteOption BLOB_WRITE_METAGENERATION = + Storage.BlobWriteOption.metagenerationMatch(); + private static final Storage.BlobWriteOption BLOB_WRITE_NOT_EXIST = + Storage.BlobWriteOption.doesNotExist(); + private static final Storage.BlobWriteOption BLOB_WRITE_PREDEFINED_ACL = + Storage.BlobWriteOption.predefinedAcl(Storage.PredefinedAcl.PRIVATE); + private static final Storage.BlobWriteOption BLOB_WRITE_MD5_HASH = + Storage.BlobWriteOption.md5Match(); + private static final Storage.BlobWriteOption BLOB_WRITE_CRC2C = + Storage.BlobWriteOption.crc32cMatch(); + // Bucket source options private static final Storage.BucketSourceOption BUCKET_SOURCE_METAGENERATION = Storage.BucketSourceOption.metagenerationMatch(BUCKET_INFO1.metageneration()); @@ -250,10 +264,10 @@ public void testCreateBlob() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); Capture capturedStream = Capture.newInstance(); - EasyMock - .expect( - storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream), - EasyMock.eq(EMPTY_RPC_OPTIONS))) + EasyMock.expect(storageRpcMock.create( + EasyMock.eq(BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build().toPb()), + EasyMock.capture(capturedStream), + EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); @@ -271,10 +285,14 @@ public void testCreateEmptyBlob() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); Capture capturedStream = Capture.newInstance(); - EasyMock - .expect( - storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream), - EasyMock.eq(EMPTY_RPC_OPTIONS))) + EasyMock.expect(storageRpcMock.create( + EasyMock.eq(BLOB_INFO1.toBuilder() + .md5("1B2M2Y8AsgTpgAmY7PhCfg==") + .crc32c("AAAAAA==") + .build() + .toPb()), + EasyMock.capture(capturedStream), + EasyMock.eq(EMPTY_RPC_OPTIONS))) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); @@ -290,9 +308,14 @@ public void testCreateBlobWithOptions() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); Capture capturedStream = Capture.newInstance(); - EasyMock.expect( - storageRpcMock.create(EasyMock.eq(BLOB_INFO1.toPb()), EasyMock.capture(capturedStream), - EasyMock.eq(BLOB_TARGET_OPTIONS_CREATE))) + EasyMock.expect(storageRpcMock.create( + EasyMock.eq(BLOB_INFO1.toBuilder() + .md5(CONTENT_MD5) + .crc32c(CONTENT_CRC32C) + .build() + .toPb()), + EasyMock.capture(capturedStream), + EasyMock.eq(BLOB_TARGET_OPTIONS_CREATE))) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); @@ -312,11 +335,14 @@ public void testCreateBlobFromStream() throws IOException { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock); EasyMock.expect(optionsMock.retryParams()).andReturn(RetryParams.noRetries()); ByteArrayInputStream fileStream = new ByteArrayInputStream(BLOB_CONTENT); - EasyMock.expect(storageRpcMock.create(BLOB_INFO1.toPb(), fileStream, EMPTY_RPC_OPTIONS)) + BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); + BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build(); + BlobInfo infoWithoutHashes = infoBuilder.md5(null).crc32c(null).build(); + EasyMock.expect(storageRpcMock.create(infoWithoutHashes.toPb(), fileStream, EMPTY_RPC_OPTIONS)) .andReturn(BLOB_INFO1.toPb()); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); - BlobInfo blob = storage.create(BLOB_INFO1, fileStream); + BlobInfo blob = storage.create(infoWithHashes, fileStream); assertEquals(BLOB_INFO1, blob); } @@ -775,11 +801,14 @@ public void testReaderWithOptions() throws IOException { @Test public void testWriter() { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock).times(2); - EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), EMPTY_RPC_OPTIONS)) + BlobInfo.Builder infoBuilder = BLOB_INFO1.toBuilder(); + BlobInfo infoWithHashes = infoBuilder.md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build(); + BlobInfo infoWithoutHashes = infoBuilder.md5(null).crc32c(null).build(); + EasyMock.expect(storageRpcMock.open(infoWithoutHashes.toPb(), EMPTY_RPC_OPTIONS)) .andReturn("upload-id"); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); - BlobWriteChannel channel = storage.writer(BLOB_INFO1); + BlobWriteChannel channel = storage.writer(infoWithHashes); assertNotNull(channel); assertTrue(channel.isOpen()); } @@ -787,12 +816,13 @@ public void testWriter() { @Test public void testWriterWithOptions() { EasyMock.expect(optionsMock.storageRpc()).andReturn(storageRpcMock).times(2); - EasyMock.expect(storageRpcMock.open(BLOB_INFO1.toPb(), BLOB_TARGET_OPTIONS_CREATE)) + BlobInfo info = BLOB_INFO1.toBuilder().md5(CONTENT_MD5).crc32c(CONTENT_CRC32C).build(); + EasyMock.expect(storageRpcMock.open(info.toPb(), BLOB_TARGET_OPTIONS_CREATE)) .andReturn("upload-id"); EasyMock.replay(optionsMock, storageRpcMock); storage = StorageFactory.instance().get(optionsMock); - BlobWriteChannel channel = storage.writer(BLOB_INFO1, BLOB_TARGET_METAGENERATION, - BLOB_TARGET_NOT_EXIST, BLOB_TARGET_PREDEFINED_ACL); + BlobWriteChannel channel = storage.writer(info, BLOB_WRITE_METAGENERATION, BLOB_WRITE_NOT_EXIST, + BLOB_WRITE_PREDEFINED_ACL, BLOB_WRITE_CRC2C, BLOB_WRITE_MD5_HASH); assertNotNull(channel); assertTrue(channel.isOpen()); }