diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java index 738bbd10d0..6fa31d71c8 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/PackagePrivateMethodWorkarounds.java @@ -86,17 +86,6 @@ public static Function> maybeGetBlobInfoFunctio }; } - public static Function> maybeGetBlobInfoFunction() { - return writeChannel -> { - Optional so = maybeGetStorageObjectFunction().apply(writeChannel); - if (so.isPresent()) { - return Optional.of(Conversions.apiary().blobInfo().decode(so.get())); - } else { - return Optional.empty(); - } - }; - } - public static ApiFuture getBlobInfoFromReadChannelFunction(ReadChannel c) { if (c instanceof StorageReadChannel) { StorageReadChannel src = (StorageReadChannel) c; diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java index cb008c054b..56a1e74d57 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DirectDownloadCallable.java @@ -17,12 +17,11 @@ package com.google.cloud.storage.transfermanager; import com.google.cloud.ReadChannel; +import com.google.cloud.storage.BlobId; import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.Storage; import com.google.cloud.storage.Storage.BlobSourceOption; -import com.google.cloud.storage.StorageException; import com.google.common.io.ByteStreams; -import java.io.IOException; import java.nio.channels.FileChannel; import java.nio.file.Path; import java.nio.file.StandardOpenOption; @@ -50,16 +49,26 @@ final class DirectDownloadCallable implements Callable { @Override public DownloadResult call() { Path path = TransferManagerUtils.createDestPath(parallelDownloadConfig, originalBlob); - try (ReadChannel rc = storage.reader(originalBlob.getBlobId(), opts)) { + long bytesCopied = -1L; + try (ReadChannel rc = + storage.reader( + BlobId.of(parallelDownloadConfig.getBucketName(), originalBlob.getName()), opts)) { FileChannel wc = FileChannel.open( path, StandardOpenOption.WRITE, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING); - ByteStreams.copy(rc, wc); - } catch (IOException e) { - throw new StorageException(e); + bytesCopied = ByteStreams.copy(rc, wc); + } catch (Exception e) { + if (bytesCopied == -1) { + return DownloadResult.newBuilder(originalBlob, TransferStatus.FAILED_TO_START) + .setException(e) + .build(); + } + return DownloadResult.newBuilder(originalBlob, TransferStatus.FAILED_TO_FINISH) + .setException(e) + .build(); } DownloadResult result = DownloadResult.newBuilder(originalBlob, TransferStatus.SUCCESS) diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DownloadResult.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DownloadResult.java index 143154d7fc..12feafce02 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DownloadResult.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/DownloadResult.java @@ -20,7 +20,6 @@ import static com.google.common.base.Preconditions.checkState; import com.google.cloud.storage.BlobInfo; -import com.google.cloud.storage.StorageException; import com.google.common.base.MoreObjects; import java.nio.file.Path; import java.util.Comparator; @@ -35,13 +34,13 @@ public final class DownloadResult { @NonNull private final BlobInfo input; @MonotonicNonNull private final Path outputDestination; @NonNull private final TransferStatus status; - @MonotonicNonNull private final StorageException exception; + @MonotonicNonNull private final Exception exception; private DownloadResult( @NonNull BlobInfo input, Path outputDestination, @NonNull TransferStatus status, - StorageException exception) { + Exception exception) { this.input = input; this.outputDestination = outputDestination; this.status = status; @@ -64,7 +63,7 @@ private DownloadResult( return status; } - public @NonNull StorageException getException() { + public @NonNull Exception getException() { checkState( status == TransferStatus.FAILED_TO_FINISH || status == TransferStatus.FAILED_TO_START, "getException() is only valid when an unexpected error has occurred but status was %s", @@ -111,7 +110,7 @@ public static final class Builder { private @NonNull BlobInfo input; private @MonotonicNonNull Path outputDestination; private @NonNull TransferStatus status; - private @MonotonicNonNull StorageException exception; + private @MonotonicNonNull Exception exception; private Builder(@NonNull BlobInfo input, @NonNull TransferStatus status) { this.input = input; @@ -133,7 +132,7 @@ public Builder setStatus(@NonNull TransferStatus status) { return this; } - public Builder setException(@NonNull StorageException exception) { + public Builder setException(@NonNull Exception exception) { this.exception = exception; return this; } diff --git a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/UploadCallable.java b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/UploadCallable.java index deebc5b92c..7aca60342b 100644 --- a/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/UploadCallable.java +++ b/google-cloud-storage/src/main/java/com/google/cloud/storage/transfermanager/UploadCallable.java @@ -61,7 +61,7 @@ public UploadResult call() throws Exception { } private UploadResult uploadWithoutChunking() { - long bytesCopied = 0L; + long bytesCopied = -1L; try { Optional newBlob; try (FileChannel r = FileChannel.open(sourceFile, StandardOpenOption.READ); @@ -74,7 +74,7 @@ private UploadResult uploadWithoutChunking() { .setUploadedBlob(newBlob.get()) .build(); } catch (Exception e) { - if (bytesCopied == 0) { + if (bytesCopied == -1) { return UploadResult.newBuilder(originalBlob, TransferStatus.FAILED_TO_START) .setException(e) .build(); diff --git a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java index ab3f2f4964..21ed72ed01 100644 --- a/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java +++ b/google-cloud-storage/src/test/java/com/google/cloud/storage/it/ITTransferManagerTest.java @@ -253,6 +253,53 @@ public void uploadNonexistentFile() throws Exception { } } + @Test + public void downloadNonexistentBucket() throws Exception { + TransferManagerConfig config = + TransferManagerConfigTestingInstances.defaults(storage.getOptions()); + try (TransferManager transferManager = config.getService()) { + String bucketName = bucket.getName() + "-does-not-exist"; + ParallelDownloadConfig parallelDownloadConfig = + ParallelDownloadConfig.newBuilder() + .setBucketName(bucketName) + .setDownloadDirectory(baseDir) + .build(); + DownloadJob job = transferManager.downloadBlobs(blobs, parallelDownloadConfig); + List downloadResults = ApiFutures.allAsList(job.getDownloadResults()).get(); + List failedToStart = + downloadResults.stream() + .filter(x -> x.getStatus() == TransferStatus.FAILED_TO_START) + .collect(Collectors.toList()); + assertThat(failedToStart).hasSize(3); + } + } + + @Test + public void downloadBlobsChunkedFail() throws Exception { + TransferManagerConfig config = + TransferManagerConfigTestingInstances.defaults(storage.getOptions()) + .toBuilder() + .setAllowDivideAndConquer(true) + .setPerWorkerBufferSize(128 * 1024) + .build(); + try (TransferManager transferManager = config.getService()) { + String bucketName = bucket.getName() + "-does-not-exist"; + ParallelDownloadConfig parallelDownloadConfig = + ParallelDownloadConfig.newBuilder() + .setBucketName(bucketName) + .setDownloadDirectory(baseDir) + .build(); + DownloadJob job = transferManager.downloadBlobs(blobs, parallelDownloadConfig); + List downloadResults = ApiFutures.allAsList(job.getDownloadResults()).get(); + assertThat(downloadResults).hasSize(3); + List failedToStart = + downloadResults.stream() + .filter(x -> x.getStatus() == TransferStatus.FAILED_TO_START) + .collect(Collectors.toList()); + assertThat(failedToStart).hasSize(3); + } + } + private void cleanUpFiles(List results) throws IOException { // Cleanup downloaded blobs and the parent directory for (DownloadResult res : results) {