From 9fd0c2a2607d39256edcdc48449a9a7016e44bed Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 15 Apr 2024 12:30:23 -0700 Subject: [PATCH 1/6] chunked wip --- .../AllowDivideAndConquerDownload.java | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java new file mode 100644 index 0000000000..adc9e2a527 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java @@ -0,0 +1,52 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.example.storage.transfermanager; + +import com.google.cloud.storage.BlobInfo; +import com.google.cloud.storage.transfermanager.DownloadResult; +import com.google.cloud.storage.transfermanager.ParallelDownloadConfig; +import com.google.cloud.storage.transfermanager.TransferManager; +import com.google.cloud.storage.transfermanager.TransferManagerConfig; +import java.nio.file.Path; +import java.util.List; + +// [START storage_transfer_manager_download_chunks_concurrently] +class AllowDivideAndConquerDownload { + + public static void allowDivideAndConquerDownload(List blobs, + String bucketName, Path destinationDirectory) { + TransferManager transferManager = TransferManagerConfig.newBuilder() + .setAllowDivideAndConquer(true) + .build() + .getService(); + ParallelDownloadConfig parallelDownloadConfig = ParallelDownloadConfig.newBuilder() + .setBucketName(bucketName) + .setDownloadDirectory(destinationDirectory) + .build(); + List results = transferManager + .downloadBlobs(blobs, parallelDownloadConfig) + .getDownloadResults(); + + for (DownloadResult result : results) { + System.out.println("Download of " + result.getInput().getName() + + " completed with status " + + result.getStatus()); + } + + } +} From c3a6a399808c8b5113520f55b8cb6e5abee52c62 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 29 Apr 2024 10:13:29 -0700 Subject: [PATCH 2/6] docs: Add in Transfer Manager chunked upload/download samples --- .../AllowDivideAndConquerDownload.java | 2 +- .../AllowParallelCompositeUpload.java | 51 +++++++++++++++++++ .../ITTransferManagerSamples.java | 28 ++++++++++ 3 files changed, 80 insertions(+), 1 deletion(-) create mode 100644 samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java index adc9e2a527..5d074a361f 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java @@ -28,7 +28,7 @@ // [START storage_transfer_manager_download_chunks_concurrently] class AllowDivideAndConquerDownload { - public static void allowDivideAndConquerDownload(List blobs, + public static void divideAndConquerDownloadAllowed(List blobs, String bucketName, Path destinationDirectory) { TransferManager transferManager = TransferManagerConfig.newBuilder() .setAllowDivideAndConquer(true) diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java new file mode 100644 index 0000000000..59e8ed36a6 --- /dev/null +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java @@ -0,0 +1,51 @@ +/* + * Copyright 2024 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package com.example.storage.transfermanager; + +import com.google.cloud.storage.transfermanager.ParallelUploadConfig; +import com.google.cloud.storage.transfermanager.TransferManager; +import com.google.cloud.storage.transfermanager.TransferManagerConfig; +import com.google.cloud.storage.transfermanager.UploadResult; +import java.io.IOException; +import java.nio.file.Path; +import java.util.List; + +// [START storage_transfer_manager_upload_chunks_concurrently] +class AllowParallelCompositeUpload { + + public static void parallelCompositeUploadAllowed(String bucketName, List files) + throws IOException { + TransferManager transferManager = TransferManagerConfig + .newBuilder() + .setAllowDivideAndConquer(true) + .build() + .getService(); + ParallelUploadConfig parallelUploadConfig = + ParallelUploadConfig.newBuilder().setBucketName(bucketName).build(); + List results = + transferManager.uploadFiles(files, parallelUploadConfig).getUploadResults(); + for (UploadResult result : results) { + System.out.println( + "Upload for " + + result.getInput().getName() + + " completed with status " + + result.getStatus()); + } + } +} +// [END storage_transfer_manager_upload_chunks_concurrently] diff --git a/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java b/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java index 4a9a3d0440..48a59f26b3 100644 --- a/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java +++ b/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java @@ -25,11 +25,19 @@ import com.google.cloud.storage.testing.RemoteStorageHelper; import com.google.cloud.testing.junit4.StdOutCaptureRule; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; +import java.nio.file.Files; +import java.nio.file.OpenOption; import java.nio.file.Path; +import java.nio.file.StandardOpenOption; import java.util.Arrays; +import java.util.Collections; import java.util.List; +import java.util.Set; import org.junit.BeforeClass; import org.junit.Rule; import org.junit.Test; @@ -39,6 +47,7 @@ public class ITTransferManagerSamples { private static final String BUCKET = RemoteStorageHelper.generateBucketName(); private static Storage storage; private static List blobs; + private static List bigBlob; private static final String PROJECT_ID = System.getenv("GOOGLE_CLOUD_PROJECT"); @Rule public final StdOutCaptureRule stdOutCaptureRule = new StdOutCaptureRule(); @Rule public final TemporaryFolder tmp = new TemporaryFolder(); @@ -112,4 +121,23 @@ public void downloadFiles() { assertThat(snippetOutput.contains("blob2")).isTrue(); assertThat(snippetOutput.contains("blob3")).isTrue(); } + + @Test + public void uploadAllowPCU() throws IOException { + File tmpFile = tmpDirectory.newFile("fileDirUpload.txt"); + AllowParallelCompositeUpload + .parallelCompositeUploadAllowed(BUCKET, Collections.singletonList(tmpFile.toPath())); + String snippetOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String(); + assertThat(snippetOutput.contains("fileDirUpload.txt")).isTrue(); + } + + @Test + public void downloadAllowDivideAndConquer() { + AllowDivideAndConquerDownload + .divideAndConquerDownloadAllowed(blobs, BUCKET,tmp.getRoot().toPath()); + String snippetOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String(); + assertThat(snippetOutput.contains("blob1")).isTrue(); + assertThat(snippetOutput.contains("blob2")).isTrue(); + assertThat(snippetOutput.contains("blob3")).isTrue(); + } } From 8c2946def61b2626b6a981ef7a0070cfbf761e41 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 29 Apr 2024 10:15:20 -0700 Subject: [PATCH 3/6] adding in regiontags --- .../transfermanager/AllowDivideAndConquerDownload.java | 4 ++-- .../storage/transfermanager/AllowParallelCompositeUpload.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java index 5d074a361f..2e8d973657 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java @@ -17,6 +17,7 @@ package com.example.storage.transfermanager; +// [START storage_transfer_manager_download_chunks_concurrently] import com.google.cloud.storage.BlobInfo; import com.google.cloud.storage.transfermanager.DownloadResult; import com.google.cloud.storage.transfermanager.ParallelDownloadConfig; @@ -24,8 +25,6 @@ import com.google.cloud.storage.transfermanager.TransferManagerConfig; import java.nio.file.Path; import java.util.List; - -// [START storage_transfer_manager_download_chunks_concurrently] class AllowDivideAndConquerDownload { public static void divideAndConquerDownloadAllowed(List blobs, @@ -50,3 +49,4 @@ public static void divideAndConquerDownloadAllowed(List blobs, } } +// [END storage_transfer_manager_download_chunks_concurrently] diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java index 59e8ed36a6..d92e8bf33c 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java @@ -17,6 +17,7 @@ package com.example.storage.transfermanager; +// [START storage_transfer_manager_upload_chunks_concurrently] import com.google.cloud.storage.transfermanager.ParallelUploadConfig; import com.google.cloud.storage.transfermanager.TransferManager; import com.google.cloud.storage.transfermanager.TransferManagerConfig; @@ -25,7 +26,6 @@ import java.nio.file.Path; import java.util.List; -// [START storage_transfer_manager_upload_chunks_concurrently] class AllowParallelCompositeUpload { public static void parallelCompositeUploadAllowed(String bucketName, List files) From 754648fbae0f59dec58e1cd374ceac775879e367 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Mon, 29 Apr 2024 10:18:53 -0700 Subject: [PATCH 4/6] lint --- .../storage/transfermanager/AllowDivideAndConquerDownload.java | 1 + .../storage/transfermanager/ITTransferManagerSamples.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java index 2e8d973657..dd7d84901d 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java @@ -25,6 +25,7 @@ import com.google.cloud.storage.transfermanager.TransferManagerConfig; import java.nio.file.Path; import java.util.List; + class AllowDivideAndConquerDownload { public static void divideAndConquerDownloadAllowed(List blobs, diff --git a/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java b/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java index 48a59f26b3..efc41086fd 100644 --- a/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java +++ b/samples/snippets/src/test/java/com/example/storage/transfermanager/ITTransferManagerSamples.java @@ -134,7 +134,7 @@ public void uploadAllowPCU() throws IOException { @Test public void downloadAllowDivideAndConquer() { AllowDivideAndConquerDownload - .divideAndConquerDownloadAllowed(blobs, BUCKET,tmp.getRoot().toPath()); + .divideAndConquerDownloadAllowed(blobs, BUCKET, tmp.getRoot().toPath()); String snippetOutput = stdOutCaptureRule.getCapturedOutputAsUtf8String(); assertThat(snippetOutput.contains("blob1")).isTrue(); assertThat(snippetOutput.contains("blob2")).isTrue(); From da63af3ba7783f7a7799ff538839e95704415293 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Tue, 30 Apr 2024 09:35:13 -0700 Subject: [PATCH 5/6] fix pcu config --- .../storage/transfermanager/AllowParallelCompositeUpload.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java index d92e8bf33c..e1ac51347b 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowParallelCompositeUpload.java @@ -32,7 +32,7 @@ public static void parallelCompositeUploadAllowed(String bucketName, List throws IOException { TransferManager transferManager = TransferManagerConfig .newBuilder() - .setAllowDivideAndConquer(true) + .setAllowParallelCompositeUpload(true) .build() .getService(); ParallelUploadConfig parallelUploadConfig = From abff6def593ff56a75301f4edbbce9470a613c37 Mon Sep 17 00:00:00 2001 From: Sydney Munro Date: Wed, 1 May 2024 15:05:23 -0700 Subject: [PATCH 6/6] fix compliation issue --- .../storage/transfermanager/AllowDivideAndConquerDownload.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java index dd7d84901d..fc1921a28f 100644 --- a/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java +++ b/samples/snippets/src/main/java/com/example/storage/transfermanager/AllowDivideAndConquerDownload.java @@ -31,7 +31,7 @@ class AllowDivideAndConquerDownload { public static void divideAndConquerDownloadAllowed(List blobs, String bucketName, Path destinationDirectory) { TransferManager transferManager = TransferManagerConfig.newBuilder() - .setAllowDivideAndConquer(true) + .setAllowDivideAndConquerDownload(true) .build() .getService(); ParallelDownloadConfig parallelDownloadConfig = ParallelDownloadConfig.newBuilder()