Allow GCS backend to upload more than 32 parts

This recursively combines up to 32 sets of 32 parts, allowing 1024 part multipart uploads. Fixes #330.
gaul · Sep 22, 2020 · 66b20fb · 66b20fb
1 parent 2b920c4
commit 66b20fb
Showing 1 changed file with 35 additions and 1 deletion.
diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java
@@ -2228,7 +2228,6 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
         for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
             builder.put(part.partNumber(), part);
         }
-        ImmutableMap<Integer, MultipartPart> partsByListing = builder.build();
 
         final List<MultipartPart> parts = new ArrayList<>();
         String blobStoreType = getBlobStoreType(blobStore);
@@ -2237,6 +2236,28 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
             for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
                 parts.add(part);
             }
+        } else if (blobStoreType.equals("google-cloud-storage")) {
+            // GCS only supports 32 parts but we can support up to 1024 by
+            // recursively combining objects.
+            for (int partNumber = 1;; ++partNumber) {
+                MultipartUpload mpu2 = MultipartUpload.create(
+                        containerName,
+                        String.format("%s_%08d", mpu.id(), partNumber),
+                        String.format("%s_%08d", mpu.id(), partNumber),
+                        new MutableBlobMetadataImpl(), new PutOptions());
+                List<MultipartPart> subParts = blobStore.listMultipartUpload(
+                        mpu2);
+                if (subParts.isEmpty()) {
+                    break;
+                }
+                long partSize = 0;
+                for (MultipartPart part : subParts) {
+                    partSize += part.partSize();
+                }
+                String eTag = blobStore.completeMultipartUpload(mpu2, subParts);
+                parts.add(MultipartPart.create(
+                        partNumber, partSize, eTag, /*lastModified=*/ null));
+            }
         } else {
             CompleteMultipartUploadRequest cmu;
             try {
@@ -2253,6 +2274,9 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
                     requestParts.put(part.partNumber, part.eTag);
                 }
             }
+
+            ImmutableMap<Integer, MultipartPart> partsByListing =
+                    builder.build();
             for (Iterator<Map.Entry<Integer, String>> it =
                     requestParts.entrySet().iterator(); it.hasNext();) {
                 Map.Entry<Integer, String> entry = it.next();
@@ -2367,6 +2391,7 @@ blobName, uploadId, createFakeBlobMetadata(blobStore),
         response.sendError(HttpServletResponse.SC_NO_CONTENT);
     }
 
+    // TODO: how to handle?
     private void handleListParts(HttpServletRequest request,
             HttpServletResponse response, BlobStore blobStore,
             String containerName, String blobName, String uploadId)
@@ -2715,6 +2740,15 @@ private static void handleUploadPart(HttpServletRequest request,
                             "ArgumentValue", partNumberString));
         }
 
+        // GCS only supports 32 parts so partition MPU into 32-part chunks.
+        String blobStoreType = getBlobStoreType(blobStore);
+        if (blobStoreType.equals("google-cloud-storage")) {
+            // fix up 1-based part numbers
+            uploadId = String.format(
+                    "%s_%08d", uploadId, ((partNumber - 1) / 32) + 1);
+            partNumber = ((partNumber - 1) % 32) + 1;
+        }
+
         // TODO: how to reconstruct original mpu?
         BlobMetadata blobMetadata;
         if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(