From 0cf5205af174cff84e8bceb1bd9b7172df3dadf0 Mon Sep 17 00:00:00 2001 From: sinhaashish Date: Mon, 15 Jun 2020 13:35:27 +0530 Subject: [PATCH] Address review comments --- api/src/main/java/io/minio/BaseArgs.java | 4 +- .../main/java/io/minio/ComposeObjectArgs.java | 14 +- api/src/main/java/io/minio/ComposeSource.java | 223 +++++----- .../main/java/io/minio/ComposeSourceArgs.java | 266 ------------ api/src/main/java/io/minio/MinioClient.java | 411 ++++-------------- docs/API.md | 46 +- examples/ComposeObject.java | 30 +- examples/ComposeObjectEncrypted.java | 20 +- functional/FunctionalTest.java | 74 ++-- 9 files changed, 300 insertions(+), 788 deletions(-) delete mode 100644 api/src/main/java/io/minio/ComposeSourceArgs.java diff --git a/api/src/main/java/io/minio/BaseArgs.java b/api/src/main/java/io/minio/BaseArgs.java index bb521a5f2..1c6c070e5 100644 --- a/api/src/main/java/io/minio/BaseArgs.java +++ b/api/src/main/java/io/minio/BaseArgs.java @@ -65,8 +65,8 @@ protected void validateNullOrNotEmptyString(String arg, String argName) { } } - protected void validateNullOrEmpty(Long arg, String argName) { - if (arg != null && arg < 0) { + protected void validateNullOrPositive(Number arg, String argName) { + if (arg != null && arg.longValue() < 0) { throw new IllegalArgumentException(argName + " cannot be non-negative."); } } diff --git a/api/src/main/java/io/minio/ComposeObjectArgs.java b/api/src/main/java/io/minio/ComposeObjectArgs.java index 9f53b3966..588bc181d 100644 --- a/api/src/main/java/io/minio/ComposeObjectArgs.java +++ b/api/src/main/java/io/minio/ComposeObjectArgs.java @@ -19,9 +19,9 @@ import java.util.List; public class ComposeObjectArgs extends ObjectWriteArgs { - List sources; + List sources; - public List sources() { + public List sources() { return sources; } @@ -30,13 +30,19 @@ public static Builder builder() { } public static final class Builder extends ObjectWriteArgs.Builder { - public Builder sources(List sources) { + @Override + protected void validate(ComposeObjectArgs args) { + super.validate(args); + validateSources(args.sources); + } + + public Builder sources(List sources) { validateSources(sources); operations.add(args -> args.sources = sources); return this; } - private void validateSources(List sources) { + private void validateSources(List sources) { if (sources.isEmpty()) { throw new IllegalArgumentException("compose sources cannot be empty"); } diff --git a/api/src/main/java/io/minio/ComposeSource.java b/api/src/main/java/io/minio/ComposeSource.java index 2ff5c9b67..bf5a46348 100644 --- a/api/src/main/java/io/minio/ComposeSource.java +++ b/api/src/main/java/io/minio/ComposeSource.java @@ -17,105 +17,108 @@ package io.minio; -import java.util.Collections; -import java.util.Map; -import java.util.TreeMap; +import com.google.common.collect.HashMultimap; +import com.google.common.collect.Multimap; +import com.google.common.collect.Multimaps; +import java.time.ZonedDateTime; /** Source information to compose object. */ -public class ComposeSource { - private String bucketName; - private String objectName; +public class ComposeSource extends ObjectVersionArgs { private Long offset; private Long length; - private Map headerMap; - private CopyConditions copyConditions; - private ServerSideEncryptionCustomerKey ssec; private long objectSize; - private Map headers; + private String matchETag; + private String notMatchETag; + private ZonedDateTime modifiedSince; + private ZonedDateTime unmodifiedSince; + private ServerSideEncryptionCustomerKey ssec; + + private Multimap headers; + + public Long offset() { + return offset; + } + + public Long length() { + return length; + } + + public long objectSize() { + return objectSize; + } + + public ServerSideEncryptionCustomerKey srcSsec() { + return ssec; + } - /** Create new ComposeSource for given bucket and object. */ - public ComposeSource(String bucketName, String objectName) throws IllegalArgumentException { - this(bucketName, objectName, null, null, null, null, null); + public String matchETag() { + return matchETag; } - /** Create new ComposeSource for given bucket, object, offset and length. */ - public ComposeSource(String bucketName, String objectName, Long offset, Long length) - throws IllegalArgumentException { - this(bucketName, objectName, offset, length, null, null, null); + public String notMatchETag() { + return notMatchETag; } - /** Create new ComposeSource for given bucket, object, offset, length and headerMap. */ - public ComposeSource( - String bucketName, String objectName, Long offset, Long length, Map headerMap) - throws IllegalArgumentException { - this(bucketName, objectName, offset, length, headerMap, null, null); + public ZonedDateTime modifiedSince() { + return modifiedSince; } - /** - * Create new ComposeSource for given bucket, object, offset, length, headerMap and - * CopyConditions. - */ - public ComposeSource( - String bucketName, - String objectName, - Long offset, - Long length, - Map headerMap, - CopyConditions copyConditions) - throws IllegalArgumentException { - this(bucketName, objectName, offset, length, headerMap, copyConditions, null); + public ZonedDateTime unmodifiedSince() { + return unmodifiedSince; } - /** - * Creates new ComposeSource for given bucket, object, offset, length, headerMap, CopyConditions - * and server side encryption. - * - * @throws IllegalArgumentException upon invalid value is passed to a method. - */ - public ComposeSource( - String bucketName, - String objectName, - Long offset, - Long length, - Map headerMap, - CopyConditions copyConditions, - ServerSideEncryptionCustomerKey ssec) - throws IllegalArgumentException { - if (bucketName == null) { - throw new IllegalArgumentException("Source bucket name cannot be empty"); + public Multimap headers() { + return headers; + } + + public ServerSideEncryptionCustomerKey ssec() { + return ssec; + } + + public static Builder builder() { + return new Builder(); + } + + /** Constructs header . */ + public void buildHeaders(long objectSize, String etag) throws IllegalArgumentException { + validateSize(objectSize); + Multimap headers = HashMultimap.create(); + headers.put("x-amz-copy-source", S3Escaper.encodePath(bucketName + "/" + objectName)); + headers.put("x-amz-copy-source-if-match", etag); + + if (extraHeaders() != null) { + headers.putAll(extraHeaders()); } - if (objectName == null) { - throw new IllegalArgumentException("Source object name cannot be empty"); + if (matchETag != null) { + headers.put("x-amz-copy-source-if-match", matchETag); } - if (offset != null && offset < 0) { - throw new IllegalArgumentException("Offset cannot be negative"); + if (ssec != null) { + headers.putAll(Multimaps.forMap(ssec.copySourceHeaders())); } - if (length != null && length < 0) { - throw new IllegalArgumentException("Length cannot be negative"); + if (notMatchETag != null) { + headers.put("x-amz-copy-source-if-none-match", notMatchETag); } - if (length != null && offset == null) { - offset = 0L; + if (modifiedSince != null) { + headers.put( + "x-amz-copy-source-if-modified-since", + modifiedSince.format(Time.HTTP_HEADER_DATE_FORMAT)); } - this.bucketName = bucketName; - this.objectName = objectName; - this.offset = offset; - this.length = length; - if (headerMap != null) { - this.headerMap = Collections.unmodifiableMap(headerMap); - } else { - this.headerMap = null; + if (unmodifiedSince != null) { + headers.put( + "x-amz-copy-source-if-unmodified-since", + unmodifiedSince.format(Time.HTTP_HEADER_DATE_FORMAT)); } - this.copyConditions = copyConditions; - this.ssec = ssec; + + this.objectSize = objectSize; + this.headers = headers; } - /** Constructs header . */ - public void buildHeaders(long objectSize, String etag) throws IllegalArgumentException { + private void validateSize(long objectSize) throws IllegalArgumentException { if (offset != null && offset >= objectSize) { throw new IllegalArgumentException( "source " @@ -153,56 +156,52 @@ public void buildHeaders(long objectSize, String etag) throws IllegalArgumentExc + objectSize); } } + } - Map headers = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); - headers.put("x-amz-copy-source", S3Escaper.encodePath(bucketName + "/" + objectName)); - headers.put("x-amz-copy-source-if-match", etag); - - if (headerMap != null) { - headers.putAll(headerMap); + /** Argument builder of {@link ComposeSource}. */ + public static final class Builder extends ObjectVersionArgs.Builder { + @Override + protected void validate(ComposeSource args) { + super.validate(args); } - if (copyConditions != null) { - headers.putAll(copyConditions.getConditions()); + public Builder offset(long offset) { + validateNullOrPositive(offset, "offset"); + operations.add(args -> args.offset = offset); + return this; } - if (ssec != null) { - headers.putAll(ssec.copySourceHeaders()); + public Builder length(long length) { + validateNullOrPositive(length, "length"); + operations.add(args -> args.length = length); + return this; } - this.objectSize = objectSize; - this.headers = Collections.unmodifiableMap(headers); - } - - public String bucketName() { - return bucketName; - } - - public String objectName() { - return objectName; - } - - public Long offset() { - return offset; - } - - public Long length() { - return length; - } + public Builder ssec(ServerSideEncryptionCustomerKey ssec) { + operations.add(args -> args.ssec = ssec); + return this; + } - public CopyConditions copyConditions() { - return copyConditions; - } + public Builder matchETag(String etag) { + validateNullOrNotEmptyString(etag, "etag"); + operations.add(args -> args.matchETag = etag); + return this; + } - public ServerSideEncryptionCustomerKey ssec() { - return ssec; - } + public Builder notMatchETag(String etag) { + validateNullOrNotEmptyString(etag, "etag"); + operations.add(args -> args.notMatchETag = etag); + return this; + } - public Map headers() { - return headers; - } + public Builder modifiedSince(ZonedDateTime modifiedTime) { + operations.add(args -> args.modifiedSince = modifiedTime); + return this; + } - public long objectSize() { - return objectSize; + public Builder unmodifiedSince(ZonedDateTime unmodifiedTime) { + operations.add(args -> args.unmodifiedSince = unmodifiedTime); + return this; + } } } diff --git a/api/src/main/java/io/minio/ComposeSourceArgs.java b/api/src/main/java/io/minio/ComposeSourceArgs.java deleted file mode 100644 index 7e214e98c..000000000 --- a/api/src/main/java/io/minio/ComposeSourceArgs.java +++ /dev/null @@ -1,266 +0,0 @@ -/* - * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2020 MinIO, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.minio; - -import com.google.common.collect.HashMultimap; -import com.google.common.collect.Multimap; -import com.google.common.collect.Multimaps; -import java.time.ZonedDateTime; -import okhttp3.HttpUrl; - -/** Argument class of MinioClient.composeObject(). */ -public class ComposeSourceArgs extends ObjectWriteArgs { - private String srcVersionId; - private Long srcOffset; - private Long srcLength; - private long srcObjectSize; - private ServerSideEncryptionCustomerKey srcSsec; - private Multimap headers; - - private String srcMatchETag; - private String srcNotMatchETag; - private ZonedDateTime srcModifiedSince; - private ZonedDateTime srcUnmodifiedSince; - private Directive metadataDirective; - private Directive taggingDirective; - - public String srcBucket() { - return bucketName; - } - - public String srcObject() { - return objectName; - } - - public String srcVersionId() { - return srcVersionId; - } - - public Long srcOffset() { - return srcOffset; - } - - public Long srcLength() { - return srcLength; - } - - public long objectSize() { - return srcObjectSize; - } - - public ServerSideEncryptionCustomerKey srcSsec() { - return srcSsec; - } - - public String srcMatchETag() { - return srcMatchETag; - } - - public String srcNotMatchETag() { - return srcNotMatchETag; - } - - public ZonedDateTime srcModifiedSince() { - return srcModifiedSince; - } - - public ZonedDateTime srcUnmodifiedSince() { - return srcUnmodifiedSince; - } - - public Directive metadataDirective() { - return metadataDirective; - } - - public Directive taggingDirective() { - return taggingDirective; - } - - public static Builder builder() { - return new Builder(); - } - - @Override - public void validateSse(HttpUrl url) { - super.validateSse(url); - checkSse(srcSsec, url); - } - - public Multimap headers() { - return headers; - } - - /** Constructs header . */ - public void buildHeaders(long objectSize, String etag) throws IllegalArgumentException { - if (srcOffset != null && srcOffset >= objectSize) { - throw new IllegalArgumentException( - "source " - + bucketName - + "/" - + objectName - + ": offset " - + srcOffset - + " is beyond object size " - + objectSize); - } - - if (srcLength != null) { - if (srcLength > objectSize) { - throw new IllegalArgumentException( - "source " - + bucketName - + "/" - + objectName - + ": length " - + srcLength - + " is beyond object size " - + objectSize); - } - - if (srcOffset + srcLength > objectSize) { - throw new IllegalArgumentException( - "source " - + bucketName - + "/" - + objectName - + ": compose size " - + (srcOffset + srcLength) - + " is beyond object size " - + objectSize); - } - } - - Multimap headers = HashMultimap.create(); - headers.put("x-amz-copy-source", S3Escaper.encodePath(bucketName + "/" + objectName)); - headers.put("x-amz-copy-source-if-match", etag); - - if (extraHeaders() != null) { - headers.putAll(extraHeaders()); - } - - if (srcMatchETag != null) { - headers.put("x-amz-copy-source-if-match", srcMatchETag); - } - - if (srcSsec != null) { - headers.putAll(Multimaps.forMap(srcSsec.copySourceHeaders())); - } - - if (srcNotMatchETag != null) { - headers.put("x-amz-copy-source-if-none-match", srcNotMatchETag); - } - - if (srcModifiedSince != null) { - headers.put( - "x-amz-copy-source-if-modified-since", - srcModifiedSince.format(Time.HTTP_HEADER_DATE_FORMAT)); - } - - if (srcUnmodifiedSince != null) { - headers.put( - "x-amz-copy-source-if-unmodified-since", - srcUnmodifiedSince.format(Time.HTTP_HEADER_DATE_FORMAT)); - } - - if (metadataDirective != null) { - headers.put("x-amz-metadata-directive", metadataDirective.name()); - } - - if (taggingDirective != null) { - headers.put("x-amz-tagging-directive", taggingDirective.name()); - } - - this.srcObjectSize = objectSize; - this.headers = headers; - } - - /** Argument builder of {@link ComposeSourceArgs}. */ - public static final class Builder extends ObjectWriteArgs.Builder { - @Override - protected void validate(ComposeSourceArgs args) { - super.validate(args); - validateBucketName(args.bucket()); - } - - public Builder srcBucket(String srcBucket) { - validateBucketName(srcBucket); - operations.add(args -> args.bucketName = srcBucket); - return this; - } - - public Builder srcObject(String srcObject) { - validateNullOrNotEmptyString(srcObject, "source object"); - operations.add(args -> args.objectName = srcObject); - return this; - } - - public Builder srcVersionId(String srcVersionId) { - validateNullOrNotEmptyString(srcVersionId, "source version ID"); - operations.add(args -> args.srcVersionId = srcVersionId); - return this; - } - - public Builder srcOffset(long offset) { - validateNullOrEmpty(offset, "offset"); - operations.add(args -> args.srcOffset = offset); - return this; - } - - public Builder srcLength(long length) { - validateNullOrEmpty(length, "length"); - operations.add(args -> args.srcLength = length); - return this; - } - - public Builder srcSsec(ServerSideEncryptionCustomerKey ssec) { - operations.add(args -> args.srcSsec = ssec); - return this; - } - - public Builder srcMatchETag(String etag) { - validateNullOrNotEmptyString(etag, "etag"); - operations.add(args -> args.srcMatchETag = etag); - return this; - } - - public Builder srcNotMatchETag(String etag) { - validateNullOrNotEmptyString(etag, "etag"); - operations.add(args -> args.srcNotMatchETag = etag); - return this; - } - - public Builder srcModifiedSince(ZonedDateTime modifiedTime) { - operations.add(args -> args.srcModifiedSince = modifiedTime); - return this; - } - - public Builder srcUnmodifiedSince(ZonedDateTime modifiedTime) { - operations.add(args -> args.srcUnmodifiedSince = modifiedTime); - return this; - } - - public Builder metadataDirective(Directive directive) { - operations.add(args -> args.metadataDirective = directive); - return this; - } - - public Builder taggingDirective(Directive directive) { - operations.add(args -> args.taggingDirective = directive); - return this; - } - } -} diff --git a/api/src/main/java/io/minio/MinioClient.java b/api/src/main/java/io/minio/MinioClient.java index be91460f0..9cf7ad6ae 100755 --- a/api/src/main/java/io/minio/MinioClient.java +++ b/api/src/main/java/io/minio/MinioClient.java @@ -2555,202 +2555,16 @@ public void composeObject( InternalException, InvalidBucketNameException, InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException, ServerException, XmlParserException { - if ((bucketName == null) || (bucketName.isEmpty())) { - throw new IllegalArgumentException("bucket name cannot be empty"); - } - - checkObjectName(objectName); - - if (sources.isEmpty()) { - throw new IllegalArgumentException("compose sources cannot be empty"); - } - - checkWriteRequestSse(sse); - - long objectSize = 0; - int partsCount = 0; - for (int i = 0; i < sources.size(); i++) { - ComposeSource src = sources.get(i); - - checkReadRequestSse(src.ssec()); - - ObjectStat stat = statObject(src.bucketName(), src.objectName(), src.ssec()); - src.buildHeaders(stat.length(), stat.etag()); - - if (i != 0 && src.headers().containsKey("x-amz-meta-x-amz-key")) { - throw new IllegalArgumentException( - "Client side encryption is not supported for more than one source"); - } - - long size = stat.length(); - if (src.length() != null) { - size = src.length(); - } else if (src.offset() != null) { - size -= src.offset(); - } - - if (size < PutObjectOptions.MIN_MULTIPART_SIZE - && sources.size() != 1 - && i != (sources.size() - 1)) { - throw new IllegalArgumentException( - "source " - + src.bucketName() - + "/" - + src.objectName() - + ": size " - + size - + " must be greater than " - + PutObjectOptions.MIN_MULTIPART_SIZE); - } - - objectSize += size; - if (objectSize > PutObjectOptions.MAX_OBJECT_SIZE) { - throw new IllegalArgumentException( - "Destination object size must be less than " + PutObjectOptions.MAX_OBJECT_SIZE); - } - - if (size > PutObjectOptions.MAX_PART_SIZE) { - long count = size / PutObjectOptions.MAX_PART_SIZE; - long lastPartSize = size - (count * PutObjectOptions.MAX_PART_SIZE); - if (lastPartSize > 0) { - count++; - } else { - lastPartSize = PutObjectOptions.MAX_PART_SIZE; - } - - if (lastPartSize < PutObjectOptions.MIN_MULTIPART_SIZE - && sources.size() != 1 - && i != (sources.size() - 1)) { - throw new IllegalArgumentException( - "source " - + src.bucketName() - + "/" - + src.objectName() - + ": " - + "for multipart split upload of " - + size - + ", last part size is less than " - + PutObjectOptions.MIN_MULTIPART_SIZE); - } - - partsCount += (int) count; - } else { - partsCount++; - } - - if (partsCount > PutObjectOptions.MAX_MULTIPART_COUNT) { - throw new IllegalArgumentException( - "Compose sources create more than allowed multipart count " - + PutObjectOptions.MAX_MULTIPART_COUNT); - } - } - - if (partsCount == 1) { - ComposeSource src = sources.get(0); - if (headerMap == null) { - headerMap = new HashMap<>(); - } - if ((src.offset() != null) && (src.length() == null)) { - headerMap.put("x-amz-copy-source-range", "bytes=" + src.offset() + "-"); - } - - if ((src.offset() != null) && (src.length() != null)) { - headerMap.put( - "x-amz-copy-source-range", - "bytes=" + src.offset() + "-" + (src.offset() + src.length() - 1)); - } - copyObject( - bucketName, - objectName, - headerMap, - sse, - src.bucketName(), - src.objectName(), - src.ssec(), - src.copyConditions()); - return; - } - - Map sseHeaders = null; - if (sse != null) { - sseHeaders = sse.headers(); - if (headerMap == null) { - headerMap = new HashMap<>(); - } - headerMap.putAll(sseHeaders); - } - - String uploadId = createMultipartUpload(bucketName, objectName, headerMap); - - int partNumber = 0; - Part[] totalParts = new Part[partsCount]; - try { - for (int i = 0; i < sources.size(); i++) { - ComposeSource src = sources.get(i); - - long size = src.objectSize(); - if (src.length() != null) { - size = src.length(); - } else if (src.offset() != null) { - size -= src.offset(); - } - long offset = 0; - if (src.offset() != null) { - offset = src.offset(); - } - - if (size <= PutObjectOptions.MAX_PART_SIZE) { - partNumber++; - Map headers = new HashMap<>(); - if (src.headers() != null) { - headers.putAll(src.headers()); - } - if (src.length() != null) { - headers.put( - "x-amz-copy-source-range", "bytes=" + offset + "-" + (offset + src.length() - 1)); - } else if (src.offset() != null) { - headers.put("x-amz-copy-source-range", "bytes=" + offset + "-" + (offset + size - 1)); - } - if (sseHeaders != null) { - headers.putAll(sseHeaders); - } - String eTag = uploadPartCopy(bucketName, objectName, uploadId, partNumber, headers); - - totalParts[partNumber - 1] = new Part(partNumber, eTag); - continue; - } - - while (size > 0) { - partNumber++; - long startBytes = offset; - long endBytes = startBytes + PutObjectOptions.MAX_PART_SIZE; - if (size < PutObjectOptions.MAX_PART_SIZE) { - endBytes = startBytes + size; - } - - Map headers = src.headers(); - headers.put("x-amz-copy-source-range", "bytes=" + startBytes + "-" + endBytes); - if (sseHeaders != null) { - headers.putAll(sseHeaders); - } - String eTag = uploadPartCopy(bucketName, objectName, uploadId, partNumber, headers); - - totalParts[partNumber - 1] = new Part(partNumber, eTag); - - offset = startBytes; - size -= (endBytes - startBytes); - } - } + ComposeObjectArgs.Builder builder = + ComposeObjectArgs.builder() + .bucket(bucketName) + .object(objectName) + .extraHeaders(Multimaps.forMap(headerMap)) + .sources(sources) + .sse(sse); - completeMultipartUpload(bucketName, objectName, uploadId, totalParts); - } catch (RuntimeException e) { - abortMultipartUpload(bucketName, objectName, uploadId); - throw e; - } catch (Exception e) { - abortMultipartUpload(bucketName, objectName, uploadId); - throw e; - } + composeObject(builder.build()); } /** @@ -2758,23 +2572,43 @@ public void composeObject( * * 
Example:{@code
    * List sourceObjectList = new ArrayList();
-   * sourceObjectList.add(new ComposeSource("my-job-bucket", "my-objectname-part-one"));
-   * sourceObjectList.add(new ComposeSource("my-job-bucket", "my-objectname-part-two"));
-   * sourceObjectList.add(new ComposeSource("my-job-bucket", "my-objectname-part-three"));
+   *
+   * sourceObjectList.add(
+   *    ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-one").build());
+   * sourceObjectList.add(
+   *    ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-two").build());
+   * sourceObjectList.add(
+   *    ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-three").build());
    *
    * // Create my-bucketname/my-objectname by combining source object list.
-   * minioClient.composeObject("my-bucketname", "my-objectname", sourceObjectList,
-   *     null, null);
+   * minioClient.composeObject(
+   *    ComposeObjectArgs.builder()
+   *        .bucket("my-bucketname")
+   *        .object("my-objectname")
+   *        .sources(sourceObjectList)
+   *        .build());
    *
    * // Create my-bucketname/my-objectname with user metadata by combining source object
    * // list.
-   * minioClient.composeObject("my-bucketname", "my-objectname", sourceObjectList,
-   *     userMetadata, null);
+   * minioClient.composeObject(
+   *     ComposeObjectArgs.builder()
+   *        .bucket("my-bucketname")
+   *        .object("my-objectname")
+   *        .sources(sourceObjectList)
+   *        .extraHeaders(Multimaps.forMap(userMetadata))
+   *        .build());
    *
    * // Create my-bucketname/my-objectname with user metadata and server-side encryption
    * // by combining source object list.
-   * minioClient.composeObject("my-bucketname", "my-objectname", sourceObjectList,
-   *     userMetadata, sse);
+   * minioClient.composeObject(
+   *   ComposeObjectArgs.builder()
+   *        .bucket("my-bucketname")
+   *        .object("my-objectname")
+   *        .sources(sourceObjectList)
+   *        .extraHeaders(Multimaps.forMap(userMetadata))
+   *        .ssec(sse)
+   *        .build());
+   *
    * }
* * @param args {@link ComposeObjectArgs} object. @@ -2800,19 +2634,19 @@ public void composeObject(ComposeObjectArgs args) long objectSize = 0; int partsCount = 0; - List sources = args.sources(); + List sources = args.sources(); for (int i = 0; i < sources.size(); i++) { - ComposeSourceArgs src = sources.get(i); + ComposeSource src = sources.get(i); checkReadRequestSse(src.srcSsec()); ObjectStat stat = statObject( StatObjectArgs.builder() - .bucket(src.srcBucket()) - .object(src.srcObject()) - .versionId(src.srcVersionId()) - .ssec(src.srcSsec()) + .bucket(src.bucket()) + .object(src.object()) + .versionId(src.versionId()) + .ssec(src.ssec()) .build()); src.buildHeaders(stat.length(), stat.etag()); @@ -2823,10 +2657,10 @@ public void composeObject(ComposeObjectArgs args) } long size = stat.length(); - if (src.srcLength() != null) { - size = src.srcLength(); - } else if (src.srcOffset() != null) { - size -= src.srcOffset(); + if (src.length() != null) { + size = src.length(); + } else if (src.offset() != null) { + size -= src.offset(); } if (size < PutObjectOptions.MIN_MULTIPART_SIZE @@ -2834,9 +2668,9 @@ public void composeObject(ComposeObjectArgs args) && i != (sources.size() - 1)) { throw new IllegalArgumentException( "source " - + src.srcBucket() + + src.bucket() + "/" - + src.srcObject() + + src.object() + ": size " + size + " must be greater than " @@ -2863,9 +2697,9 @@ public void composeObject(ComposeObjectArgs args) && i != (sources.size() - 1)) { throw new IllegalArgumentException( "source " - + src.srcBucket() + + src.bucket() + "/" - + src.srcObject() + + src.object() + ": " + "for multipart split upload of " + size @@ -2886,19 +2720,19 @@ public void composeObject(ComposeObjectArgs args) } if (partsCount == 1) { - ComposeSourceArgs src = sources.get(0); + ComposeSource src = sources.get(0); Multimap headers = HashMultimap.create(); if (args.extraHeaders() != null) { headers.putAll(args.extraHeaders()); } - if ((src.srcOffset() != null) && (src.srcLength() == null)) { - headers.put("x-amz-copy-source-range", "bytes=" + src.srcOffset() + "-"); + if ((src.offset() != null) && (src.length() == null)) { + headers.put("x-amz-copy-source-range", "bytes=" + src.offset() + "-"); } - if ((src.srcOffset() != null) && (src.srcLength() != null)) { + if ((src.offset() != null) && (src.length() != null)) { headers.put( "x-amz-copy-source-range", - "bytes=" + src.srcOffset() + "-" + (src.srcOffset() + src.srcLength() - 1)); + "bytes=" + src.offset() + "-" + (src.offset() + src.length() - 1)); } copyObject( CopyObjectArgs.builder() @@ -2906,18 +2740,15 @@ public void composeObject(ComposeObjectArgs args) .object(args.object()) .extraHeaders(args.extraHeaders) .sse(args.sse()) - .srcBucket(src.srcBucket()) - .srcObject(src.srcObject()) - .srcVersionId(src.srcVersionId()) - .srcSsec(src.srcSsec()) - .srcMatchETag(src.srcMatchETag()) - .srcNotMatchETag(src.srcNotMatchETag()) - .srcModifiedSince(src.srcModifiedSince()) - .srcUnmodifiedSince(src.srcUnmodifiedSince()) - .metadataDirective(src.metadataDirective()) - .taggingDirective(src.taggingDirective()) + .srcBucket(src.bucket()) + .srcObject(src.object()) + .srcVersionId(src.versionId()) + .srcSsec(src.ssec()) + .srcMatchETag(src.matchETag()) + .srcNotMatchETag(src.notMatchETag()) + .srcModifiedSince(src.modifiedSince()) + .srcUnmodifiedSince(src.unmodifiedSince()) .build()); - return; } @@ -2931,23 +2762,22 @@ public void composeObject(ComposeObjectArgs args) headerMap.putAll(sseHeaders); } - String uploadId = createMultipartUpload(args.bucket(), null, args.object(), headerMap, null); + String uploadId = createMultipartUpload(args.bucket(), null, args.object(), headerMap); int partNumber = 0; Part[] totalParts = new Part[partsCount]; try { for (int i = 0; i < sources.size(); i++) { - ComposeSourceArgs src = sources.get(i); - + ComposeSource src = sources.get(i); long size = src.objectSize(); - if (src.srcLength() != null) { - size = src.srcLength(); - } else if (src.srcOffset() != null) { - size -= src.srcOffset(); + if (src.length() != null) { + size = src.length(); + } else if (src.offset() != null) { + size -= src.offset(); } long offset = 0; - if (src.srcOffset() != null) { - offset = src.srcOffset(); + if (src.offset() != null) { + offset = src.offset(); } if (size <= PutObjectOptions.MAX_PART_SIZE) { @@ -2956,11 +2786,10 @@ public void composeObject(ComposeObjectArgs args) if (src.headers() != null) { headers.putAll(src.headers()); } - if (src.srcLength() != null) { + if (src.length() != null) { headers.put( - "x-amz-copy-source-range", - "bytes=" + offset + "-" + (offset + src.srcLength() - 1)); - } else if (src.srcOffset() != null) { + "x-amz-copy-source-range", "bytes=" + offset + "-" + (offset + src.length() - 1)); + } else if (src.offset() != null) { headers.put("x-amz-copy-source-range", "bytes=" + offset + "-" + (offset + size - 1)); } if (sseHeaders != null) { @@ -7594,27 +7423,16 @@ protected void completeMultipartUpload( * @throws NoSuchAlgorithmException thrown to indicate missing of MD5 or SHA-256 digest library. * @throws XmlParserException thrown to indicate XML parsing error. */ - @Deprecated protected String createMultipartUpload( String bucketName, String objectName, Map headerMap) throws InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { - // set content type if not set already - if ((headerMap != null) && (headerMap.get("Content-Type") == null)) { - headerMap.put("Content-Type", "application/octet-stream"); - } - - Map queryParamMap = new HashMap<>(); - queryParamMap.put("uploads", ""); - - Response response = executePost(bucketName, objectName, headerMap, queryParamMap, ""); - - try (ResponseBody body = response.body()) { - InitiateMultipartUploadResult result = - Xml.unmarshal(InitiateMultipartUploadResult.class, body.charStream()); - return result.uploadId(); - } + return createMultipartUpload( + bucketName, + getRegion(bucketName), + objectName, + Multimaps.forMap(normalizeHeaders(headerMap))); } /** @@ -7623,6 +7441,7 @@ protected String createMultipartUpload( * S3 API. * * @param bucketName Name of the bucket. + * @param region Region name of buckets in S3 service. * @param objectName Object name in the bucket. * @param headers Request headers. * @return String - Contains upload ID. @@ -7639,36 +7458,26 @@ protected String createMultipartUpload( * @throws XmlParserException thrown to indicate XML parsing error. */ protected String createMultipartUpload( - String bucketName, - String region, - String objectName, - Multimap headers, - Multimap extraQueryParams) + String bucketName, String region, String objectName, Multimap headers) throws InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, InsufficientDataException, IOException, InvalidKeyException, ServerException, XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { - Multimap queryParams = HashMultimap.create(); - if (extraQueryParams != null) { - queryParams.putAll(extraQueryParams); - } - queryParams.put("uploads", ""); - Multimap headersCopy = HashMultimap.create(); - if (headers != null) { - headersCopy.putAll(headers); - } // set content type if not set already - if (!headersCopy.containsKey("Content-Type")) { - headersCopy.put("Content-Type", "application/octet-stream"); + if (!headers.containsKey("Content-Type")) { + headers.put("Content-Type", "application/octet-stream"); } + Multimap queryParams = HashMultimap.create(); + queryParams.put("uploads", ""); + try (Response response = execute( Method.POST, bucketName, objectName, (region != null) ? region : getRegion(bucketName), - headersCopy, + headers, queryParams, null, 0)) { @@ -8035,49 +7844,7 @@ protected String uploadPart( * @throws NoSuchAlgorithmException thrown to indicate missing of MD5 or SHA-256 digest library. * @throws XmlParserException thrown to indicate XML parsing error. */ - protected String uploadPartCopy( - String bucketName, - String objectName, - String uploadId, - int partNumber, - Map headerMap) - throws InvalidBucketNameException, IllegalArgumentException, NoSuchAlgorithmException, - InsufficientDataException, IOException, InvalidKeyException, ServerException, - XmlParserException, ErrorResponseException, InternalException, InvalidResponseException { - Map queryParamMap = new HashMap<>(); - queryParamMap.put("partNumber", Integer.toString(partNumber)); - queryParamMap.put("uploadId", uploadId); - Response response = executePut(bucketName, objectName, headerMap, queryParamMap, "", 0); - try (ResponseBody body = response.body()) { - CopyPartResult result = Xml.unmarshal(CopyPartResult.class, body.charStream()); - return result.etag(); - } - } - - /** - * Do UploadPartCopy - * S3 API. - * - * @param bucketName Name of the bucket. - * @param objectName Object name in the bucket. - * @param uploadId Upload ID. - * @param partNumber Part number. - * @param headerMap Source object definitions. - * @return String - Contains ETag. - * @throws ErrorResponseException thrown to indicate S3 service returned an error response. - * @throws IllegalArgumentException throws to indicate invalid argument passed. - * @throws InsufficientDataException thrown to indicate not enough data available in InputStream. - * @throws InternalException thrown to indicate internal library error. - * @throws InvalidBucketNameException thrown to indicate invalid bucket name passed. - * @throws InvalidKeyException thrown to indicate missing of HMAC SHA-256 library. - * @throws InvalidResponseException thrown to indicate S3 service returned invalid or no error - * response. - * @throws IOException thrown to indicate I/O error on S3 operation. - * @throws NoSuchAlgorithmException thrown to indicate missing of MD5 or SHA-256 digest library. - * @throws XmlParserException thrown to indicate XML parsing error. - */ - protected String uploadPartCopy( + private String uploadPartCopy( String bucketName, String objectName, String uploadId, diff --git a/docs/API.md b/docs/API.md index b570662e0..0385c0258 100644 --- a/docs/API.md +++ b/docs/API.md @@ -989,10 +989,48 @@ Creates an object by combining data from different source objects using server-s __Example__ ```java -List sourceObjectList = new ArrayList(); -sourceObjectList.add(ComposeSourceArgs.builder().bucket("my-job-bucket").object("my-objectname-part-one").build()); -sourceObjectList.add(ComposeSourceArgs.builder().bucket("my-job-bucket").object("my-objectname-part-two").build(); -sourceObjectList.add(ComposeSourceArgs.builder().bucket("my-job-bucket").object("my-objectname-part-three").build()); +List sourceObjectList = new ArrayList(); +sourceObjectList.add( + ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-one").build()); +sourceObjectList.add( + ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-two").build()); +sourceObjectList.add( + ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-three").build()); + +// Create my-bucketname/my-objectname by combining source object list. +minioClient.composeObject( + ComposeObjectArgs.builder() + .bucket("my-bucketname") + .object("my-objectname") + .sources(sourceObjectList) + .build()); + +// Create my-bucketname/my-objectname with user metadata by combining source object +// list. +minioClient.composeObject( + ComposeObjectArgs.builder() + .bucket("my-bucketname") + .object("my-objectname") + .sources(sourceObjectList) + .extraHeaders(Multimaps.forMap(userMetadata)) + .build()); + +// Create my-bucketname/my-objectname with user metadata and server-side encryption +// by combining source object list. +minioClient.composeObject( + ComposeObjectArgs.builder() + .bucket("my-bucketname") + .object("my-objectname") + .sources(sourceObjectList) + .extraHeaders(Multimaps.forMap(userMetadata)) + .ssec(sse) + .build()); + + +List sourceObjectList = new ArrayList(); +sourceObjectList.add(ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-one").build()); +sourceObjectList.add(ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-two").build(); +sourceObjectList.add(ComposeSource.builder().bucket("my-job-bucket").object("my-objectname-part-three").build()); // Create my-bucketname/my-objectname by combining source object list. minioClient.composeObject(ComposeObjectArgs.builder().bucket("my-bucketname").object("my-objectname").source(sourceObjectList).build()); diff --git a/examples/ComposeObject.java b/examples/ComposeObject.java index 79d9ae61d..b5f26ab1b 100644 --- a/examples/ComposeObject.java +++ b/examples/ComposeObject.java @@ -15,7 +15,7 @@ */ import io.minio.ComposeObjectArgs; -import io.minio.ComposeSourceArgs; +import io.minio.ComposeSource; import io.minio.MinioClient; import io.minio.errors.MinioException; import java.io.IOException; @@ -36,25 +36,19 @@ public static void main(String[] args) "Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"); - // Create a ComposeSourceArgs to compose Object. - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() - .srcBucket("my-bucketname-one") - .srcObject("my-objectname-one") - .build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder() - .srcBucket("my-bucketname-two") - .srcObject("my-objectname-two") - .build(); - ComposeSourceArgs s3 = - ComposeSourceArgs.builder() - .srcBucket("my-bucketname-three") - .srcObject("my-objectname-three") + // Create a ComposeSource to compose Object. + ComposeSource s1 = + ComposeSource.builder().bucket("my-bucketname-one").object("my-objectname-one").build(); + ComposeSource s2 = + ComposeSource.builder().bucket("my-bucketname-two").object("my-objectname-two").build(); + ComposeSource s3 = + ComposeSource.builder() + .bucket("my-bucketname-three") + .object("my-objectname-three") .build(); - // Adding the ComposeSourceArgs to an ArrayList - List sourceObjectList = new ArrayList(); + // Adding the ComposeSource to an ArrayList + List sourceObjectList = new ArrayList(); sourceObjectList.add(s1); sourceObjectList.add(s2); sourceObjectList.add(s3); diff --git a/examples/ComposeObjectEncrypted.java b/examples/ComposeObjectEncrypted.java index 91436b9e4..937636a8e 100644 --- a/examples/ComposeObjectEncrypted.java +++ b/examples/ComposeObjectEncrypted.java @@ -15,7 +15,7 @@ */ import io.minio.ComposeObjectArgs; -import io.minio.ComposeSourceArgs; +import io.minio.ComposeSource; import io.minio.MinioClient; import io.minio.PutObjectOptions; import io.minio.ServerSideEncryption; @@ -68,20 +68,12 @@ public static void main(String[] args) options.setSse(ssePut); minioClient.putObject(bucketName, sourceObject2, inputfile2, options); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() - .srcBucket(bucketName) - .srcObject(sourceObject1) - .srcSsec(ssePut) - .build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder() - .srcBucket(bucketName) - .srcObject(sourceObject2) - .srcSsec(ssePut) - .build(); + ComposeSource s1 = + ComposeSource.builder().bucket(bucketName).object(sourceObject1).ssec(ssePut).build(); + ComposeSource s2 = + ComposeSource.builder().bucket(bucketName).object(sourceObject2).ssec(ssePut).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); diff --git a/functional/FunctionalTest.java b/functional/FunctionalTest.java index 922a8d467..57adc354c 100644 --- a/functional/FunctionalTest.java +++ b/functional/FunctionalTest.java @@ -22,7 +22,7 @@ import io.minio.BucketExistsArgs; import io.minio.CloseableIterator; import io.minio.ComposeObjectArgs; -import io.minio.ComposeSourceArgs; +import io.minio.ComposeSource; import io.minio.CopyObjectArgs; import io.minio.DeleteBucketEncryptionArgs; import io.minio.DeleteBucketLifeCycleArgs; @@ -2671,7 +2671,7 @@ public static void copyObject_test11() throws Exception { false); } - /** Test: ccomposeObject(ComposeObjectArgs args). */ + /** Test: composeObject(ComposeObjectArgs args). */ public static void composeObject_test1() throws Exception { String methodName = "composeObject(ComposeObjectArgs args)"; if (!mintEnv) { @@ -2687,12 +2687,10 @@ public static void composeObject_test1() throws Exception { PutObjectOptions options = new PutObjectOptions(6 * MB, -1); client.putObject(bucketName, filename1, filename1, options); client.putObject(bucketName, filename2, filename2, options); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder().srcBucket(bucketName).srcObject(filename1).build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder().srcBucket(bucketName).srcObject(filename2).build(); + ComposeSource s1 = ComposeSource.builder().bucket(bucketName).object(filename1).build(); + ComposeSource s2 = ComposeSource.builder().bucket(bucketName).object(filename2).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); try { @@ -2732,17 +2730,16 @@ public static void composeObject_test2() throws Exception { PutObjectOptions options = new PutObjectOptions(6 * MB, -1); client.putObject(bucketName, filename1, filename1, options); client.putObject(bucketName, filename2, filename2, options); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() + ComposeSource s1 = + ComposeSource.builder() .bucket(bucketName) .object(filename1) - .srcOffset(10L) - .srcLength(6291436L) + .offset(10L) + .length(6291436L) .build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder().bucket(bucketName).object(filename2).build(); + ComposeSource s2 = ComposeSource.builder().bucket(bucketName).object(filename2).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); try { @@ -2778,15 +2775,15 @@ public static void composeObject_test3() throws Exception { String filename1 = createFile6Mb(); client.putObject(bucketName, filename1, filename1, new PutObjectOptions(6 * MB, -1)); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() + ComposeSource s1 = + ComposeSource.builder() .bucket(bucketName) .object(filename1) - .srcOffset(10L) - .srcLength(6291436L) + .offset(10L) + .length(6291436L) .build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); try { client.composeObject( @@ -2836,20 +2833,12 @@ public static void composeObject_test4() throws Exception { options.setSse(ssePut); client.putObject(bucketName, filename1, filename1, options); client.putObject(bucketName, filename2, filename2, options); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() - .srcBucket(bucketName) - .srcObject(filename1) - .srcSsec(ssePut) - .build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder() - .srcBucket(bucketName) - .srcObject(filename2) - .srcSsec(ssePut) - .build(); + ComposeSource s1 = + ComposeSource.builder().bucket(bucketName).object(filename1).ssec(ssePut).build(); + ComposeSource s2 = + ComposeSource.builder().bucket(bucketName).object(filename2).ssec(ssePut).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); try { @@ -2895,16 +2884,11 @@ public static void composeObject_test5() throws Exception { options.setSse(ssePut); client.putObject(bucketName, filename1, filename1, options); client.putObject(bucketName, filename2, filename2, new PutObjectOptions(6 * MB, -1)); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder() - .srcBucket(bucketName) - .object(filename1) - .srcSsec(ssePut) - .build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder().srcBucket(bucketName).object(filename2).build(); + ComposeSource s1 = + ComposeSource.builder().bucket(bucketName).object(filename1).ssec(ssePut).build(); + ComposeSource s2 = ComposeSource.builder().bucket(bucketName).object(filename2).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); try { @@ -2949,12 +2933,10 @@ public static void composeObject_test6() throws Exception { PutObjectOptions options = new PutObjectOptions(6 * MB, -1); client.putObject(bucketName, filename1, filename1, options); client.putObject(bucketName, filename2, filename2, options); - ComposeSourceArgs s1 = - ComposeSourceArgs.builder().bucket(bucketName).object(filename1).build(); - ComposeSourceArgs s2 = - ComposeSourceArgs.builder().bucket(bucketName).object(filename2).build(); + ComposeSource s1 = ComposeSource.builder().bucket(bucketName).object(filename1).build(); + ComposeSource s2 = ComposeSource.builder().bucket(bucketName).object(filename2).build(); - List listSourceObjects = new ArrayList(); + List listSourceObjects = new ArrayList(); listSourceObjects.add(s1); listSourceObjects.add(s2); try {