diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot index d62a217e606..e630fe6cdae 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot @@ -20,6 +20,7 @@ Library String Library DateTime Resource ../commonlib.robot Resource commonawslib.robot +Resource mpu_lib.robot Test Timeout 5 minutes Suite Setup Setup Multipart Tests Suite Teardown Teardown Multipart Tests @@ -61,17 +62,8 @@ Test Multipart Upload With Adjusted Length Verify Multipart Upload ${BUCKET} multipart/adjusted_length_${PREFIX} /tmp/part1 /tmp/part2 Test Multipart Upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId -# initiate again - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey - ${nextUploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey + ${nextUploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey Should Not Be Equal ${uploadID} ${nextUploadID} # upload part @@ -79,33 +71,15 @@ Test Multipart Upload # upload we get error entity too small. So, considering further complete # multipart upload, uploading each part as 5MB file, exception is for last part - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} - Should contain ${result} ETag -# override part - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/part1 --upload-id ${nextUploadID} - Should contain ${result} ETag + Upload MPU part ${BUCKET} ${PREFIX}/multipartKey ${nextUploadID} 1 /tmp/part1 + Upload MPU part ${BUCKET} ${PREFIX}/multipartKey ${nextUploadID} 1 /tmp/part1 Test Multipart Upload Complete - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --metadata="custom-key1=custom-value1,custom-key2=custom-value2,gdprEnabled=true" --tagging="tag-key1=tag-value1&tag-key2=tag-value2" - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey1 0 --metadata="custom-key1=custom-value1,custom-key2=custom-value2,gdprEnabled=true" --tagging="tag-key1=tag-value1&tag-key2=tag-value2" -#upload parts - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - ${part1Md5Sum} = Execute md5sum /tmp/part1 | awk '{print $1}' - Should Be Equal As Strings ${eTag1} ${part1Md5Sum} - - Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - ${part2Md5Sum} = Execute md5sum /tmp/part2 | awk '{print $1}' - Should Be Equal As Strings ${eTag2} ${part2Md5Sum} + ${eTag1} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey1 ${uploadID} 1 /tmp/part1 + ${eTag2} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey1 ${uploadID} 2 /tmp/part2 #complete multipart upload without any parts ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 255 @@ -113,12 +87,8 @@ Test Multipart Upload Complete Should contain ${result} must specify at least one part #complete multipart upload - ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey1 - ${resultETag} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 + ${resultETag} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey1 ${uploadID} {ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2} ${expectedResultETag} = Execute echo -n ${eTag1}${eTag2} | md5sum | awk '{print $1}' - Should contain ${result} ETag Should Be Equal As Strings ${resultETag} "${expectedResultETag}-2" #check whether the user defined metadata and parts count can be retrieved @@ -163,116 +133,69 @@ Test Multipart Upload Complete Test Multipart Upload with user defined metadata size larger than 2 KB ${custom_metadata_value} = Generate Random String 3000 - ${result} = Execute AWSS3APICli and checkrc create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/mpuWithLargeMetadata --metadata="custom-key1=${custom_metadata_value}" 255 + ${result} = Initiate MPU ${BUCKET} ${PREFIX}/mpuWithLargeMetadata 255 --metadata="custom-key1=${custom_metadata_value}" Should contain ${result} MetadataTooLarge Should not contain ${result} custom-key1: ${custom_metadata_value} Test Multipart Upload Complete Entity too small - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId - -#upload parts - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 1 --body /tmp/10kb --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --part-number 2 --body /tmp/10kb --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - -#complete multipart upload - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey2 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' 255 + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey2 + ${parts} = Upload MPU parts ${BUCKET} ${PREFIX}/multipartKey2 ${uploadID} /tmp/10kb /tmp/10kb + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey2 ${uploadID} ${parts} 255 Should contain ${result} EntityTooSmall Test Multipart Upload Complete Invalid part errors and complete mpu with few parts - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey3 #complete multipart upload when no parts uploaded - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2} 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1}]' 255 + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=etag1,PartNumber=2},{ETag=etag2,PartNumber=1} 255 Should contain ${result} InvalidPart #upload parts - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - - - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 2 --body /tmp/part1 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - - Execute echo "Part3" > /tmp/part3 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 3 --body /tmp/part3 --upload-id ${uploadID} - ${eTag3} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${eTag1} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} 1 /tmp/part1 + ${eTag2} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} 2 /tmp/part1 + ${eTag3} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} 3 /tmp/part2 #complete multipart upload - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=etag1,PartNumber=1},{ETag=etag2,PartNumber=2} 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=${eTag1},PartNumber=1},{ETag=etag2,PartNumber=2} 255 Should contain ${result} InvalidPart - ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2}]' 255 + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=${eTag1},PartNumber=4},{ETag=etag2,PartNumber=2} 255 Should contain ${result} InvalidPartOrder #complete multipart upload(merge with few parts) - ${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3}]' - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey3 - Should contain ${result} ETag + ${result} = Complete MPU ${BUCKET} ${PREFIX}/multipartKey3 ${uploadID} {ETag=${eTag1},PartNumber=1},{ETag=${eTag3},PartNumber=3} ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 /tmp/${PREFIX}-multipartKey3.result - Execute cat /tmp/part1 /tmp/part3 > /tmp/${PREFIX}-multipartKey3 + Execute cat /tmp/part1 /tmp/part2 > /tmp/${PREFIX}-multipartKey3 Compare files /tmp/${PREFIX}-multipartKey3 /tmp/${PREFIX}-multipartKey3.result ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 1 /tmp/${PREFIX}-multipartKey3-part1.result Compare files /tmp/part1 /tmp/${PREFIX}-multipartKey3-part1.result - ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 3 /tmp/${PREFIX}-multipartKey3-part3.result - Compare files /tmp/part3 /tmp/${PREFIX}-multipartKey3-part3.result + ${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey3 --part-number 3 /tmp/${PREFIX}-multipartKey3-part2.result + Compare files /tmp/part2 /tmp/${PREFIX}-multipartKey3-part2.result Test abort Multipart upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey4 --storage-class REDUCED_REDUNDANCY - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId - - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey4 --upload-id ${uploadID} 0 + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey4 0 --storage-class REDUCED_REDUNDANCY + ${result} = Abort MPU ${BUCKET} ${PREFIX}/multipartKey4 ${uploadID} 0 Test abort Multipart upload with invalid uploadId - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id "random" 255 + ${result} = Abort MPU ${BUCKET} ${PREFIX}/multipartKey5 "random" 255 Upload part with Incorrect uploadID - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey - Execute echo "Multipart upload" > /tmp/testfile - ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey --part-number 1 --body /tmp/testfile --upload-id "random" 255 + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey + ${result} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey "no-such-upload-id" 1 /tmp/10kb 255 Should contain ${result} NoSuchUpload Test list parts #initiate multipart upload - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/multipartKey - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/multipartKey5 #upload parts - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 1 --body /tmp/part1 --upload-id ${uploadID} - ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag - - Execute echo "Part2" > /tmp/part2 - ${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --part-number 2 --body /tmp/part2 --upload-id ${uploadID} - ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0 - Should contain ${result} ETag + ${eTag1} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey5 ${uploadID} 1 /tmp/part1 + ${eTag2} = Upload MPU part ${BUCKET} ${PREFIX}/multipartKey5 ${uploadID} 2 /tmp/part2 #list parts ${result} = Execute AWSS3APICli list-parts --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} @@ -295,7 +218,7 @@ Test list parts Should contain ${result} STANDARD #finally abort it - ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/multipartKey5 --upload-id ${uploadID} 0 + ${result} = Abort MPU ${BUCKET} ${PREFIX}/multipartKey5 ${uploadID} 0 Test Multipart Upload with the simplified aws s3 cp API Execute AWSS3Cli cp /tmp/22mb s3://${BUCKET}/mpyawscli @@ -307,19 +230,14 @@ Test Multipart Upload Put With Copy ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copytest/source --body /tmp/part1 - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copytest/destination - - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} UploadId - + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/copytest/destination ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key ${PREFIX}/copytest/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/${PREFIX}/copytest/source Should contain ${result} ETag Should contain ${result} LastModified ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copytest/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1}]' + Complete MPU ${BUCKET} ${PREFIX}/copytest/destination ${uploadID} {ETag=${eTag1},PartNumber=1} Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copytest/destination /tmp/part-result Compare files /tmp/part1 /tmp/part-result @@ -328,11 +246,7 @@ Test Multipart Upload Put With Copy and range ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination - - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/copyrange/destination ${result} = Execute AWSS3APICli upload-part-copy --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --upload-id ${uploadID} --part-number 1 --copy-source ${BUCKET}/${PREFIX}/copyrange/source --copy-source-range bytes=0-10485757 Should contain ${result} ETag @@ -345,7 +259,7 @@ Test Multipart Upload Put With Copy and range ${eTag2} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' + Complete MPU ${BUCKET} ${PREFIX}/copyrange/destination ${uploadID} {ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2} Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result Compare files /tmp/10mb /tmp/part-result @@ -357,11 +271,7 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince ${result} = Execute AWSS3APICli put-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source --body /tmp/10mb - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination - - ${uploadID} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} UploadId + ${uploadID} = Initiate MPU ${BUCKET} ${PREFIX}/copyrange/destination #calc time-to-sleep from time-last-modified plus a few seconds ${result} = Execute AWSS3APICli head-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/source @@ -396,24 +306,14 @@ Test Multipart Upload Put With Copy and range with IfModifiedSince ${eTag1} = Execute and checkrc echo '${result}' | jq -r '.CopyPartResult.ETag' 0 - - Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]' + Complete MPU ${BUCKET} ${PREFIX}/copyrange/destination ${uploadID} {ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2} Execute AWSS3APICli get-object --bucket ${BUCKET} --key ${PREFIX}/copyrange/destination /tmp/part-result Compare files /tmp/10mb /tmp/part-result Test Multipart Upload list - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key1 - ${uploadID1} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/listtest/key1 - Should contain ${result} UploadId - - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${BUCKET} --key ${PREFIX}/listtest/key2 - ${uploadID2} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - Should contain ${result} ${BUCKET} - Should contain ${result} ${PREFIX}/listtest/key2 - Should contain ${result} UploadId + ${uploadID1} = Initiate MPU ${BUCKET} ${PREFIX}/listtest/key1 + ${uploadID2} = Initiate MPU ${BUCKET} ${PREFIX}/listtest/key2 ${result} = Execute AWSS3APICli list-multipart-uploads --bucket ${BUCKET} --prefix ${PREFIX}/listtest Should contain ${result} ${uploadID1} diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot b/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot index 607a7dee960..a382970a6de 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/bucketdelete.robot @@ -19,6 +19,7 @@ Library OperatingSystem Library String Resource ../commonlib.robot Resource commonawslib.robot +Resource mpu_lib.robot Test Timeout 5 minutes Suite Setup Setup s3 tests @@ -48,17 +49,13 @@ Delete bucket with incomplete multipart uploads [tags] no-bucket-type ${bucket} = Create bucket - # initiate incomplete multipart uploads (multipart upload is initiated but not completed/aborted) - ${initiate_result} = Execute AWSS3APICli create-multipart-upload --bucket ${bucket} --key incomplete-multipartkey - ${uploadID} = Execute echo '${initiate_result}' | jq -r '.UploadId' - Should contain ${initiate_result} ${bucket} - Should contain ${initiate_result} incomplete-multipartkey - Should contain ${initiate_result} UploadId + # initiate incomplete multipart upload (multipart upload is initiated but not completed/aborted) + ${uploadID} = Initiate MPU ${bucket} incomplete-multipartkey # bucket deletion should fail since there is still incomplete multipart upload ${delete_fail_result} = Execute AWSS3APICli and checkrc delete-bucket --bucket ${bucket} 255 Should contain ${delete_fail_result} BucketNotEmpty # after aborting the multipart upload, the bucket deletion should succeed - ${abort_result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${bucket} --key incomplete-multipartkey --upload-id ${uploadID} 0 - ${delete_result} = Execute AWSS3APICli and checkrc delete-bucket --bucket ${bucket} 0 \ No newline at end of file + ${abort_result} = Abort MPU ${bucket} incomplete-multipartkey ${uploadID} + ${delete_result} = Execute AWSS3APICli and checkrc delete-bucket --bucket ${bucket} 0 diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot index 45dee9270bd..44ad919555d 100644 --- a/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot +++ b/hadoop-ozone/dist/src/main/smoketest/s3/commonawslib.robot @@ -172,33 +172,6 @@ Generate random prefix ${random} = Generate Ozone String Set Global Variable ${PREFIX} ${random} -Perform Multipart Upload - [arguments] ${bucket} ${key} @{files} - - ${result} = Execute AWSS3APICli create-multipart-upload --bucket ${bucket} --key ${key} - ${upload_id} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 - - @{etags} = Create List - FOR ${i} ${file} IN ENUMERATE @{files} - ${part} = Evaluate ${i} + 1 - ${result} = Execute AWSS3APICli upload-part --bucket ${bucket} --key ${key} --part-number ${part} --body ${file} --upload-id ${upload_id} - ${etag} = Execute echo '${result}' | jq -r '.ETag' - Append To List ${etags} {ETag=${etag},PartNumber=${part}} - END - - ${parts} = Catenate SEPARATOR=, @{etags} - Execute AWSS3APICli complete-multipart-upload --bucket ${bucket} --key ${key} --upload-id ${upload_id} --multipart-upload 'Parts=[${parts}]' - -Verify Multipart Upload - [arguments] ${bucket} ${key} @{files} - - ${random} = Generate Ozone String - - Execute AWSS3APICli get-object --bucket ${bucket} --key ${key} /tmp/verify${random} - ${tmp} = Catenate @{files} - Execute cat ${tmp} > /tmp/original${random} - Compare files /tmp/original${random} /tmp/verify${random} - Revoke S3 secrets Execute and Ignore Error ozone s3 revokesecret -y Execute and Ignore Error ozone s3 revokesecret -y -u testuser diff --git a/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot b/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot new file mode 100644 index 00000000000..0aaa0affec1 --- /dev/null +++ b/hadoop-ozone/dist/src/main/smoketest/s3/mpu_lib.robot @@ -0,0 +1,105 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +*** Settings *** +Documentation Keywords for Multipart Upload +Library OperatingSystem +Library String +Resource commonawslib.robot + +*** Keywords *** + +Initiate MPU + [arguments] ${bucket} ${key} ${expected_rc}=0 ${opts}=${EMPTY} + + ${result} = Execute AWSS3APICli and checkrc create-multipart-upload --bucket ${bucket} --key ${key} ${opts} ${expected_rc} + IF '${expected_rc}' == '0' + Should contain ${result} ${bucket} + Should contain ${result} ${key} + ${upload_id} = Execute and checkrc echo '${result}' | jq -r '.UploadId' 0 + RETURN ${upload_id} + ELSE + RETURN ${result} + END + + +Upload MPU part + [arguments] ${bucket} ${key} ${upload_id} ${part} ${file} ${expected_rc}=0 + + ${result} = Execute AWSS3APICli and checkrc upload-part --bucket ${bucket} --key ${key} --part-number ${part} --body ${file} --upload-id ${upload_id} ${expected_rc} + IF '${expected_rc}' == '0' + Should contain ${result} ETag + ${etag} = Execute echo '${result}' | jq -r '.ETag' + ${md5sum} = Execute md5sum ${file} | awk '{print $1}' + Should Be Equal As Strings ${etag} ${md5sum} + RETURN ${etag} + ELSE + RETURN ${result} + END + + +Complete MPU + [arguments] ${bucket} ${key} ${upload_id} ${parts} ${expected_rc}=0 + + ${result} = Execute AWSS3APICli and checkrc complete-multipart-upload --bucket ${bucket} --key ${key} --upload-id ${upload_id} --multipart-upload 'Parts=[${parts}]' ${expected_rc} + IF '${expected_rc}' == '0' + Should contain ${result} ${bucket} + Should contain ${result} ${key} + Should contain ${result} ETag + ${etag} = Execute echo '${result}' | jq -r '.ETag' + RETURN ${etag} + ELSE + RETURN ${result} + END + + +Abort MPU + [arguments] ${bucket} ${key} ${upload_id} ${expected_rc}=0 + + ${result} = Execute AWSS3APICli and checkrc abort-multipart-upload --bucket ${bucket} --key ${key} --upload-id ${upload_id} ${expected_rc} + + +Upload MPU parts + [arguments] ${bucket} ${key} ${upload_id} @{files} + + @{etags} = Create List + FOR ${i} ${file} IN ENUMERATE @{files} + ${part} = Evaluate ${i} + 1 + ${etag} = Upload MPU part ${bucket} ${key} ${upload_id} ${part} ${file} + Append To List ${etags} {ETag=${etag},PartNumber=${part}} + END + ${parts} = Catenate SEPARATOR=, @{etags} + + RETURN ${parts} + + +Perform Multipart Upload + [arguments] ${bucket} ${key} @{files} + + ${upload_id} = Initiate MPU ${bucket} ${key} + ${parts} = Upload MPU parts ${bucket} ${key} ${upload_id} @{files} + Complete MPU ${bucket} ${key} ${upload_id} ${parts} + + +Verify Multipart Upload + [arguments] ${bucket} ${key} @{files} + + ${random} = Generate Ozone String + + Execute AWSS3APICli get-object --bucket ${bucket} --key ${key} /tmp/verify${random} + ${tmp} = Catenate @{files} + Execute cat ${tmp} > /tmp/original${random} + Compare files /tmp/original${random} /tmp/verify${random} +