Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

HDDS-9680. Use md5 hash of multipart object part's content as ETag #5668

Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
66cee2b
HDDS-9680. provide a loaded key part's ETag as an md5 hash (S3G multi…
Nov 14, 2023
fd3d762
HDDS-9680. tiny refactoring - move eTag computer to static variable i…
Nov 27, 2023
7e0ef01
Update OmClientProtocol.proto
vtutrinov Nov 30, 2023
d20291d
Merge remote-tracking branch 'origin/master' into HDDS-9680-multipart…
adoroszlai Dec 26, 2023
0a17eb3
Merge remote-tracking branch 'origin/master' into HDDS-9680-multipart…
adoroszlai Jan 5, 2024
eb731bd
Merge branch 'master' into HDDS-9680-multipart-uploaded-part-eTag-imp…
kerneltime Jan 11, 2024
0b36b8a
Merge branch 'master' into HDDS-9680-multipart-uploaded-part-eTag-imp…
kerneltime Jan 16, 2024
060a9f8
fix compile error
adoroszlai Jan 17, 2024
48a1af2
Merge remote-tracking branch 'origin/master' into HDDS-9680-multipart…
adoroszlai Jan 17, 2024
5eea2be
HDDS-9680. support ETag as an md5-hash for multipart upload parts req…
Jan 28, 2024
2647df0
HDDS-9680. provide OzoneConst.ETAG constant and replace "ETag" occurr…
Jan 28, 2024
2790d2a
HDDS-9680. fix checkstyle issues
Jan 28, 2024
1b47c00
HDDS-9680. revert ozone client interface proto.lock changes
Jan 28, 2024
fd71ce1
HDDS-9680. fix unit tests issues
Jan 29, 2024
72ecefe
HDDS-9680. makepartName of MultipartCommitUploadPartResponse as optio…
Jan 29, 2024
255d7ab
HDDS-9680. improve an eTag property reading from the key metadata, fi…
Jan 29, 2024
d9429a1
Merge remote-tracking branch 'origin/master' into HDDS-9680-multipart…
adoroszlai Feb 6, 2024
6cde5b3
Merge remote-tracking branch 'origin/master' into HDDS-9680-multipart…
adoroszlai Feb 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -104,12 +104,15 @@ public static class PartInfo {
private String partName;
private long modificationTime;
private long size;
private String eTag;

public PartInfo(int number, String name, long time, long size) {
public PartInfo(int number, String name, long time, long size,
String eTag) {
this.partNumber = number;
this.partName = name;
this.modificationTime = time;
this.size = size;
this.eTag = eTag;
}

public int getPartNumber() {
Expand All @@ -127,5 +130,9 @@ public long getModificationTime() {
public long getSize() {
return size;
}

public String getETag() {
return eTag;
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1944,7 +1944,8 @@ public OzoneMultipartUploadPartListParts listParts(String volumeName,
ozoneMultipartUploadPartListParts.addPart(
new OzoneMultipartUploadPartListParts.PartInfo(
omPartInfo.getPartNumber(), omPartInfo.getPartName(),
omPartInfo.getModificationTime(), omPartInfo.getSize()));
omPartInfo.getModificationTime(), omPartInfo.getSize(),
omPartInfo.geteTag()));
}
return ozoneMultipartUploadPartListParts;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,15 @@ public class OmMultipartCommitUploadPartInfo {

private final String partName;

public OmMultipartCommitUploadPartInfo(String name) {
this.partName = name;
private final String eTag;

public OmMultipartCommitUploadPartInfo(String partName, String eTag) {
this.partName = partName;
this.eTag = eTag;
}

public String getETag() {
return eTag;
}

public String getPartName() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -56,8 +56,8 @@ public Map<Integer, String> getMultipartMap() {
*/
public List<Part> getPartsList() {
List<Part> partList = new ArrayList<>();
multipartMap.forEach((partNumber, partName) -> partList.add(Part
.newBuilder().setPartName(partName).setPartNumber(partNumber).build()));
multipartMap.forEach((partNumber, eTag) -> partList.add(Part
.newBuilder().setETag(eTag).setPartNumber(partNumber).build()));
vtutrinov marked this conversation as resolved.
Show resolved Hide resolved
return partList;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ public void addPartList(List<OmPartInfo> partInfos) {
public void addProtoPartList(List<PartInfo> partInfos) {
partInfos.forEach(partInfo -> partInfoList.add(new OmPartInfo(
partInfo.getPartNumber(), partInfo.getPartName(),
partInfo.getModificationTime(), partInfo.getSize())));
partInfo.getModificationTime(), partInfo.getSize(),
partInfo.getETag())));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,15 @@ public class OmPartInfo {
private String partName;
private long modificationTime;
private long size;
private String eTag;

public OmPartInfo(int number, String name, long time, long size) {
public OmPartInfo(int number, String name, long time, long size,
String eTag) {
this.partNumber = number;
this.partName = name;
this.modificationTime = time;
this.size = size;
this.eTag = eTag;
}

public int getPartNumber() {
Expand All @@ -52,9 +55,13 @@ public long getSize() {
return size;
}

public String geteTag() {
vtutrinov marked this conversation as resolved.
Show resolved Hide resolved
return eTag;
}

public PartInfo getProto() {
return PartInfo.newBuilder().setPartNumber(partNumber).setPartName(partName)
.setModificationTime(modificationTime)
.setSize(size).build();
.setSize(size).setETag(eTag).build();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -1627,7 +1627,8 @@ public OmMultipartCommitUploadPartInfo commitMultipartUploadPart(
.getCommitMultiPartUploadResponse();

OmMultipartCommitUploadPartInfo info = new
OmMultipartCommitUploadPartInfo(response.getPartName());
OmMultipartCommitUploadPartInfo(response.getPartName(),
response.getETag());
return info;
}

Expand Down
33 changes: 20 additions & 13 deletions hadoop-ozone/dist/src/main/smoketest/s3/MultipartUpload.robot
Original file line number Diff line number Diff line change
Expand Up @@ -94,21 +94,28 @@ Test Multipart Upload Complete
Should contain ${result} UploadId

#upload parts
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
Run Keyword Create Random file 5
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 1 --body /tmp/part1 --upload-id ${uploadID}
${eTag1} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${part1Md5Sum} = Execute md5sum /tmp/part1 | awk '{print $1}'
Should Be Equal As Strings ${eTag1} ${part1Md5Sum}

Execute echo "Part2" > /tmp/part2
${result} = Execute AWSS3APICli upload-part --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --part-number 2 --body /tmp/part2 --upload-id ${uploadID}
${eTag2} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
Should contain ${result} ETag
${part2Md5Sum} = Execute md5sum /tmp/part2 | awk '{print $1}'
Should Be Equal As Strings ${eTag2} ${part2Md5Sum}

#complete multipart upload
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Should contain ${result} ${BUCKET}
Should contain ${result} ${PREFIX}/multipartKey1
Should contain ${result} ETag
${result} = Execute AWSS3APICli complete-multipart-upload --upload-id ${uploadID} --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 --multipart-upload 'Parts=[{ETag=${eTag1},PartNumber=1},{ETag=${eTag2},PartNumber=2}]'
Should contain ${result} ${BUCKET}
Should contain ${result} ${PREFIX}/multipartKey1
${resultETag} = Execute and checkrc echo '${result}' | jq -r '.ETag' 0
${expectedResultETag} = Execute echo -n ${eTag1}${eTag2} | md5sum | awk '{print $1}'
Should contain ${result} ETag
Should Be Equal As Strings ${resultETag} "${expectedResultETag}-2"

#read file and check the key
${result} = Execute AWSS3ApiCli get-object --bucket ${BUCKET} --key ${PREFIX}/multipartKey1 /tmp/${PREFIX}-multipartKey1.result
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@

package org.apache.hadoop.fs.ozone;

import javax.xml.bind.DatatypeConverter;
import org.apache.commons.lang3.RandomStringUtils;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
Expand Down Expand Up @@ -53,6 +54,7 @@
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
Expand Down Expand Up @@ -304,10 +306,13 @@ public void testMPUFailDuetoDirectoryCreationBeforeComplete()

// This should succeed, as we check during creation of part or during
// complete MPU.
ozoneOutputStream.getMetadata().put("ETag",
DatatypeConverter.printHexBinary(MessageDigest.getInstance("Md5")
vtutrinov marked this conversation as resolved.
Show resolved Hide resolved
.digest(b)).toLowerCase());
ozoneOutputStream.close();

Map<Integer, String> partsMap = new HashMap<>();
partsMap.put(1, ozoneOutputStream.getCommitUploadPartInfo().getPartName());
partsMap.put(1, ozoneOutputStream.getCommitUploadPartInfo().getETag());

// Should fail, as we have directory with same name.
try {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,7 @@ private CompleteMultipartUploadRequest.Part uploadPart(String uploadID,

CompleteMultipartUploadRequest.Part
part = new CompleteMultipartUploadRequest.Part();
part.seteTag(response.getHeaderString("ETag"));
part.setETag(response.getHeaderString("ETag"));
part.setPartNumber(partNumber);
return part;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import java.net.URI;
import java.nio.ByteBuffer;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
import java.util.ArrayList;
Expand All @@ -34,6 +35,7 @@
import java.util.UUID;

import com.google.common.cache.Cache;
import javax.xml.bind.DatatypeConverter;
import org.apache.hadoop.conf.StorageUnit;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
Expand Down Expand Up @@ -96,6 +98,8 @@

class TestOzoneAtRestEncryption {

private static final String ETAG = "ETag";

private static MiniOzoneCluster cluster = null;
private static MiniKMS miniKMS;
private static OzoneClient ozClient = null;
Expand All @@ -117,6 +121,7 @@ class TestOzoneAtRestEncryption {
private static final int DEFAULT_CRYPTO_BUFFER_SIZE = 8 * 1024; // 8KB
// (this is the default Crypto Buffer size as determined by the config
// hadoop.security.crypto.buffer.size)
private static MessageDigest eTagProvider;

@BeforeAll
static void init() throws Exception {
Expand Down Expand Up @@ -166,6 +171,7 @@ static void init() throws Exception {

// create test key
createKey(TEST_KEY, cluster.getOzoneManager().getKmsProvider(), conf);
eTagProvider = MessageDigest.getInstance("Md5");
}

@AfterAll
Expand Down Expand Up @@ -629,29 +635,35 @@ private String uploadStreamPart(OzoneBucket bucket, String keyName,

ByteBuffer dataBuffer = ByteBuffer.wrap(data);
multipartStreamKey.write(dataBuffer, 0, length);
multipartStreamKey.getMetadata().put(ETAG,
DatatypeConverter.printHexBinary(eTagProvider.digest(data))
.toLowerCase());
multipartStreamKey.close();

OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
multipartStreamKey.getCommitUploadPartInfo();

assertNotNull(omMultipartCommitUploadPartInfo);
assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
return omMultipartCommitUploadPartInfo.getPartName();
assertNotNull(omMultipartCommitUploadPartInfo.getETag());
return omMultipartCommitUploadPartInfo.getETag();
}

private String uploadPart(OzoneBucket bucket, String keyName,
String uploadID, int partNumber, byte[] data) throws Exception {
OzoneOutputStream ozoneOutputStream = bucket.createMultipartKey(keyName,
data.length, partNumber, uploadID);
ozoneOutputStream.write(data, 0, data.length);
ozoneOutputStream.getMetadata().put(ETAG,
DatatypeConverter.printHexBinary(eTagProvider.digest(data))
.toLowerCase());
ozoneOutputStream.close();

OmMultipartCommitUploadPartInfo omMultipartCommitUploadPartInfo =
ozoneOutputStream.getCommitUploadPartInfo();

assertNotNull(omMultipartCommitUploadPartInfo);
assertNotNull(omMultipartCommitUploadPartInfo.getPartName());
return omMultipartCommitUploadPartInfo.getPartName();
assertNotNull(omMultipartCommitUploadPartInfo.getETag());
return omMultipartCommitUploadPartInfo.getETag();
}

private void completeMultipartUpload(OzoneBucket bucket, String keyName,
Expand Down
Loading
Loading