Skip to content

Commit

Permalink
Merge pull request #1316 from adobe/1215-add-listobjectversions
Browse files Browse the repository at this point in the history
Add ListObjectVersions API
  • Loading branch information
afranken authored Nov 6, 2023
2 parents c34d323 + fe8d6d3 commit 5fd0be2
Show file tree
Hide file tree
Showing 9 changed files with 447 additions and 111 deletions.
34 changes: 23 additions & 11 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -129,23 +129,35 @@ Version 3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Jav
* Better description of S3Mock API usage and SDK usage (fixes #219, #125, #1196)
* Presigned URLs were working all along.
* Added documentation and tests.
* Version updates
* Bump aws-v2.version from 2.20.115 to 2.20.146
* Bump com.amazonaws:aws-java-sdk-s3 from 1.12.519 to 1.12.548
* Bump spring-boot.version from 3.1.0 to 3.1.3
* Bump alpine from 3.18.2 to 3.18.3 in /docker
* Bump testcontainers.version from 1.18.3 to 1.19.0
* Bump kotlin.version from 1.9.0 to 1.9.10
* Bump com.puppycrawl.tools:checkstyle from 10.12.2 to 10.12.3
* Add ListObjectVersions API - dummy implementation (fixes #1215)
* Version updates
* Bump aws-v2.version from 2.20.115 to 2.21.14
* Bump com.amazonaws:aws-java-sdk-s3 from 1.12.519 to 1.12.580
* Bump spring-boot.version from 3.1.0 to 3.1.5
* Bump alpine from 3.18.2 to 3.18.4 in /docker
* Bump testcontainers.version from 1.18.3 to 1.19.1
* Bump kotlin.version from 1.9.0 to 1.9.20
* Bump commons-io:commons-io from 2.13.0 to 2.15.0
* Bump com.puppycrawl.tools:checkstyle from 10.12.2 to 10.12.4
* Bump io.fabric8:docker-maven-plugin from 0.43.2 to 0.43.4
* Bump org.apache.maven.plugins:maven-enforcer-plugin from 3.3.0 to 3.4.1
* Bump org.apache.maven.plugins:maven-javadoc-plugin from 3.5.0 to 3.6.0
* Bump org.apache.maven.plugins:maven-dependency-plugin from 3.6.0 to 3.6.1
* Bump org.apache.maven.plugins:maven-surefire-plugin from 3.1.2 to 3.2.1
* Bump org.apache.maven.plugins:maven-failsafe-plugin from 3.1.2 to 3.2.1
* Bump org.apache.maven.plugins:maven-checkstyle-plugin from 3.3.0 to 3.3.1
* Bump org.apache.maven.plugins:maven-clean-plugin from 3.3.1 to 3.3.2
* Bump license-maven-plugin-git.version from 4.2 to 4.3
* Bump org.jacoco:jacoco-maven-plugin from 0.8.10 to 0.8.11
* Bump org.mockito.kotlin:mockito-kotlin from 5.0.0 to 5.1.0
* Bump actions/checkout from 3.5.3 to 4.0.0
* Bump actions/setup-java from 3.12.0 to 3.13.0
* Bump actions/checkout from 3.5.3 to 4.1.1
* Bump actions/upload-artifact from 3.1.2 to 3.1.3
* Bump actions/dependency-review-action from 3.0.6 to 3.1.0
* Bump github/codeql-action from 2.21.2 to 2.21.5
* Bump github/codeql-action from 2.21.2 to 2.22.5
* Bump docker/setup-qemu-action from 2.2.0 to 3.0.0
* Bump step-security/harden-runner from 2.5.0 to 2.5.1
* Bump step-security/harden-runner from 2.5.0 to 2.6.0
* Bump ossf/scorecard-action from 2.2.0 to 2.3.1

## 3.1.0
3.x is JDK17 LTS bytecode compatible, with Docker and JUnit / direct Java integration.
Expand Down
198 changes: 99 additions & 99 deletions README.md

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Copyright 2017-2023 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.adobe.testing.s3mock.its

import org.assertj.core.api.Assertions.assertThat
import org.assertj.core.groups.Tuple
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInfo
import software.amazon.awssdk.core.sync.RequestBody
import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm
import software.amazon.awssdk.services.s3.model.ListObjectVersionsRequest
import software.amazon.awssdk.services.s3.model.ObjectVersion
import software.amazon.awssdk.services.s3.model.PutObjectRequest
import java.io.File

internal class ListObjectVersionsV2IT : S3TestBase() {

@Test
fun testPutObjects_listObjectVersions(testInfo: TestInfo) {
val uploadFile = File(UPLOAD_FILE_NAME)
val bucketName = givenBucketV2(testInfo)

s3ClientV2.putObject(
PutObjectRequest.builder()
.bucket(bucketName).key("$UPLOAD_FILE_NAME-1")
.checksumAlgorithm(ChecksumAlgorithm.SHA256)
.build(),
RequestBody.fromFile(uploadFile)
)

s3ClientV2.putObject(
PutObjectRequest.builder()
.bucket(bucketName).key("$UPLOAD_FILE_NAME-2")
.checksumAlgorithm(ChecksumAlgorithm.SHA256)
.build(),
RequestBody.fromFile(uploadFile)
)

val listObjectVersionsResponse = s3ClientV2.listObjectVersions(
ListObjectVersionsRequest.builder()
.bucket(bucketName)
.build()
)

assertThat(listObjectVersionsResponse.versions())
.hasSize(2)
.extracting(ObjectVersion::checksumAlgorithm)
.containsOnly(
Tuple(arrayListOf(ChecksumAlgorithm.SHA256)),
Tuple(arrayListOf(ChecksumAlgorithm.SHA256))
)
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
import static com.adobe.testing.s3mock.util.AwsHttpHeaders.X_AMZ_BUCKET_OBJECT_LOCK_ENABLED;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.CONTINUATION_TOKEN;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.ENCODING_TYPE;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.KEY_MARKER;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.LIFECYCLE;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.LIST_TYPE_V2;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.LOCATION;
Expand All @@ -28,14 +29,18 @@
import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_LOCATION;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_OBJECT_LOCK;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_UPLOADS;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.NOT_VERSIONS;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.OBJECT_LOCK;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.START_AFTER;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSIONS;
import static com.adobe.testing.s3mock.util.AwsHttpParameters.VERSION_ID_MARKER;
import static org.springframework.http.MediaType.APPLICATION_XML_VALUE;

import com.adobe.testing.s3mock.dto.BucketLifecycleConfiguration;
import com.adobe.testing.s3mock.dto.ListAllMyBucketsResult;
import com.adobe.testing.s3mock.dto.ListBucketResult;
import com.adobe.testing.s3mock.dto.ListBucketResultV2;
import com.adobe.testing.s3mock.dto.ListVersionsResult;
import com.adobe.testing.s3mock.dto.LocationConstraint;
import com.adobe.testing.s3mock.dto.ObjectLockConfiguration;
import com.adobe.testing.s3mock.service.BucketService;
Expand Down Expand Up @@ -350,7 +355,8 @@ public ResponseEntity<LocationConstraint> getBucketLocation(
NOT_OBJECT_LOCK,
NOT_LIST_TYPE,
NOT_LIFECYCLE,
NOT_LOCATION
NOT_LOCATION,
NOT_VERSIONS
},
produces = APPLICATION_XML_VALUE
)
Expand Down Expand Up @@ -412,4 +418,49 @@ public ResponseEntity<ListBucketResultV2> listObjectsV2(

return ResponseEntity.ok(listBucketResultV2);
}

/**
* Retrieve list of versions of an object of a bucket.
* <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html">API Reference</a>
*
* @param bucketName {@link String} set bucket name
* @param prefix {@link String} find object names they start with prefix
* @param startAfter {@link String} return key names after a specific object key in your key
* space
* @param maxKeys {@link Integer} set maximum number of keys to be returned
* @param continuationToken {@link String} pagination token returned by previous request
*
* @return {@link ListVersionsResult} a list of objects in Bucket
*/
@GetMapping(
value = {
//AWS SDK V2 pattern
"/{bucketName:.+}",
//AWS SDK V1 pattern
"/{bucketName:.+}/"
},
params = {
VERSIONS
},
produces = APPLICATION_XML_VALUE
)
public ResponseEntity<ListVersionsResult> listObjectVersions(
@PathVariable String bucketName,
@RequestParam(required = false) String prefix,
@RequestParam(required = false) String delimiter,
@RequestParam(name = KEY_MARKER, required = false) String keyMarker,
@RequestParam(name = VERSION_ID_MARKER, required = false) String versionIdMarker,
@RequestParam(name = ENCODING_TYPE, required = false) String encodingType,
@RequestParam(name = START_AFTER, required = false) String startAfter,
@RequestParam(name = MAX_KEYS, defaultValue = "1000", required = false) Integer maxKeys,
@RequestParam(name = CONTINUATION_TOKEN, required = false) String continuationToken) {
bucketService.verifyBucketExists(bucketName);
bucketService.verifyMaxKeys(maxKeys);
bucketService.verifyEncodingType(encodingType);
var listVersionsResult =
bucketService.listVersions(bucketName, prefix, delimiter, encodingType, startAfter,
maxKeys, continuationToken, keyMarker, versionIdMarker);

return ResponseEntity.ok(listVersionsResult);
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
/*
* Copyright 2017-2023 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.adobe.testing.s3mock.dto;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;

/**
* <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteMarkerEntry.html">API Reference</a>.
*/
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public record DeleteMarkerEntry(
@JsonProperty("IsLatest")
Boolean isLatest,
@JsonProperty("Key")
String key,
@JsonProperty("LastModified")
String lastModified,
@JsonProperty("Owner")
Owner owner,
@JsonProperty("VersionId")
String versionId
) {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
/*
* Copyright 2017-2023 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.adobe.testing.s3mock.dto;

import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.annotation.JsonRootName;
import com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlElementWrapper;
import java.util.List;

/**
* Represents a result of listing object versions that reside in a Bucket.
* <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectVersions.html">API Reference</a>
*/
@JsonRootName("ListBucketResult")
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public record ListVersionsResult(
@JsonProperty("Name")
String name,
@JsonProperty("Prefix")
String prefix,
@JsonProperty("MaxKeys")
int maxKeys,
@JsonProperty("IsTruncated")
boolean isTruncated,
@JsonProperty("CommonPrefixes")
@JacksonXmlElementWrapper(useWrapping = false)
List<Prefix> commonPrefixes,
@JsonProperty("Delimiter")
String delimiter,
@JsonProperty("EncodingType")
String encodingType,
@JsonProperty("KeyMarker")
String keyMarker,
@JsonProperty("VersionIdMarker")
String versionIdMarker,
@JsonProperty("NextKeyMarker")
String nextKeyMarker,
@JsonProperty("NextVersionIdMarker")
String nextVersionIdMarker,
@JsonProperty("Version")
@JacksonXmlElementWrapper(useWrapping = false)
List<ObjectVersion> objectVersions,
@JsonProperty("DeleteMarker")
@JacksonXmlElementWrapper(useWrapping = false)
List<DeleteMarkerEntry> deleteMarkers

) {

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
/*
* Copyright 2017-2023 Adobe.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package com.adobe.testing.s3mock.dto;

import static com.adobe.testing.s3mock.util.EtagUtil.normalizeEtag;

import com.adobe.testing.s3mock.store.S3ObjectMetadata;
import com.fasterxml.jackson.annotation.JsonInclude;
import com.fasterxml.jackson.annotation.JsonProperty;

/**
* <a href="https://docs.aws.amazon.com/AmazonS3/latest/API/API_ObjectVersion.html">API Reference</a>.
*/
@JsonInclude(JsonInclude.Include.NON_EMPTY)
public record ObjectVersion(
@JsonProperty("Key")
String key,
@JsonProperty("LastModified")
String lastModified,
@JsonProperty("ETag")
String etag,
@JsonProperty("Size")
String size,
@JsonProperty("StorageClass")
StorageClass storageClass,
@JsonProperty("Owner")
Owner owner,
@JsonProperty("ChecksumAlgorithm")
ChecksumAlgorithm checksumAlgorithm,
@JsonProperty("IsLatest")
Boolean isLatest,
@JsonProperty("VersionId")
String versionId
) {

public ObjectVersion {
etag = normalizeEtag(etag);
}

public static ObjectVersion from(S3ObjectMetadata s3ObjectMetadata) {
return new ObjectVersion(s3ObjectMetadata.key(),
s3ObjectMetadata.modificationDate(), s3ObjectMetadata.etag(),
s3ObjectMetadata.size(), StorageClass.STANDARD, Owner.DEFAULT_OWNER,
s3ObjectMetadata.checksumAlgorithm(), true, "staticVersion");
}

public static ObjectVersion from(S3Object s3Object) {
return new ObjectVersion(s3Object.key(),
s3Object.lastModified(), s3Object.etag(),
s3Object.size(), StorageClass.STANDARD, Owner.DEFAULT_OWNER,
s3Object.checksumAlgorithm(), true, "staticVersion");
}
}
Loading

0 comments on commit 5fd0be2

Please sign in to comment.