From 403f50f98980647d3550226490662fff2ef7c0d8 Mon Sep 17 00:00:00 2001 From: Bhavay Pahuja Date: Tue, 21 Nov 2023 11:30:46 +0530 Subject: [PATCH 1/7] HADOOP-14837: Glacier read restored objects support --- .../org/apache/hadoop/fs/s3a/Constants.java | 6 + .../org/apache/hadoop/fs/s3a/Listing.java | 5 +- .../apache/hadoop/fs/s3a/S3AFileSystem.java | 12 ++ .../fs/s3a/S3ObjectStorageClassFilter.java | 84 ++++++++ .../hadoop/fs/s3a/api/RequestFactory.java | 13 ++ .../fs/s3a/impl/RequestFactoryImpl.java | 21 ++ .../hadoop/fs/s3a/impl/StoreContext.java | 16 +- .../fs/s3a/impl/StoreContextBuilder.java | 12 +- .../site/markdown/tools/hadoop-aws/index.md | 14 ++ .../ITestS3AReadRestoredGlacierObjects.java | 179 ++++++++++++++++++ 10 files changed, 358 insertions(+), 4 deletions(-) create mode 100644 hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java create mode 100644 hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 67df37e5ebfcd..9c19335bc123e 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1519,6 +1519,12 @@ private Constants() { */ public static final int DEFAULT_PREFETCH_MAX_BLOCKS_COUNT = 4; + /** + * Read Restored Glacier objects config. + * Value = {@value} + */ + public static final String READ_RESTORED_GLACIER_OBJECTS = "fs.s3a.glacier.read.restored.objects"; + /** * The bucket region header. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index e0868a2e13087..92b8ee3c0edcf 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -76,6 +76,7 @@ public class Listing extends AbstractStoreOperation { private static final Logger LOG = S3AFileSystem.LOG; private final boolean isCSEEnabled; + private final S3ObjectStorageClassFilter s3ObjectStorageClassFilter; static final FileStatusAcceptor ACCEPT_ALL_BUT_S3N = new AcceptAllButS3nDirs(); @@ -87,6 +88,7 @@ public Listing(ListingOperationCallbacks listingOperationCallbacks, super(storeContext); this.listingOperationCallbacks = listingOperationCallbacks; this.isCSEEnabled = storeContext.isCSEEnabled(); + this.s3ObjectStorageClassFilter = storeContext.getS3ObjectsStorageClassFilter(); } /** @@ -462,7 +464,8 @@ private boolean buildNextStatusBatch(S3ListResult objects) { LOG.debug("{}: {}", keyPath, stringify(s3Object)); } // Skip over keys that are ourselves and old S3N _$folder$ files - if (acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { + // Handle Glacier Storage Class objects based on the config fs.s3a.glacier.read.restored.objects value set + if ( s3ObjectStorageClassFilter.getFilter().apply(s3Object) && acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { S3AFileStatus status = createFileStatus(keyPath, s3Object, listingOperationCallbacks.getDefaultBlockSize(keyPath), getStoreContext().getUsername(), diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 0e2ae0f74dd0a..291dc316edaa9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -444,6 +444,12 @@ public class S3AFileSystem extends FileSystem implements StreamCapabilities, */ private boolean isCSEEnabled; + /** + * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the + * {@code fs.s3a.glacier.read.restored.objects} configuration. + */ + private S3ObjectStorageClassFilter s3ObjectStorageClassFilter; + /** * Bucket AccessPoint. */ @@ -585,6 +591,11 @@ public void initialize(URI name, Configuration originalConf) s3aInternals = createS3AInternals(); + s3ObjectStorageClassFilter = Optional.of(conf.getTrimmed(READ_RESTORED_GLACIER_OBJECTS, + S3ObjectStorageClassFilter.READ_ALL.toString())) + .map(String::toUpperCase) + .map(S3ObjectStorageClassFilter::valueOf).get(); + // look for encryption data // DT Bindings may override this setEncryptionSecrets( @@ -5686,6 +5697,7 @@ public StoreContext createStoreContext() { .setContextAccessors(new ContextAccessorsImpl()) .setAuditor(getAuditor()) .setEnableCSE(isCSEEnabled) + .setS3ObjectStorageClassFilter(s3ObjectStorageClassFilter) .build(); } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java new file mode 100644 index 0000000000000..dcdae697f61f1 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a; + +import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; +import java.util.Set; +import java.util.function.Function; +import software.amazon.awssdk.services.s3.model.ObjectStorageClass; +import software.amazon.awssdk.services.s3.model.S3Object; + + +/** + * 
+ * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link S3AFileSystem}
+ * The config can have 3 values:
+ * {@code READ_ALL}: Retrieval of Glacier files will fail with InvalidObjectStateException: The operation is not valid for the object's storage class.
+ * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3 Objects which are tagged with Glacier storage classes and retrieve the others.
+ * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored status of the Glacier object will be checked, if restored the objects would be read like normal S3 objects else they will be ignored as the objects would not have been retrieved from the S3 Glacier.
+ *
+ */ +public enum S3ObjectStorageClassFilter { + READ_ALL(o -> true), + SKIP_ALL_GLACIER(S3ObjectStorageClassFilter::isNotGlacierObject), + READ_RESTORED_GLACIER_OBJECTS(S3ObjectStorageClassFilter::isCompletedRestoredObject); + + private static final Set GLACIER_STORAGE_CLASSES = Sets.newHashSet(ObjectStorageClass.GLACIER, ObjectStorageClass.DEEP_ARCHIVE); + + private final Function filter; + + S3ObjectStorageClassFilter(Function filter) { + this.filter = filter; + } + + /** + * Checks if the s3 object is not an object with a storage class of glacier/deep_archive + * @param object s3 object + * @return if the s3 object is not an object with a storage class of glacier/deep_archive + */ + private static boolean isNotGlacierObject(S3Object object) { + return !GLACIER_STORAGE_CLASSES.contains(object.storageClass()); + } + + /** + * Checks if the s3 object is an object with a storage class of glacier/deep_archive + * @param object s3 object + * @return if the s3 object is an object with a storage class of glacier/deep_archive + */ + private static boolean isGlacierObject(S3Object object) { + return GLACIER_STORAGE_CLASSES.contains(object.storageClass()); + } + + /** + * Checks if the s3 object is completely restored + * @param object s3 object + * @return if the s3 object is completely restored + */ + private static boolean isCompletedRestoredObject(S3Object object) { + if(isGlacierObject(object)) { + return object.restoreStatus() != null && !object.restoreStatus().isRestoreInProgress(); + } + return true; + } + + public Function getFilter() { + return filter; + } + +} diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java index 73ad137a86d3c..2e70b0db089dc 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java @@ -37,7 +37,9 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.Tier; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import org.apache.hadoop.classification.InterfaceAudience; @@ -251,4 +253,15 @@ ListObjectsV2Request.Builder newListObjectsV2RequestBuilder(String key, DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder( List keysToDelete); + /** + * Create a request builder to initiate a restore of Glacier object + * @param key object to restore + * @param tier glacier retrieval tier at which the restore will be processed. + * @param expirationDays lifetime of the active restored copy in days. + * @return the request builder + */ + RestoreObjectRequest.Builder newRestoreObjectRequestBuilder(String key, + Tier tier, + int expirationDays); + } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index c91324da7cb15..15b2eefd74156 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -34,6 +34,7 @@ import software.amazon.awssdk.services.s3.model.DeleteObjectsRequest; import software.amazon.awssdk.services.s3.model.GetObjectRequest; import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest; +import software.amazon.awssdk.services.s3.model.GlacierJobParameters; import software.amazon.awssdk.services.s3.model.HeadBucketRequest; import software.amazon.awssdk.services.s3.model.HeadObjectRequest; import software.amazon.awssdk.services.s3.model.HeadObjectResponse; @@ -42,9 +43,13 @@ import software.amazon.awssdk.services.s3.model.ListObjectsV2Request; import software.amazon.awssdk.services.s3.model.MetadataDirective; import software.amazon.awssdk.services.s3.model.ObjectIdentifier; +import software.amazon.awssdk.services.s3.model.OptionalObjectAttributes; import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; +import software.amazon.awssdk.services.s3.model.RestoreRequest; import software.amazon.awssdk.services.s3.model.ServerSideEncryption; import software.amazon.awssdk.services.s3.model.StorageClass; +import software.amazon.awssdk.services.s3.model.Tier; import software.amazon.awssdk.services.s3.model.UploadPartRequest; import software.amazon.awssdk.utils.Md5Utils; import org.apache.hadoop.util.Preconditions; @@ -609,6 +614,7 @@ public ListObjectsV2Request.Builder newListObjectsV2RequestBuilder( final ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() .bucket(bucket) .maxKeys(maxKeys) + .optionalObjectAttributes(OptionalObjectAttributes.RESTORE_STATUS) // Optional Attribute to get the Restored Status of the Glacier Objects .prefix(key); if (delimiter != null) { @@ -632,6 +638,21 @@ public DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder( .delete(d -> d.objects(keysToDelete).quiet(!LOG.isTraceEnabled()))); } + @Override + public RestoreObjectRequest.Builder newRestoreObjectRequestBuilder(String key, + Tier tier, + int expirationDays) { + return prepareRequest(RestoreObjectRequest + .builder() + .bucket(bucket) + .key(key) + .restoreRequest(RestoreRequest + .builder() + .days(expirationDays) + .glacierJobParameters(GlacierJobParameters.builder().tier(tier).build()) + .build())); + } + @Override public void setEncryptionSecrets(final EncryptionSecrets secrets) { encryptionSecrets = secrets; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java index 4b8a28f3e7bb0..936c22781a949 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java @@ -26,7 +26,6 @@ import java.util.concurrent.ExecutorService; import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.ListeningExecutorService; - import org.apache.hadoop.thirdparty.com.google.common.util.concurrent.MoreExecutors; import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; @@ -38,6 +37,7 @@ import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; +import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; @@ -117,6 +117,8 @@ public class StoreContext implements ActiveThreadSpanSource { /** Is client side encryption enabled? */ private final boolean isCSEEnabled; + private final S3ObjectStorageClassFilter s3ObjectStorageClassFilter; + /** * Instantiate. */ @@ -137,7 +139,8 @@ public class StoreContext implements ActiveThreadSpanSource { final boolean useListV1, final ContextAccessors contextAccessors, final AuditSpanSource auditor, - final boolean isCSEEnabled) { + final boolean isCSEEnabled, + final S3ObjectStorageClassFilter s3ObjectStorageClassFilter) { this.fsURI = fsURI; this.bucket = bucket; this.configuration = configuration; @@ -158,6 +161,7 @@ public class StoreContext implements ActiveThreadSpanSource { this.contextAccessors = contextAccessors; this.auditor = auditor; this.isCSEEnabled = isCSEEnabled; + this.s3ObjectStorageClassFilter = s3ObjectStorageClassFilter; } public URI getFsURI() { @@ -411,4 +415,12 @@ public RequestFactory getRequestFactory() { public boolean isCSEEnabled() { return isCSEEnabled; } + + /** + * Return the S3ObjectStorageClassFilter object for S3A, whose value is set according to the config {@code fs.s3a.glacier.read.restored.objects} + * @return {@link S3ObjectStorageClassFilter} object + */ + public S3ObjectStorageClassFilter getS3ObjectsStorageClassFilter() { + return s3ObjectStorageClassFilter; + } } diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java index cff38b9fc4b7d..b182db1bfd337 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; +import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.store.audit.AuditSpanSource; @@ -69,6 +70,8 @@ public class StoreContextBuilder { private boolean isCSEEnabled; + private S3ObjectStorageClassFilter s3ObjectStorageClassFilter; + public StoreContextBuilder setFsURI(final URI fsURI) { this.fsURI = fsURI; return this; @@ -175,6 +178,12 @@ public StoreContextBuilder setEnableCSE( return this; } + public StoreContextBuilder setS3ObjectStorageClassFilter( + S3ObjectStorageClassFilter value) { + s3ObjectStorageClassFilter = value; + return this; + } + public StoreContext build() { return new StoreContext(fsURI, bucket, @@ -192,6 +201,7 @@ public StoreContext build() { useListV1, contextAccessors, auditor, - isCSEEnabled); + isCSEEnabled, + s3ObjectStorageClassFilter); } } diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 7412a4cebcc4f..f42972b6630b1 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -928,6 +928,20 @@ The switch to turn S3A auditing on or off. + + +fs.s3a.glacier.read.restored.objects +READ_ALL + + The config can have 3 values: + + * READ_ALL: Retrieval of Glacier files will fail with InvalidObjectStateException: The operation is not valid for the object's storage class. + * SKIP_ALL_GLACIER: If this value is set then this will ignore any S3 Objects which are tagged with Glacier storage classes and retrieve the others. + * READ_RESTORED_GLACIER_OBJECTS: If this value is set then restored status of the Glacier object will be checked, if restored the objects would be read like normal S3 objects else they will be ignored as the objects would not have been retrieved from the S3 Glacier. + + ``` ## Retry and Recovery diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java new file mode 100644 index 0000000000000..77688c3082d15 --- /dev/null +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java @@ -0,0 +1,179 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.s3a.list; + +import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collection; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.contract.s3a.S3AContract; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; +import org.apache.hadoop.fs.s3a.S3ListRequest; +import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; +import org.apache.hadoop.fs.store.audit.AuditSpan; +import org.apache.hadoop.test.LambdaTestUtils; +import org.assertj.core.api.Assertions; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.RestoreObjectRequest; +import software.amazon.awssdk.services.s3.model.S3Object; +import software.amazon.awssdk.services.s3.model.Tier; + +/** + * Tests of various cases related to Glacier/Deep Archive Storage class. + */ +@RunWith(Parameterized.class) +public class ITestS3AReadRestoredGlacierObjects extends AbstractS3ATestBase { + + enum Type { GLACIER_AND_DEEP_ARCHIVE, GLACIER } + + @Parameterized.Parameters(name = "storage-class-{1}") + public static Collection data(){ + return Arrays.asList(new Object[][] { + {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_GLACIER}, + {Type.GLACIER_AND_DEEP_ARCHIVE, STORAGE_CLASS_DEEP_ARCHIVE}, + {Type.GLACIER, STORAGE_CLASS_GLACIER} + }); + } + + private final int MAX_RETRIES = 100; + private final int RETRY_DELAY_MS = 5000; + + private final Type type; + private final String glacierClass; + + public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) { + this.type = type; + this.glacierClass = glacierClass; + } + + private FileSystem createFiles(String s3ObjectStorageClassFilter) throws Throwable { + Configuration conf = this.createConfiguration(); + conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter); + conf.set(STORAGE_CLASS, glacierClass); // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER + S3AContract contract = (S3AContract) createContract(conf); + contract.init(); + + FileSystem fs = contract.getTestFileSystem(); + Path dir = methodPath(); + fs.mkdirs(dir); + Path path = new Path(dir, "file1"); + ContractTestUtils.touch(fs, path); + return fs; + } + + @Override + protected Configuration createConfiguration() { + Configuration newConf = super.createConfiguration(); + skipIfStorageClassTestsDisabled(newConf); + disableFilesystemCaching(newConf); + removeBaseAndBucketOverrides(newConf, STORAGE_CLASS); + return newConf; + } + + @Test + public void testIgnoreGlacierObject() throws Throwable { + Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE); + try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.SKIP_ALL_GLACIER.name())) { + Assertions.assertThat( + fs.listStatus(methodPath())) + .describedAs("FileStatus List of %s", methodPath()).isEmpty(); + } + } + + @Test + public void testIgnoreRestoringGlacierObject() throws Throwable { + Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE); + try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) { + Assertions.assertThat( + fs.listStatus( + methodPath())) + .describedAs("FileStatus List of %s", methodPath()).isEmpty(); + } + } + + @Test + public void testRestoredGlacierObject() throws Throwable { + Assume.assumeTrue(type == Type.GLACIER); // Skipping this test for Deep Archive as expedited retrieval is not supported + try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) { + restoreGlacierObject(getFilePrefixForListObjects(), 2); + Assertions.assertThat( + fs.listStatus( + methodPath())) + .describedAs("FileStatus List of %s", methodPath()).isNotEmpty(); + } + } + + @Test + public void testDefault() throws Throwable { + Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE); + try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.READ_ALL.name())) { + Assertions.assertThat( + fs.listStatus(methodPath())) + .describedAs("FileStatus List of %s", methodPath()).isNotEmpty(); + } + } + + + private void restoreGlacierObject(String glacierObjectKey, int expirationDays) throws Exception { + try (AuditSpan auditSpan = getSpanSource().createSpan(OBJECT_LIST_REQUEST, "", "").activate()) { + + S3Client s3Client = getFileSystem().getS3AInternals().getAmazonS3Client("test"); + + // Create a restore object request + RestoreObjectRequest requestRestore = getFileSystem().getRequestFactory() + .newRestoreObjectRequestBuilder(glacierObjectKey, Tier.EXPEDITED, expirationDays).build(); + + s3Client.restoreObject(requestRestore); + + // fetch the glacier object + S3ListRequest s3ListRequest = getFileSystem().createListObjectsRequest( + getFilePrefixForListObjects(), "/"); + + LambdaTestUtils.await(MAX_RETRIES * RETRY_DELAY_MS, RETRY_DELAY_MS, + () -> !getS3GlacierObject(s3Client, s3ListRequest).restoreStatus().isRestoreInProgress()); + } + } + + + private String getFilePrefixForListObjects() throws IOException { + return getFileSystem().pathToKey(new Path(methodPath(), "file1")); + } + + private S3Object getS3GlacierObject(S3Client s3Client, S3ListRequest s3ListRequest) { + return s3Client.listObjectsV2(s3ListRequest.getV2()).contents() + .stream() + .findFirst().orElse(null); + } +} From 6110c7ce28de33623c8e193d94e6a218fbd98f5b Mon Sep 17 00:00:00 2001 From: Bhavay Pahuja Date: Thu, 8 Feb 2024 12:14:31 +0530 Subject: [PATCH 2/7] Fixing checkstyle --- .../org/apache/hadoop/fs/s3a/Listing.java | 6 +++-- .../fs/s3a/S3ObjectStorageClassFilter.java | 22 ++++++++++++------- .../hadoop/fs/s3a/api/RequestFactory.java | 2 +- .../fs/s3a/impl/RequestFactoryImpl.java | 3 ++- .../hadoop/fs/s3a/impl/StoreContext.java | 3 ++- .../ITestS3AReadRestoredGlacierObjects.java | 12 +++++----- 6 files changed, 30 insertions(+), 18 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index 92b8ee3c0edcf..f5d2bdecd5b1c 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -464,8 +464,10 @@ private boolean buildNextStatusBatch(S3ListResult objects) { LOG.debug("{}: {}", keyPath, stringify(s3Object)); } // Skip over keys that are ourselves and old S3N _$folder$ files - // Handle Glacier Storage Class objects based on the config fs.s3a.glacier.read.restored.objects value set - if ( s3ObjectStorageClassFilter.getFilter().apply(s3Object) && acceptor.accept(keyPath, s3Object) && filter.accept(keyPath)) { + // Handle Glacier Storage Class based on the config fs.s3a.glacier.read.restored.objects + if (s3ObjectStorageClassFilter.getFilter().apply(s3Object) && + acceptor.accept(keyPath, s3Object) && + filter.accept(keyPath)) { S3AFileStatus status = createFileStatus(keyPath, s3Object, listingOperationCallbacks.getDefaultBlockSize(keyPath), getStoreContext().getUsername(), diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java index dcdae697f61f1..258152a26d3e5 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java @@ -27,11 +27,16 @@ /** * 
- * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link S3AFileSystem}
+ * {@link S3ObjectStorageClassFilter} will filter the S3 files based on the
+ * {@code fs.s3a.glacier.read.restored.objects} configuration set in {@link S3AFileSystem}
  * The config can have 3 values:
- * {@code READ_ALL}: Retrieval of Glacier files will fail with InvalidObjectStateException: The operation is not valid for the object's storage class.
- * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3 Objects which are tagged with Glacier storage classes and retrieve the others.
- * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored status of the Glacier object will be checked, if restored the objects would be read like normal S3 objects else they will be ignored as the objects would not have been retrieved from the S3 Glacier.
+ * {@code READ_ALL}: Retrieval of Glacier files will fail with InvalidObjectStateException:
+ * The operation is not valid for the object's storage class.
+ * {@code SKIP_ALL_GLACIER}: If this value is set then this will ignore any S3 Objects which are
+ * tagged with Glacier storage classes and retrieve the others.
+ * {@code READ_RESTORED_GLACIER_OBJECTS}: If this value is set then restored status of the Glacier
+ * object will be checked, if restored the objects would be read like normal S3 objects
+ * else they will be ignored as the objects would not have been retrieved from the S3 Glacier.
  *
*/ public enum S3ObjectStorageClassFilter { @@ -39,7 +44,8 @@ public enum S3ObjectStorageClassFilter { SKIP_ALL_GLACIER(S3ObjectStorageClassFilter::isNotGlacierObject), READ_RESTORED_GLACIER_OBJECTS(S3ObjectStorageClassFilter::isCompletedRestoredObject); - private static final Set GLACIER_STORAGE_CLASSES = Sets.newHashSet(ObjectStorageClass.GLACIER, ObjectStorageClass.DEEP_ARCHIVE); + private static final Set GLACIER_STORAGE_CLASSES = Sets.newHashSet( + ObjectStorageClass.GLACIER, ObjectStorageClass.DEEP_ARCHIVE); private final Function filter; @@ -48,7 +54,7 @@ public enum S3ObjectStorageClassFilter { } /** - * Checks if the s3 object is not an object with a storage class of glacier/deep_archive + * Checks if the s3 object is not an object with a storage class of glacier/deep_archive. * @param object s3 object * @return if the s3 object is not an object with a storage class of glacier/deep_archive */ @@ -57,7 +63,7 @@ private static boolean isNotGlacierObject(S3Object object) { } /** - * Checks if the s3 object is an object with a storage class of glacier/deep_archive + * Checks if the s3 object is an object with a storage class of glacier/deep_archive. * @param object s3 object * @return if the s3 object is an object with a storage class of glacier/deep_archive */ @@ -66,7 +72,7 @@ private static boolean isGlacierObject(S3Object object) { } /** - * Checks if the s3 object is completely restored + * Checks if the s3 object is completely restored. * @param object s3 object * @return if the s3 object is completely restored */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java index 2e70b0db089dc..d293d880b7d79 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/RequestFactory.java @@ -254,7 +254,7 @@ DeleteObjectsRequest.Builder newBulkDeleteRequestBuilder( List keysToDelete); /** - * Create a request builder to initiate a restore of Glacier object + * Create a request builder to initiate a restore of Glacier object. * @param key object to restore * @param tier glacier retrieval tier at which the restore will be processed. * @param expirationDays lifetime of the active restored copy in days. diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java index 15b2eefd74156..c6df6965c6ea9 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/RequestFactoryImpl.java @@ -614,7 +614,8 @@ public ListObjectsV2Request.Builder newListObjectsV2RequestBuilder( final ListObjectsV2Request.Builder requestBuilder = ListObjectsV2Request.builder() .bucket(bucket) .maxKeys(maxKeys) - .optionalObjectAttributes(OptionalObjectAttributes.RESTORE_STATUS) // Optional Attribute to get the Restored Status of the Glacier Objects + // Optional Attribute to get the Restored Status of the Glacier Objects + .optionalObjectAttributes(OptionalObjectAttributes.RESTORE_STATUS) .prefix(key); if (delimiter != null) { diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java index 936c22781a949..93f30ea577843 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java @@ -417,7 +417,8 @@ public boolean isCSEEnabled() { } /** - * Return the S3ObjectStorageClassFilter object for S3A, whose value is set according to the config {@code fs.s3a.glacier.read.restored.objects} + * Return the S3ObjectStorageClassFilter object for S3A, + * whose value is set according to the config {@code fs.s3a.glacier.read.restored.objects}. * @return {@link S3ObjectStorageClassFilter} object */ public S3ObjectStorageClassFilter getS3ObjectsStorageClassFilter() { diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java index 77688c3082d15..e61d6c91afcc1 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java @@ -67,8 +67,8 @@ public static Collection data(){ }); } - private final int MAX_RETRIES = 100; - private final int RETRY_DELAY_MS = 5000; + private final int maxRetries = 100; + private final int retryDelayMs = 5000; private final Type type; private final String glacierClass; @@ -81,7 +81,8 @@ public ITestS3AReadRestoredGlacierObjects(Type type, String glacierClass) { private FileSystem createFiles(String s3ObjectStorageClassFilter) throws Throwable { Configuration conf = this.createConfiguration(); conf.set(READ_RESTORED_GLACIER_OBJECTS, s3ObjectStorageClassFilter); - conf.set(STORAGE_CLASS, glacierClass); // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER + // Create Glacier objects:Storage Class:DEEP_ARCHIVE/GLACIER + conf.set(STORAGE_CLASS, glacierClass); S3AContract contract = (S3AContract) createContract(conf); contract.init(); @@ -125,7 +126,8 @@ public void testIgnoreRestoringGlacierObject() throws Throwable { @Test public void testRestoredGlacierObject() throws Throwable { - Assume.assumeTrue(type == Type.GLACIER); // Skipping this test for Deep Archive as expedited retrieval is not supported + // Skipping this test for Deep Archive as expedited retrieval is not supported + Assume.assumeTrue(type == Type.GLACIER); try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.READ_RESTORED_GLACIER_OBJECTS.name())) { restoreGlacierObject(getFilePrefixForListObjects(), 2); Assertions.assertThat( @@ -161,7 +163,7 @@ private void restoreGlacierObject(String glacierObjectKey, int expirationDays) t S3ListRequest s3ListRequest = getFileSystem().createListObjectsRequest( getFilePrefixForListObjects(), "/"); - LambdaTestUtils.await(MAX_RETRIES * RETRY_DELAY_MS, RETRY_DELAY_MS, + LambdaTestUtils.await(maxRetries * retryDelayMs, retryDelayMs, () -> !getS3GlacierObject(s3Client, s3ListRequest).restoreStatus().isRestoreInProgress()); } } From de6d1b0e3d0928293926bc8ddf5dfc25c8df2a2d Mon Sep 17 00:00:00 2001 From: Bhavay Pahuja Date: Tue, 13 Feb 2024 14:07:06 +0530 Subject: [PATCH 3/7] updating documentation and moving S3ObjectStorageClassFilter.java to s3-api and some minor changes --- .../org/apache/hadoop/fs/s3a/Constants.java | 7 +++ .../org/apache/hadoop/fs/s3a/Listing.java | 1 + .../apache/hadoop/fs/s3a/S3AFileSystem.java | 3 +- .../{ => api}/S3ObjectStorageClassFilter.java | 3 +- .../hadoop/fs/s3a/impl/StoreContext.java | 2 +- .../fs/s3a/impl/StoreContextBuilder.java | 2 +- .../site/markdown/tools/hadoop-aws/index.md | 14 +++++- .../ITestS3AReadRestoredGlacierObjects.java | 49 ++++++++++--------- 8 files changed, 53 insertions(+), 28 deletions(-) rename hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/{ => api}/S3ObjectStorageClassFilter.java (97%) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index 9c19335bc123e..b763fa285afe4 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -21,6 +21,7 @@ import org.apache.hadoop.classification.InterfaceAudience; import org.apache.hadoop.classification.InterfaceStability; import org.apache.hadoop.fs.Options; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; import org.apache.hadoop.security.ssl.DelegatingSSLSocketFactory; import java.time.Duration; @@ -1525,6 +1526,12 @@ private Constants() { */ public static final String READ_RESTORED_GLACIER_OBJECTS = "fs.s3a.glacier.read.restored.objects"; + /** + * Default value of Read Restored Glacier objects config. + * Value = {@value} + */ + public static final String DEFAULT_READ_RESTORED_GLACIER_OBJECTS = S3ObjectStorageClassFilter.READ_ALL.toString(); + /** * The bucket region header. */ diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java index f5d2bdecd5b1c..8bc3d3c9bb273 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Listing.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.s3a; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; import software.amazon.awssdk.services.s3.model.CommonPrefix; import software.amazon.awssdk.services.s3.model.S3Object; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 291dc316edaa9..3fc572616913b 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@ -52,6 +52,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import javax.annotation.Nullable; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; import software.amazon.awssdk.core.ResponseInputStream; import software.amazon.awssdk.core.exception.SdkException; import software.amazon.awssdk.services.s3.S3AsyncClient; @@ -592,7 +593,7 @@ public void initialize(URI name, Configuration originalConf) s3aInternals = createS3AInternals(); s3ObjectStorageClassFilter = Optional.of(conf.getTrimmed(READ_RESTORED_GLACIER_OBJECTS, - S3ObjectStorageClassFilter.READ_ALL.toString())) + DEFAULT_READ_RESTORED_GLACIER_OBJECTS)) .map(String::toUpperCase) .map(S3ObjectStorageClassFilter::valueOf).get(); diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java similarity index 97% rename from hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java rename to hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java index 258152a26d3e5..1ab874ca77f79 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ObjectStorageClassFilter.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/api/S3ObjectStorageClassFilter.java @@ -16,8 +16,9 @@ * limitations under the License. */ -package org.apache.hadoop.fs.s3a; +package org.apache.hadoop.fs.s3a.api; +import org.apache.hadoop.fs.s3a.S3AFileSystem; import org.apache.hadoop.thirdparty.com.google.common.collect.Sets; import java.util.Set; import java.util.function.Function; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java index 93f30ea577843..8f93cfcef193f 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContext.java @@ -37,7 +37,7 @@ import org.apache.hadoop.fs.s3a.S3AFileStatus; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; -import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; import org.apache.hadoop.fs.s3a.Statistic; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.store.audit.ActiveThreadSpanSource; diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java index b182db1bfd337..b245d582851db 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/impl/StoreContextBuilder.java @@ -25,7 +25,7 @@ import org.apache.hadoop.fs.s3a.Invoker; import org.apache.hadoop.fs.s3a.S3AInputPolicy; import org.apache.hadoop.fs.s3a.S3AStorageStatistics; -import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; import org.apache.hadoop.fs.s3a.audit.AuditSpanS3A; import org.apache.hadoop.fs.s3a.statistics.S3AStatisticsContext; import org.apache.hadoop.fs.store.audit.AuditSpanSource; diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index f42972b6630b1..461b44a8c1903 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -926,8 +926,18 @@ The switch to turn S3A auditing on or off. Should auditing of S3A requests be enabled? - + Glacier Object Support + +[Amazon S3 Glacier (S3 Glacier)](https://docs.aws.amazon.com/amazonglacier/latest/dev/introduction.html) is a secure and durable service for low-cost data archiving and +long-term backup. +With S3 Glacier, you can store your data cost effectively for months, years, or even decades. +This support introduces a new config, which decides the objects returned from listStatus. +Note : This is not available on all AWS S3 store types, or on third party stores. +The config is as follows: +```xml @@ -943,6 +953,8 @@ The switch to control how S3A handles glacier storage classes. ``` + + ## Retry and Recovery The S3A client makes a best-effort attempt at recovering from network failures; diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java index e61d6c91afcc1..d060439d6a778 100644 --- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java +++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/list/ITestS3AReadRestoredGlacierObjects.java @@ -18,28 +18,10 @@ package org.apache.hadoop.fs.s3a.list; -import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS; -import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; -import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE; -import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; -import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled; -import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; - import java.io.IOException; import java.util.Arrays; import java.util.Collection; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.apache.hadoop.fs.contract.s3a.S3AContract; -import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; -import org.apache.hadoop.fs.s3a.S3ListRequest; -import org.apache.hadoop.fs.s3a.S3ObjectStorageClassFilter; -import org.apache.hadoop.fs.store.audit.AuditSpan; -import org.apache.hadoop.test.LambdaTestUtils; + import org.assertj.core.api.Assertions; import org.junit.Assume; import org.junit.Test; @@ -50,6 +32,27 @@ import software.amazon.awssdk.services.s3.model.S3Object; import software.amazon.awssdk.services.s3.model.Tier; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.apache.hadoop.fs.contract.s3a.S3AContract; +import org.apache.hadoop.fs.s3a.AbstractS3ATestBase; +import org.apache.hadoop.fs.s3a.S3ListRequest; +import org.apache.hadoop.fs.s3a.api.S3ObjectStorageClassFilter; +import org.apache.hadoop.fs.store.audit.AuditSpan; +import org.apache.hadoop.test.LambdaTestUtils; + +import static org.apache.hadoop.fs.s3a.Constants.READ_RESTORED_GLACIER_OBJECTS; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_DEEP_ARCHIVE; +import static org.apache.hadoop.fs.s3a.Constants.STORAGE_CLASS_GLACIER; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.disableFilesystemCaching; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.removeBaseAndBucketOverrides; +import static org.apache.hadoop.fs.s3a.S3ATestUtils.skipIfStorageClassTestsDisabled; +import static org.apache.hadoop.fs.statistics.StoreStatisticNames.OBJECT_LIST_REQUEST; + + /** * Tests of various cases related to Glacier/Deep Archive Storage class. */ @@ -89,7 +92,7 @@ private FileSystem createFiles(String s3ObjectStorageClassFilter) throws Throwab FileSystem fs = contract.getTestFileSystem(); Path dir = methodPath(); fs.mkdirs(dir); - Path path = new Path(dir, "file1"); + Path path = new Path(dir, "glaciated"); ContractTestUtils.touch(fs, path); return fs; } @@ -99,7 +102,7 @@ protected Configuration createConfiguration() { Configuration newConf = super.createConfiguration(); skipIfStorageClassTestsDisabled(newConf); disableFilesystemCaching(newConf); - removeBaseAndBucketOverrides(newConf, STORAGE_CLASS); + removeBaseAndBucketOverrides(newConf, STORAGE_CLASS, READ_RESTORED_GLACIER_OBJECTS); return newConf; } @@ -138,7 +141,7 @@ public void testRestoredGlacierObject() throws Throwable { } @Test - public void testDefault() throws Throwable { + public void testReadAllObjects() throws Throwable { Assume.assumeTrue(type == Type.GLACIER_AND_DEEP_ARCHIVE); try (FileSystem fs = createFiles(S3ObjectStorageClassFilter.READ_ALL.name())) { Assertions.assertThat( @@ -170,7 +173,7 @@ private void restoreGlacierObject(String glacierObjectKey, int expirationDays) t private String getFilePrefixForListObjects() throws IOException { - return getFileSystem().pathToKey(new Path(methodPath(), "file1")); + return getFileSystem().pathToKey(new Path(methodPath(), "glaciated")); } private S3Object getS3GlacierObject(S3Client s3Client, S3ListRequest s3ListRequest) { From 04548bb0dfb6f12aa5c0a26d738cdcade24b98db Mon Sep 17 00:00:00 2001 From: Bhavay Pahuja Date: Wed, 14 Feb 2024 09:15:34 +0530 Subject: [PATCH 4/7] minor updates --- .../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 1 - .../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 +++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java index b763fa285afe4..10a088d87e1d1 100644 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java @@ -1528,7 +1528,6 @@ private Constants() { /** * Default value of Read Restored Glacier objects config. - * Value = {@value} */ public static final String DEFAULT_READ_RESTORED_GLACIER_OBJECTS = S3ObjectStorageClassFilter.READ_ALL.toString(); diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md index 461b44a8c1903..8f3ae0f0c2d6b 100644 --- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md +++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md @@ -926,17 +926,17 @@ The switch to turn S3A auditing on or off. Should auditing of S3A requests be enabled? - ``` ## Glacier Object Support [Amazon S3 Glacier (S3 Glacier)](https://docs.aws.amazon.com/amazonglacier/latest/dev/introduction.html) is a secure and durable service for low-cost data archiving and long-term backup. With S3 Glacier, you can store your data cost effectively for months, years, or even decades. -This support introduces a new config, which decides the objects returned from listStatus. +This support introduces a new config, which decides the objects returned from listStatus. Note : This is not available on all AWS S3 store types, or on third party stores. -The config is as follows: +The config is as follows: ```xml