Added support for S3 legal hold operations and fixed attempted deletions of ObjectLock configurations

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog of elbencho
 
+## WIP
+
+### New Features & Enhancements
+* Added support for S3 object legal holds (See "--s3olegal", "--s3olegalverify")
+
+### Fixes
+* Fixed a bug where elbencho tried to disable object lock configurations during the delete phase (this operation is not possible)
+
 ## v3.0.11 (June 11, 2024)
 
 ### New Features & Enhancements

dist/etc/bash_completion.d/elbencho

@@ -134,6 +134,8 @@ _elbencho_opts()
         --s3nocompress
         --s3nomd5
         --s3objprefix
+        --s3olegal
+        --s3olegalverify
         --s3olockcfg
         --s3olockcfgverify
         --s3otag

source/ProgArgs.cpp

@@ -550,6 +550,10 @@ void ProgArgs::defineAllowedArgs()
 			"S3 object prefix. This will be prepended to all object names when the benchmark path "
 			"is a bucket. (A sequence of 3 to 16 \"" RAND_PREFIX_MARKS_SUBSTR "\" chars will be "
 			"replaced by a random hex string of the same length.)")
+/*s3o*/	(ARG_S3OBJLEGAL_LONG, bpo::bool_switch(&this->doS3ObjectLegalHold),
+            "Activate object legal hold configurations. Must be passed with \"--" ARG_S3OBJLOCKCFG_LONG "\"")
+/*s3o*/	(ARG_S3OBJLEGALVERIFY_LONG, bpo::bool_switch(&this->doS3ObjectLegalHoldVerify),
+            "Verify the correctness of object legal hold configurations")
 /*s3o*/	(ARG_S3OBJLOCKCFG_LONG, bpo::bool_switch(&this->doS3ObjectLockCfg),
             "Activate object lock configuration creation")
 /*s3o*/	(ARG_S3OBJLOCKCFGVERIFY_LONG, bpo::bool_switch(&this->doS3ObjectLockCfgVerify),
@@ -795,6 +799,8 @@ void ProgArgs::defineDefaults()
     this->doS3BucketVersioningVerify = false;
     this->doS3ObjectTag = false;
     this->doS3ObjectTagVerify = false;
+    this->doS3ObjectLegalHold = false;
+    this->doS3ObjectLegalHoldVerify = false;
     this->doS3ObjectLockCfg = false;
     this->doS3ObjectLockCfgVerify = false;
 	this->useOpsLogLocking = false;
@@ -3048,6 +3054,8 @@ void ProgArgs::setFromPropertyTreeForService(bpt::ptree& tree)
     doS3BucketVersioningVerify = tree.get<bool>(ARG_S3BUCKETVER_VERIFY_LONG);
     doS3ObjectTag = tree.get<bool>(ARG_S3OBJTAG_LONG);
     doS3ObjectTagVerify = tree.get<bool>(ARG_S3OBJTAGVERIFY_LONG);
+    doS3ObjectLegalHold = tree.get<bool>(ARG_S3OBJLEGAL_LONG);
+    doS3ObjectLegalHoldVerify = tree.get<bool>(ARG_S3OBJLEGALVERIFY_LONG);
     doS3ObjectLockCfg = tree.get<bool>(ARG_S3OBJLOCKCFG_LONG);
     doS3ObjectLockCfgVerify = tree.get<bool>(ARG_S3OBJLOCKCFGVERIFY_LONG);
 	doTruncate = tree.get<bool>(ARG_TRUNCATE_LONG);
@@ -3261,6 +3269,8 @@ void ProgArgs::getAsPropertyTreeForService(bpt::ptree& outTree, size_t serviceRa
     outTree.put(ARG_S3BUCKETVER_VERIFY_LONG, doS3BucketVersioningVerify);
     outTree.put(ARG_S3OBJTAG_LONG, doS3ObjectTag);
     outTree.put(ARG_S3OBJTAGVERIFY_LONG, doS3ObjectTagVerify);
+    outTree.put(ARG_S3OBJLOCKCFG_LONG, doS3ObjectLegalHold);
+    outTree.put(ARG_S3OBJLOCKCFGVERIFY_LONG, doS3ObjectLegalHoldVerify);
     outTree.put(ARG_S3OBJLOCKCFG_LONG, doS3ObjectLockCfg);
     outTree.put(ARG_S3OBJLOCKCFGVERIFY_LONG, doS3ObjectLockCfgVerify);
     outTree.put(ARG_SYNCPHASE_LONG, runSyncPhase);

source/ProgArgs.h

@@ -162,6 +162,8 @@ namespace bpt = boost::property_tree;
 #define ARG_S3NOMD5_LONG            "s3nomd5"
 #define ARG_S3NOMPCHECK_LONG		"s3nompcheck"
 #define ARG_S3OBJECTPREFIX_LONG		"s3objprefix"
+#define ARG_S3OBJLEGAL_LONG         "s3olegal"
+#define ARG_S3OBJLEGALVERIFY_LONG   "s3olegalverify"
 #define ARG_S3OBJLOCKCFG_LONG       "s3olockcfg"
 #define ARG_S3OBJLOCKCFGVERIFY_LONG "s3olockcfgverify"
 #define ARG_S3OBJTAG_LONG           "s3otag"
@@ -354,6 +356,8 @@ class ProgArgs
 		bool doS3BucketTagVerify; // do bucket tagging verification.
         bool doS3ObjectTag; // add object tagging ops during different object operations
         bool doS3ObjectTagVerify; // do bucket tagging verification.
+        bool doS3ObjectLegalHold; // do object legal hold configuration
+        bool doS3ObjectLegalHoldVerify; // do object legal hold configuration
         bool doS3ObjectLockCfg; // do S3 object lock configuration
         bool doS3ObjectLockCfgVerify; // do S3 object lock configuration verification
 		bool doTruncate; // truncate files to 0 size on open for writing
@@ -554,10 +558,14 @@ class ProgArgs
 		const IntVec& getBenchPathFDs() const { return benchPathFDsVec; }
 		BenchPathType getBenchPathType() const { return benchPathType; }
 
-        bool getS3BucketMetadataRequested() const {
+        bool getS3BucketMetadataRequested() const
+        {
             return doS3BucketTag || doS3ObjectLockCfg || doS3BucketVersioning;
         }
-        bool getS3ObjectMetadataRequested() const { return doS3ObjectTag; }
+        bool getS3ObjectMetadataRequested() const
+        {
+            return doS3ObjectTag || doS3ObjectLegalHold;
+        }
         bool getRunS3GetObjectMetadata() const { return getS3ObjectMetadataRequested(); }
         bool getRunS3PutObjectMetadata() const
             { return getS3ObjectMetadataRequested() && runCreateFilesPhase; }
@@ -597,6 +605,8 @@ class ProgArgs
         bool getDoS3BucketVersioningVerify() const { return doS3BucketVersioningVerify; }
         bool getDoS3BucketTagging() const { return doS3BucketTag; }
         bool getDoS3BucketTaggingVerify() const { return doS3BucketTagVerify; }
+        bool getDoS3ObjectLegalHold() const { return doS3ObjectLegalHold; }
+        bool getDoS3ObjectLegalHoldVerify() const { return doS3ObjectLegalHoldVerify; }
         bool getDoS3ObjectTagging() const { return doS3ObjectTag; }
         bool getDoS3ObjectTaggingVerify() const { return doS3ObjectTagVerify; }
         bool getDoS3ObjectLockConfiguration() const { return doS3ObjectLockCfg; }

source/workers/LocalWorker.cpp

@@ -41,6 +41,7 @@
 	#include <aws/s3/model/GetBucketTaggingRequest.h>
 	#include <aws/s3/model/GetObjectAclRequest.h>
 	#include <aws/s3/model/GetObjectRequest.h>
+    #include <aws/s3/model/GetObjectLegalHoldRequest.h>
     #include <aws/s3/model/GetObjectTaggingRequest.h>
     #include <aws/s3/model/GetObjectLockConfigurationRequest.h>
     #include <aws/s3/model/GetBucketVersioningRequest.h>
@@ -53,6 +54,7 @@
 	#include <aws/s3/model/PutBucketTaggingRequest.h>
 	#include <aws/s3/model/PutObjectAclRequest.h>
 	#include <aws/s3/model/PutObjectRequest.h>
+    #include <aws/s3/model/PutObjectLegalHoldRequest.h>
     #include <aws/s3/model/PutObjectTaggingRequest.h>
     #include <aws/s3/model/PutObjectLockConfigurationRequest.h>
     #include <aws/s3/model/PutBucketVersioningRequest.h>
@@ -3584,10 +3586,6 @@ void LocalWorker::s3ModeIterateBuckets()
             if (progArgs->getDoS3BucketVersioning())
                 s3ModePutBucketVersioning(bucketName, true);
 
-            // Disable lock configuration
-            if (progArgs->getDoS3ObjectLockConfiguration())
-                s3ModePutObjectLockConfiguration(bucketName, true);
-
             if (progArgs->getDoS3BucketTagging())
                 s3ModeDeleteBucketTagging(bucketName);
         }
@@ -3704,6 +3702,9 @@ void LocalWorker::s3ModeIterateObjects()
             {
                 if (progArgs->getDoS3ObjectTagging())
                     s3ModePutObjectTags(bucketVec[bucketIndex], currentObjectPath);
+
+                if (progArgs->getDoS3ObjectLegalHold())
+                    s3ModePutObjectLegalHold(bucketVec[bucketIndex], currentObjectPath);
             }
 
 			if( (benchPhase == BenchPhase_READFILES) || isRWMixedReader)
@@ -3723,6 +3724,9 @@ void LocalWorker::s3ModeIterateObjects()
             {
                 if (progArgs->getDoS3ObjectTagging())
                     s3ModeGetObjectTags(bucketVec[bucketIndex], currentObjectPath);
+
+                if (progArgs->getDoS3ObjectLegalHold())
+                    s3ModeGetObjectLegalHold(bucketVec[bucketIndex], currentObjectPath);
             }
 
 			if(benchPhase == BenchPhase_PUTOBJACL)
@@ -3735,6 +3739,9 @@ void LocalWorker::s3ModeIterateObjects()
             {
                 if (progArgs->getDoS3ObjectTagging())
                     s3ModeDeleteObjectTags(bucketVec[bucketIndex], currentObjectPath);
+
+                if (progArgs->getDoS3ObjectLegalHold())
+                    s3ModePutObjectLegalHold(bucketVec[bucketIndex], currentObjectPath, false);
             }
 
 			if(benchPhase == BenchPhase_DELETEFILES)
@@ -4334,15 +4341,15 @@ void LocalWorker::s3ModeGetBucketVersioning(const std::string& bucketName)
     if (!progArgs->getDoS3BucketVersioningVerify())
         return;
 
-    const auto statusNameMapper = Aws::S3::Model::BucketVersioningStatusMapper::GetNameForBucketVersioningStatus;
+    const auto statusNameMapper = S3::BucketVersioningStatusMapper::GetNameForBucketVersioningStatus;
     const auto& expectedStatus = S3::BucketVersioningStatus::Suspended;
 
     IF_UNLIKELY(bucketVersioningStatus != expectedStatus)
     {
         std::stringstream errStr;
         errStr << "Bucket versioning is set to '" << statusNameMapper(bucketVersioningStatus)
                << "' but the expected value was '" << statusNameMapper(expectedStatus) << "'." << std::endl
-               << "Bucket: " << bucketName << ';';
+               << "Bucket: " << bucketName << ';' << std::endl;
         throw WorkerException(errStr.str());
     }
 
@@ -5480,7 +5487,7 @@ void LocalWorker::s3ModeListObjParallel()
 
 			// build list of received keys. (std::list to preserve order; must be alphabetic)
 			IF_UNLIKELY(doListObjVerify)
-				for(const Aws::S3::Model::Object& obj : outcome.GetResult().GetContents() )
+				for(const S3::Object& obj : outcome.GetResult().GetContents() )
 					receivedObjs.push_back(obj.GetKey() );
 
 		} while(isTruncated && numObjectsLeft); // end of while numObjectsLeft in dir loop
@@ -5620,11 +5627,11 @@ void LocalWorker::s3ModeListAndMultiDeleteObjects()
 
 			// send multi-delete request for received batch of objects...
 
-			Aws::S3::Model::Delete deleteObjectList;
+			S3::Delete deleteObjectList;
 
-			for(const Aws::S3::Model::Object& obj : listOutcome.GetResult().GetContents() )
+			for(const S3::Object& obj : listOutcome.GetResult().GetContents() )
 				deleteObjectList.AddObjects(
-					Aws::S3::Model::ObjectIdentifier().WithKey(obj.GetKey() ) );
+					S3::ObjectIdentifier().WithKey(obj.GetKey() ) );
 
 			S3::DeleteObjectsRequest delRequest;
 			delRequest.SetBucket(bucketVec[bucketIndex] );
@@ -5800,6 +5807,65 @@ void LocalWorker::s3ModeGetObjectAcl(std::string bucketName, std::string objectN
 #endif // S3_SUPPORT
 }
 
+void LocalWorker::s3ModeGetObjectLegalHold(const std::string& bucketName, const std::string& objectName)
+{
+#ifndef S3_SUPPORT
+    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
+#else
+    OPLOG_PRE_OP("GetObjectLegalHold", bucketName + "/" + objectName, 0, 0);
+
+    const auto& getLegalHoldOutcome = s3Client->GetObjectLegalHold(
+        S3::GetObjectLegalHoldRequest()
+            .WithBucket(bucketName)
+            .WithKey(objectName)
+    );
+
+    OPLOG_POST_OP("GetObjectLegalHold", bucketName + "/" + objectName, 0, 0, !getLegalHoldOutcome.IsSuccess());
+
+    s3ModeThrowOnError(getLegalHoldOutcome, "Get object legal hold failed.", bucketName, objectName);
+
+    if (!progArgs->getDoS3ObjectLegalHoldVerify())
+        return;
+
+    const auto& legalHoldStatus = getLegalHoldOutcome.GetResult().GetLegalHold().GetStatus();
+    const auto& expectedStatus = S3::ObjectLockLegalHoldStatus::ON;
+    const auto& mapper = S3::ObjectLockLegalHoldStatusMapper::GetNameForObjectLockLegalHoldStatus;
+
+    IF_UNLIKELY(legalHoldStatus != expectedStatus)
+    {
+        std::stringstream errStr;
+        errStr << "Legal hold is set to '" << mapper(legalHoldStatus)
+               << "' but the expected value was '" << mapper(expectedStatus) << "'." << std::endl
+               << "Bucket: " << bucketName << "; Object: " << objectName << ";" << std::endl;
+        throw WorkerException(errStr.str());
+    }
+
+#endif // S3_SUPPORT
+}
+
+void LocalWorker::s3ModePutObjectLegalHold(const std::string& bucketName, const std::string& objectName, bool state)
+{
+#ifndef S3_SUPPORT
+    throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
+#else
+    const auto legalHold = S3::ObjectLockLegalHold().WithStatus(state ? S3::ObjectLockLegalHoldStatus::ON
+                                                                      : S3::ObjectLockLegalHoldStatus::OFF);
+
+    OPLOG_PRE_OP("PutObjectLegalHold", bucketName + "/" + objectName, state, 0);
+
+    const auto& getLegalHoldOutcome = s3Client->PutObjectLegalHold(
+        S3::PutObjectLegalHoldRequest()
+            .WithBucket(bucketName)
+            .WithKey(objectName)
+            .WithLegalHold(legalHold)
+    );
+
+    OPLOG_POST_OP("PutObjectLegalHold", bucketName + "/" + objectName, state, 0, !getLegalHoldOutcome.IsSuccess());
+
+    s3ModeThrowOnError(getLegalHoldOutcome, "Get object legal hold failed.", bucketName, objectName);
+
+#endif // S3_SUPPORT
+}
 
 void LocalWorker::s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName)
 {
@@ -5932,24 +5998,19 @@ void LocalWorker::s3ModeGetObjectLockConfiguration(const std::string &bucketName
 #endif // S3_SUPPORT
 }
 
-void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName, bool unset)
+void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName)
 {
 #ifndef S3_SUPPORT
     throw WorkerException(std::string(__func__) + "called, but this was built without S3 support");
 #else
     S3::ObjectLockConfiguration objectLockCfg;
 
-    if (unset)
-        objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::NOT_SET);
-    else
-    {
-        objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::Enabled);
-        objectLockCfg.SetRule(
-            S3::ObjectLockRule().WithDefaultRetention(
-                S3::DefaultRetention().WithMode(S3::ObjectLockRetentionMode::COMPLIANCE).WithDays(1)
-            )
-        );
-    }
+    objectLockCfg.SetObjectLockEnabled(S3::ObjectLockEnabled::Enabled);
+    objectLockCfg.SetRule(
+        S3::ObjectLockRule().WithDefaultRetention(
+            S3::DefaultRetention().WithMode(S3::ObjectLockRetentionMode::COMPLIANCE).WithDays(1)
+        )
+    );
 
     OPLOG_PRE_OP("PutObjectLockConfiguration", bucketName, 0, 0);
 
@@ -5958,6 +6019,7 @@ void LocalWorker::s3ModePutObjectLockConfiguration(const std::string &bucketName
             .WithBucket(bucketName)
             .WithObjectLockConfiguration(objectLockCfg));
 
+
     OPLOG_POST_OP("PutObjectLockConfiguration", bucketName, 0, 0, !putObjLockOutcome.IsSuccess());
 
     s3ModeThrowOnError(putObjLockOutcome, "Put object lock configuration failed.", bucketName);

source/workers/LocalWorker.h

@@ -238,11 +238,13 @@ class LocalWorker : public Worker
 		void s3ModeListAndMultiDeleteObjects();
 		void s3ModePutObjectAcl(std::string bucketName, std::string objectName);
 		void s3ModeGetObjectAcl(std::string bucketName, std::string objectName);
+        void s3ModeGetObjectLegalHold(const std::string& bucketName, const std::string& objectName);
+        void s3ModePutObjectLegalHold(const std::string& bucketName, const std::string& objectName, bool state = true);
         void s3ModeGetObjectTags(const std::string& bucketName, const std::string& objectName);
         void s3ModePutObjectTags(const std::string& bucketName, const std::string& objectName);
         void s3ModeDeleteObjectTags(const std::string& bucketName, const std::string& objectName);
         void s3ModeGetObjectLockConfiguration(const std::string& bucketName);
-        void s3ModePutObjectLockConfiguration(const std::string& bucketName, bool unset = false);
+        void s3ModePutObjectLockConfiguration(const std::string& bucketName);
 		bool getS3ModeDoReverseSeqFallback();
 		std::string getS3RandObjectPrefix(size_t workerRank, size_t dirIdx, size_t fileIdx,
 			const std::string& objectPrefix);