Merge branch 'improvement/BB-550' into q/8.6

scality · Oct 2, 2024 · 0579c54 · 0579c54
2 parents d20c311 + d8c9131
commit 0579c54
Show file tree

Hide file tree

Showing 4 changed files with 100 additions and 4 deletions.
diff --git a/extensions/lifecycle/LifecycleConfigValidator.js b/extensions/lifecycle/LifecycleConfigValidator.js
@@ -15,7 +15,7 @@ const joiSchema = joi.object({
     transitionTasksTopic: joi.string().default(parent => parent.objectTasksTopic),
     coldStorageTopics: joi.array().items(joi.string()).unique().default([]),
     auth: authJoi.optional(),
-    forceLegacyListing: joi.boolean().default(true),
+    forceLegacyListing: joi.boolean().default(false),
     autoCreateIndexes: joi.boolean().default(false),
     conductor: {
         auth: inheritedAuthJoi,

diff --git a/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js b/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js
@@ -330,7 +330,7 @@ class LifecycleBucketProcessor {
 
             let task;
 
-            if (taskVersion === lifecycleTaskVersions.v1 || !taskVersion) {
+            if (taskVersion === lifecycleTaskVersions.v1 || (!taskVersion && this._lcConfig.forceLegacyListing)) {
                 task = new LifecycleTask(this);
             } else {
                 task = new LifecycleTaskV2(this);

diff --git a/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js b/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js
@@ -2,6 +2,7 @@
 
 const assert = require('assert');
 const sinon = require('sinon');
+const async = require('async');
 
 const LifecycleBucketProcessor = require(
     '../../../extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor');
@@ -16,12 +17,20 @@ const {
     timeOptions,
 } = require('../../functional/lifecycle/configObjects');
 
+const {
+    bucketProcessorEntry,
+    bucketProcessorV1Entry,
+    bucketProcessorV2Entry
+} = require('../../utils/kafkaEntries');
+const LifecycleTask = require('../../../extensions/lifecycle/tasks/LifecycleTask');
+const LifecycleTaskV2 = require('../../../extensions/lifecycle/tasks/LifecycleTaskV2');
+
 
 describe('Lifecycle Bucket Processor', () => {
     let lbp;
     beforeEach(() => {
         lbp = new LifecycleBucketProcessor(
-            zkConfig, kafkaConfig, lcConfig, repConfig, s3Config, mongoConfig, timeOptions);
+            zkConfig, kafkaConfig, lcConfig, repConfig, s3Config, mongoConfig, 'http', timeOptions);
     });
 
     afterEach(() => {
@@ -197,4 +206,71 @@ describe('Lifecycle Bucket Processor', () => {
             });
         });
     });
+
+    describe('_processBucketEntry', () => {
+
+        [
+            {
+                it: 'should use the forceLegacyListing when listing version is not ' +
+                    'specified in the Kafka messages (v1 case)',
+                forceLegacyListing: true,
+                kafkaMessage: bucketProcessorEntry,
+                validateTask: task => task instanceof LifecycleTask && !(task instanceof LifecycleTaskV2),
+            },
+            {
+                it: 'should use the forceLegacyListing when listing version is not ' +
+                    'specified in the Kafka messages (v2 case)',
+                forceLegacyListing: false,
+                kafkaMessage: bucketProcessorEntry,
+                validateTask: task => task instanceof LifecycleTaskV2,
+            },
+            {
+                it: 'should use the listing type specified in the kafka message (v1 case)',
+                forceLegacyListing: false,
+                kafkaMessage: bucketProcessorV1Entry,
+                validateTask: task => task instanceof LifecycleTask && !(task instanceof LifecycleTaskV2),
+            },
+            {
+                it: 'should use the listing type specified in the kafka message (v2 case)',
+                forceLegacyListing: true,
+                kafkaMessage: bucketProcessorV2Entry,
+                validateTask: task => task instanceof LifecycleTaskV2,
+            },
+        ].forEach(opts => {
+            it(opts.it, done => {
+                if (opts.forceLegacyListing !== null) {
+                    lbp._lcConfig.forceLegacyListing = opts.forceLegacyListing;
+                }
+
+                lbp.clientManager = {
+                    getS3Client: () => ({}),
+                    getBackbeatMetadataProxy: () => ({}),
+                };
+
+                sinon.stub(lbp, '_getBucketLifecycleConfiguration').yields(null, {
+                    Rules: [{
+                        Status: 'Enabled',
+                        Transitions: [{
+                            Days: 10,
+                            StorageClass: 'azure'
+                        }],
+                        ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4'
+                    }]
+                });
+
+                const tasks = [];
+                lbp._internalTaskScheduler = async.queue((ctx, cb) => {
+                    tasks.push(ctx.task);
+                    cb();
+                }, 1);
+
+                lbp._processBucketEntry(opts.kafkaMessage, err => {
+                    assert.ifError(err);
+                    assert.strictEqual(tasks.length, 1);
+                    assert(opts.validateTask(tasks[0]));
+                    done();
+                });
+            });
+        });
+    });
 });
diff --git a/tests/utils/kafkaEntries.js b/tests/utils/kafkaEntries.js
@@ -6,4 +6,24 @@ const replicationEntry = {
     value: '{"type":"put","bucket":"queue-populator-test-bucket","key":"hosts\\u000098500086134471999999RG001  0","value":"{\\"md-model-version\\":2,\\"owner-display-name\\":\\"Bart\\",\\"owner-id\\":\\"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be\\",\\"content-length\\":542,\\"content-type\\":\\"text/plain\\",\\"last-modified\\":\\"2017-07-13T02:44:25.519Z\\",\\"content-md5\\":\\"01064f35c238bd2b785e34508c3d27f4\\",\\"x-amz-version-id\\":\\"null\\",\\"x-amz-server-version-id\\":\\"\\",\\"x-amz-storage-class\\":\\"sf\\",\\"x-amz-server-side-encryption\\":\\"\\",\\"x-amz-server-side-encryption-aws-kms-key-id\\":\\"\\",\\"x-amz-server-side-encryption-customer-algorithm\\":\\"\\",\\"x-amz-website-redirect-location\\":\\"\\",\\"acl\\":{\\"Canned\\":\\"private\\",\\"FULL_CONTROL\\":[],\\"WRITE_ACP\\":[],\\"READ\\":[],\\"READ_ACP\\":[]},\\"key\\":\\"\\",\\"location\\":[{\\"key\\":\\"29258f299ddfd65f6108e6cd7bd2aea9fbe7e9e0\\",\\"size\\":542,\\"start\\":0,\\"dataStoreName\\":\\"file\\",\\"dataStoreETag\\":\\"1:01064f35c238bd2b785e34508c3d27f4\\"}],\\"isDeleteMarker\\":false,\\"tags\\":{},\\"replicationInfo\\":{\\"status\\":\\"PENDING\\",\\"backends\\":[{\\"site\\":\\"sf\\",\\"status\\":\\"PENDING\\",\\"dataStoreVersionId\\":\\"B2AqTml1DtEKWJwRiOTh0tgkm8AlyH7W\\"},{\\"site\\":\\"replicationaws\\",\\"status\\":\\"PENDING\\",\\"dataStoreVersionId\\":\\"ob.rop0jdndzwVioi7v.6Q9.v9.6QOGv\\"}],\\"content\\":[\\"DATA\\",\\"METADATA\\"],\\"destination\\":\\"arn:aws:s3:::dummy-dest-bucket\\",\\"storageClass\\":\\"sf\\",\\"role\\":\\"arn:aws:iam::123456789012:role/backbeat\\"},\\"x-amz-meta-s3cmd-attrs\\":\\"uid:0/gname:root/uname:root/gid:0/mode:33188/mtime:1490807629/atime:1499845478/md5:01064f35c238bd2b785e34508c3d27f4/ctime:1490807629\\",\\"versionId\\":\\"98500086134471999999RG001  0\\",\\"isNFS\\":true}"}',
 };
 
-module.exports = { replicationEntry };
+const bucketProcessorEntry = {
+    key: null,
+    value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b306"},"target":{"bucket":"bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795"},"details":{}}'
+};
+
+const bucketProcessorV1Entry = {
+    key: null,
+    value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b307"},"target":{"bucket":"v1-bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795","taskVersion":"v1"},"details":{}}'
+};
+
+const bucketProcessorV2Entry = {
+    key: null,
+    value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b308"},"target":{"bucket":"v2-bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795","taskVersion":"v2"},"details":{}}'
+};
+
+module.exports = {
+    replicationEntry,
+    bucketProcessorEntry,
+    bucketProcessorV1Entry,
+    bucketProcessorV2Entry
+};