From 2a07f08bd3d6f4a368a3dbd4d5336bda4984933e Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Thu, 19 Sep 2024 17:34:00 +0200 Subject: [PATCH 1/3] bucketProcessor: fallback to the forceLegacyListing param when type not in the kafka message Lifecycle conductor in 7.x doesn't set the listing type in the bucketProcessor messages. Falling back to the forceLegacyListing param inside the config when encountering this case during the transition phase from backbeat 7.x to 8.x/9.x The check of forceLegacyListing in the bucket processor was moved to the conductor as the listing type also depends on the MongoDB indexes, which is not something used in S3C. Issue: BB-550 --- .../lifecycle/bucketProcessor/LifecycleBucketProcessor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js b/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js index c587fd6ca..5aa7e13b1 100644 --- a/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js +++ b/extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor.js @@ -330,7 +330,7 @@ class LifecycleBucketProcessor { let task; - if (taskVersion === lifecycleTaskVersions.v1 || !taskVersion) { + if (taskVersion === lifecycleTaskVersions.v1 || (!taskVersion && this._lcConfig.forceLegacyListing)) { task = new LifecycleTask(this); } else { task = new LifecycleTaskV2(this); From 3bc7a49b6e7143ac429ccd24c590560c248d475b Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Thu, 19 Sep 2024 17:38:09 +0200 Subject: [PATCH 2/3] Use the same default value of forceLegacyListing as in S3C This was done to avoid any unwanted behaviour change in S3C. Backbeat 8.x always passes the listing type in the Kafka messages. Issue: BB-550 --- extensions/lifecycle/LifecycleConfigValidator.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/extensions/lifecycle/LifecycleConfigValidator.js b/extensions/lifecycle/LifecycleConfigValidator.js index 152e7da46..e070bde57 100644 --- a/extensions/lifecycle/LifecycleConfigValidator.js +++ b/extensions/lifecycle/LifecycleConfigValidator.js @@ -15,7 +15,7 @@ const joiSchema = joi.object({ transitionTasksTopic: joi.string().default(parent => parent.objectTasksTopic), coldStorageTopics: joi.array().items(joi.string()).unique().default([]), auth: authJoi.optional(), - forceLegacyListing: joi.boolean().default(true), + forceLegacyListing: joi.boolean().default(false), autoCreateIndexes: joi.boolean().default(false), conductor: { auth: inheritedAuthJoi, From d8c91317a5372a2d5756ddf609152f6c63c54fca Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Thu, 19 Sep 2024 17:41:09 +0200 Subject: [PATCH 3/3] add tests Issue: BB-550 --- .../LifecycleBucketProcessor.spec.js | 78 ++++++++++++++++++- tests/utils/kafkaEntries.js | 22 +++++- 2 files changed, 98 insertions(+), 2 deletions(-) diff --git a/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js b/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js index b668faed5..f78bf8984 100644 --- a/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js +++ b/tests/unit/lifecycle/LifecycleBucketProcessor.spec.js @@ -2,6 +2,7 @@ const assert = require('assert'); const sinon = require('sinon'); +const async = require('async'); const LifecycleBucketProcessor = require( '../../../extensions/lifecycle/bucketProcessor/LifecycleBucketProcessor'); @@ -16,12 +17,20 @@ const { timeOptions, } = require('../../functional/lifecycle/configObjects'); +const { + bucketProcessorEntry, + bucketProcessorV1Entry, + bucketProcessorV2Entry +} = require('../../utils/kafkaEntries'); +const LifecycleTask = require('../../../extensions/lifecycle/tasks/LifecycleTask'); +const LifecycleTaskV2 = require('../../../extensions/lifecycle/tasks/LifecycleTaskV2'); + describe('Lifecycle Bucket Processor', () => { let lbp; beforeEach(() => { lbp = new LifecycleBucketProcessor( - zkConfig, kafkaConfig, lcConfig, repConfig, s3Config, mongoConfig, timeOptions); + zkConfig, kafkaConfig, lcConfig, repConfig, s3Config, mongoConfig, 'http', timeOptions); }); afterEach(() => { @@ -197,4 +206,71 @@ describe('Lifecycle Bucket Processor', () => { }); }); }); + + describe('_processBucketEntry', () => { + + [ + { + it: 'should use the forceLegacyListing when listing version is not ' + + 'specified in the Kafka messages (v1 case)', + forceLegacyListing: true, + kafkaMessage: bucketProcessorEntry, + validateTask: task => task instanceof LifecycleTask && !(task instanceof LifecycleTaskV2), + }, + { + it: 'should use the forceLegacyListing when listing version is not ' + + 'specified in the Kafka messages (v2 case)', + forceLegacyListing: false, + kafkaMessage: bucketProcessorEntry, + validateTask: task => task instanceof LifecycleTaskV2, + }, + { + it: 'should use the listing type specified in the kafka message (v1 case)', + forceLegacyListing: false, + kafkaMessage: bucketProcessorV1Entry, + validateTask: task => task instanceof LifecycleTask && !(task instanceof LifecycleTaskV2), + }, + { + it: 'should use the listing type specified in the kafka message (v2 case)', + forceLegacyListing: true, + kafkaMessage: bucketProcessorV2Entry, + validateTask: task => task instanceof LifecycleTaskV2, + }, + ].forEach(opts => { + it(opts.it, done => { + if (opts.forceLegacyListing !== null) { + lbp._lcConfig.forceLegacyListing = opts.forceLegacyListing; + } + + lbp.clientManager = { + getS3Client: () => ({}), + getBackbeatMetadataProxy: () => ({}), + }; + + sinon.stub(lbp, '_getBucketLifecycleConfiguration').yields(null, { + Rules: [{ + Status: 'Enabled', + Transitions: [{ + Days: 10, + StorageClass: 'azure' + }], + ID: 'dac36d89-0005-4c78-8e00-7e9ace06a9c4' + }] + }); + + const tasks = []; + lbp._internalTaskScheduler = async.queue((ctx, cb) => { + tasks.push(ctx.task); + cb(); + }, 1); + + lbp._processBucketEntry(opts.kafkaMessage, err => { + assert.ifError(err); + assert.strictEqual(tasks.length, 1); + assert(opts.validateTask(tasks[0])); + done(); + }); + }); + }); + }); }); diff --git a/tests/utils/kafkaEntries.js b/tests/utils/kafkaEntries.js index eea3c5307..c5f3fcfed 100644 --- a/tests/utils/kafkaEntries.js +++ b/tests/utils/kafkaEntries.js @@ -6,4 +6,24 @@ const replicationEntry = { value: '{"type":"put","bucket":"queue-populator-test-bucket","key":"hosts\\u000098500086134471999999RG001 0","value":"{\\"md-model-version\\":2,\\"owner-display-name\\":\\"Bart\\",\\"owner-id\\":\\"79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be\\",\\"content-length\\":542,\\"content-type\\":\\"text/plain\\",\\"last-modified\\":\\"2017-07-13T02:44:25.519Z\\",\\"content-md5\\":\\"01064f35c238bd2b785e34508c3d27f4\\",\\"x-amz-version-id\\":\\"null\\",\\"x-amz-server-version-id\\":\\"\\",\\"x-amz-storage-class\\":\\"sf\\",\\"x-amz-server-side-encryption\\":\\"\\",\\"x-amz-server-side-encryption-aws-kms-key-id\\":\\"\\",\\"x-amz-server-side-encryption-customer-algorithm\\":\\"\\",\\"x-amz-website-redirect-location\\":\\"\\",\\"acl\\":{\\"Canned\\":\\"private\\",\\"FULL_CONTROL\\":[],\\"WRITE_ACP\\":[],\\"READ\\":[],\\"READ_ACP\\":[]},\\"key\\":\\"\\",\\"location\\":[{\\"key\\":\\"29258f299ddfd65f6108e6cd7bd2aea9fbe7e9e0\\",\\"size\\":542,\\"start\\":0,\\"dataStoreName\\":\\"file\\",\\"dataStoreETag\\":\\"1:01064f35c238bd2b785e34508c3d27f4\\"}],\\"isDeleteMarker\\":false,\\"tags\\":{},\\"replicationInfo\\":{\\"status\\":\\"PENDING\\",\\"backends\\":[{\\"site\\":\\"sf\\",\\"status\\":\\"PENDING\\",\\"dataStoreVersionId\\":\\"B2AqTml1DtEKWJwRiOTh0tgkm8AlyH7W\\"},{\\"site\\":\\"replicationaws\\",\\"status\\":\\"PENDING\\",\\"dataStoreVersionId\\":\\"ob.rop0jdndzwVioi7v.6Q9.v9.6QOGv\\"}],\\"content\\":[\\"DATA\\",\\"METADATA\\"],\\"destination\\":\\"arn:aws:s3:::dummy-dest-bucket\\",\\"storageClass\\":\\"sf\\",\\"role\\":\\"arn:aws:iam::123456789012:role/backbeat\\"},\\"x-amz-meta-s3cmd-attrs\\":\\"uid:0/gname:root/uname:root/gid:0/mode:33188/mtime:1490807629/atime:1499845478/md5:01064f35c238bd2b785e34508c3d27f4/ctime:1490807629\\",\\"versionId\\":\\"98500086134471999999RG001 0\\",\\"isNFS\\":true}"}', }; -module.exports = { replicationEntry }; +const bucketProcessorEntry = { + key: null, + value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b306"},"target":{"bucket":"bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795"},"details":{}}' +}; + +const bucketProcessorV1Entry = { + key: null, + value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b307"},"target":{"bucket":"v1-bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795","taskVersion":"v1"},"details":{}}' +}; + +const bucketProcessorV2Entry = { + key: null, + value: '{"action":"processObjects","contextInfo":{"reqId":"5d37f38aef4b81d3b308"},"target":{"bucket":"v2-bucket","owner":"48ff9529e073aeb075a00f6ba571638698f39cfa2778ebf0ac098da7030b11c5","accountId":"979878005795","taskVersion":"v2"},"details":{}}' +}; + +module.exports = { + replicationEntry, + bucketProcessorEntry, + bucketProcessorV1Entry, + bucketProcessorV2Entry +};