From 7b210a883a52d61b064548c3848f60d742a27db5 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Tue, 9 Aug 2022 11:56:44 -0400 Subject: [PATCH 1/3] BB-263 Blacklist lifecycle buckets and accounts (cherry picked from commit 1f741b64577c0c7c903fc88db59b8ab7d04dd6f2) (cherry picked from commit ac0addd6a878260ddb51797a34d0b481663c07e2) --- .../lifecycle/LifecycleConfigValidator.js | 6 + .../lifecycle/conductor/LifecycleConductor.js | 73 ++++- .../unit/lifecycle/LifecycleConductor.spec.js | 276 ++++++++++++++++++ 3 files changed, 340 insertions(+), 15 deletions(-) diff --git a/extensions/lifecycle/LifecycleConfigValidator.js b/extensions/lifecycle/LifecycleConfigValidator.js index e070bde57..100c56b6c 100644 --- a/extensions/lifecycle/LifecycleConfigValidator.js +++ b/extensions/lifecycle/LifecycleConfigValidator.js @@ -31,6 +31,12 @@ const joiSchema = joi.object({ backlogControl: joi.object({ enabled: joi.boolean().default(true), }).default({ enabled: true }), + filter: joi.object({ + deny: joi.object({ + buckets: joi.array().items(joi.string()), + accounts: joi.array().items(joi.string()), + }), + }), probeServer: probeServerJoi.default(), vaultAdmin: hostPortJoi, circuitBreaker: joi.object().optional(), diff --git a/extensions/lifecycle/conductor/LifecycleConductor.js b/extensions/lifecycle/conductor/LifecycleConductor.js index c73b69cd5..7a1286baf 100644 --- a/extensions/lifecycle/conductor/LifecycleConductor.js +++ b/extensions/lifecycle/conductor/LifecycleConductor.js @@ -34,6 +34,7 @@ const DEFAULT_CRON_RULE = '* * * * *'; const DEFAULT_CONCURRENCY = 10; const BUCKET_CHECKPOINT_PUSH_NUMBER = 50; const BUCKET_CHECKPOINT_PUSH_NUMBER_BUCKETD = 50; +const ACCOUNT_SPLITTER = ':'; const LIFEYCLE_CONDUCTOR_CLIENT_ID = 'lifecycle-conductor'; @@ -116,6 +117,11 @@ class LifecycleConductor { // - some entries are expired for each listing this._accountIdCache = new AccountIdCache(this._concurrency); + const blacklist = (this.lcConfig.conductor.filter && this.lcConfig.conductor.filter.deny) || {}; + this.bucketsBlacklisted = new Set(blacklist.buckets); + const accountCanonicalIds = this._getAccountCanonicalIds(blacklist.accounts); + this.accountsBlacklisted = new Set(accountCanonicalIds); + this.logger = new Logger('Backbeat:Lifecycle:Conductor'); this.vaultClientWrapper = new VaultClientWrapper( LIFEYCLE_CONDUCTOR_CLIENT_ID, @@ -152,6 +158,30 @@ class LifecycleConductor { } } + /** + * Extract account canonical ids from configuration filter accounts + * + * @param {array} accounts from filter config - + * format: [account1:eb288756448dc58f61482903131e7ae533553d20b52b0e2ef80235599a1b9143] + * @return {array} account canonical ids + */ + _getAccountCanonicalIds(accounts) { + if (!accounts) { + return []; + } + return accounts.reduce((store, account) => { + const split = account.split(ACCOUNT_SPLITTER); + if (split.length === 2) { + store.push(split[1]); + } + return store; + }, []); + } + + _isBlacklisted(canonicalId, bucketName) { + return this.bucketsBlacklisted.has(bucketName) || this.accountsBlacklisted.has(canonicalId); + } + getBucketsZkPath() { return `${this.lcConfig.zookeeperPath}/data/buckets`; } @@ -467,18 +497,30 @@ class LifecycleConductor { return cb(err); } - const batch = buckets.map(bucket => { - const [canonicalId, bucketUID, bucketName] = - bucket.split(':'); - if (!canonicalId || !bucketUID || !bucketName) { - log.error( - 'malformed zookeeper bucket entry, skipping', - { zkPath: zkBucketsPath, bucket }); - return null; - } + const batch = buckets + .filter(bucket => { + const [canonicalId, bucketUID, bucketName] = + bucket.split(':'); + if (!canonicalId || !bucketUID || !bucketName) { + log.error( + 'malformed zookeeper bucket entry, skipping', + { zkPath: zkBucketsPath, bucket }); + return false; + } - return { canonicalId, bucketName }; - }); + if (this._isBlacklisted(canonicalId, bucketName)) { + return false; + } + + return true; + }) + .map(bucket => { + const split = bucket.split(':'); + const canonicalId = split[0]; + const bucketName = split[2]; + + return { canonicalId, bucketName }; + }); queue.push(batch); return process.nextTick(cb, null, batch.length); @@ -578,10 +620,11 @@ class LifecycleConductor { result.Contents.forEach(o => { marker = o.key; const [canonicalId, bucketName] = marker.split(constants.splitter); - nEnqueued += 1; - queue.push({ canonicalId, bucketName }); - - this.lastSentId = o.key; + if (!this._isBlacklisted(canonicalId, bucketName)) { + nEnqueued += 1; + queue.push({ canonicalId, bucketName }); + this.lastSentId = o.key; + } if (nEnqueued % BUCKET_CHECKPOINT_PUSH_NUMBER_BUCKETD === 0) { needCheckpoint = true; } diff --git a/tests/unit/lifecycle/LifecycleConductor.spec.js b/tests/unit/lifecycle/LifecycleConductor.spec.js index 9800693b2..5c373e25b 100644 --- a/tests/unit/lifecycle/LifecycleConductor.spec.js +++ b/tests/unit/lifecycle/LifecycleConductor.spec.js @@ -2,6 +2,7 @@ const assert = require('assert'); const sinon = require('sinon'); +const fakeLogger = require('../../utils/fakeLogger'); const LifecycleConductor = require( '../../../extensions/lifecycle/conductor/LifecycleConductor'); @@ -25,6 +26,94 @@ const testTask = { isLifecycled: true, }; +const accountName1 = 'account1'; +const account1 = 'ab288756448dc58f61482903131e7ae533553d20b52b0e2ef80235599a1b9143'; +const account2 = 'cd288756448dc58f61482903131e7ae533553d20b52b0e2ef80235599a1b9144'; +const bucket1 = 'bucket1'; +const bucket2 = 'bucket2'; + +class Queue { + constructor() { + this.tasks = []; + } + + push(task) { + // task can be either an object or an array of object. + if (Array.isArray(task)) { + this.tasks.push(...task); + } else { + this.tasks.push(task); + } + } + + list() { + return this.tasks; + } + + length() { + return this.tasks.length; + } + + clean() { + this.tasks = []; + } +} + +function makeLifecycleConductorWithFilters(options) { + const { bucketsDenied, accountsDenied, bucketSource } = options; + const lifecycleConfig = { + ...lcConfig, + zookeeperPath: '/test/lifecycle', + conductor: { + bucketSource, + filter: { + deny: { + buckets: bucketsDenied, + accounts: accountsDenied, + }, + }, + }, + }; + + const lcConductor = new LifecycleConductor(zkConfig.zookeeper, + kafkaConfig, lifecycleConfig, repConfig, s3Config); + + lcConductor._zkClient = { + getChildren: (a, b, cb) => { + cb(null, [ + `${account1}:bucketuid123:${bucket1}`, + `${account2}:bucketuid456:${bucket2}`, + ]); + }, + getData: (a, cb) => cb(null, null, null), + setData: (a, b, c, cb) => cb(null, {}), + }; + + lcConductor._bucketClient = { + listObject: (a, b, c, cb) => { + if (c.marker) { + return cb(null, JSON.stringify({ + Contents: [ + { + key: `${account2}..|..${bucket2}`, + }, + ], + })); + } + return cb(null, JSON.stringify({ + Contents: [ + { + key: `${account1}..|..${bucket1}`, + }, + ], + IsTruncated: true, + })); + }, + }; + + return lcConductor; +} + describe('Lifecycle Conductor', () => { let conductor; let lcc; @@ -304,4 +393,191 @@ describe('Lifecycle Conductor', () => { }); })); }); + + describe('listBuckets', () => { + const queue = new Queue(); + + beforeEach(() => { + queue.clean(); + }); + + it('should list buckets from zookeeper', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketSource: 'zookeeper', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 2); + assert.strictEqual(queue.length(), 2); + const expectedQueue = [ + { + canonicalId: account1, + bucketName: bucket1, + }, + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should list buckets from bucketd', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketSource: 'bucketd', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 2); + assert.strictEqual(queue.length(), 2); + + const expectedQueue = [ + { + canonicalId: account1, + bucketName: bucket1, + }, + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter out invalid buckets when listing from zookeeper', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketSource: 'zookeeper', + }); + sinon.stub(lcConductor._zkClient, 'getChildren').yields(null, [ + `${account1}:bucketuid123:${bucket1}`, + `${account2}:bucketuid456:${bucket2}`, + 'invalid:bucketuid789', + 'invalid', + ]); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 2); + assert.strictEqual(queue.length(), 2); + const expectedQueue = [ + { + canonicalId: account1, + bucketName: bucket1, + }, + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by bucket when listing from zookeeper', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketsDenied: [bucket1], + bucketSource: 'zookeeper', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 1); + assert.strictEqual(queue.length(), 1); + const expectedQueue = [ + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by bucket when listing from bucketd', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketsDenied: [bucket1], + bucketSource: 'bucketd', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 1); + assert.strictEqual(queue.length(), 1); + const expectedQueue = [ + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by account when listing from zookeeper', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketSource: 'zookeeper', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 1); + assert.strictEqual(queue.length(), 1); + const expectedQueue = [ + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by account when listing from bucketd', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketSource: 'bucketd', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 1); + assert.strictEqual(queue.length(), 1); + const expectedQueue = [ + { + canonicalId: account2, + bucketName: bucket2, + }, + ]; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by account and bucket when listing from zookeeper', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketsDenied: [bucket2], + bucketSource: 'zookeeper', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 0); + assert.strictEqual(queue.length(), 0); + const expectedQueue = []; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + + it('should filter by account and bucket when listing from bucketd', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketsDenied: [bucket2], + bucketSource: 'bucketd', + }); + lcConductor.listBuckets(queue, fakeLogger, (err, length) => { + assert.strictEqual(length, 0); + assert.strictEqual(queue.length(), 0); + const expectedQueue = []; + assert.deepStrictEqual(queue.list(), expectedQueue); + done(); + }); + }); + }); }); From 1dbeccbf267e8bfd06cc122018f91f10db6a6bd7 Mon Sep 17 00:00:00 2001 From: Nicolas Humbert Date: Thu, 11 Aug 2022 15:47:16 -0400 Subject: [PATCH 2/3] BB-263 optimization - conductor blacklist accounts on listing (cherry picked from commit 749153a530d86a7d245a464bf9dc965d03a2d867) (cherry picked from commit 7dfbd86024bc28e0750f919bc3ed47fa56d7a8fe) --- .../lifecycle/conductor/LifecycleConductor.js | 9 ++++++ .../unit/lifecycle/LifecycleConductor.spec.js | 28 ++++++++++++++----- 2 files changed, 30 insertions(+), 7 deletions(-) diff --git a/extensions/lifecycle/conductor/LifecycleConductor.js b/extensions/lifecycle/conductor/LifecycleConductor.js index 7a1286baf..b8cb9ee33 100644 --- a/extensions/lifecycle/conductor/LifecycleConductor.js +++ b/extensions/lifecycle/conductor/LifecycleConductor.js @@ -121,6 +121,7 @@ class LifecycleConductor { this.bucketsBlacklisted = new Set(blacklist.buckets); const accountCanonicalIds = this._getAccountCanonicalIds(blacklist.accounts); this.accountsBlacklisted = new Set(accountCanonicalIds); + this.onlyBlacklistAccounts = this.bucketsBlacklisted.size === 0 && this.accountsBlacklisted.size > 0; this.logger = new Logger('Backbeat:Lifecycle:Conductor'); this.vaultClientWrapper = new VaultClientWrapper( @@ -624,6 +625,14 @@ class LifecycleConductor { nEnqueued += 1; queue.push({ canonicalId, bucketName }); this.lastSentId = o.key; + // Optimization: + // If we only blacklist by accounts, and the last bucket is blacklisted + // we can skip listing buckets until the next account. + // To start the next listing after the blacklisted account, we construct + // a marker by appending the blacklisted account with a semicolon character. + // 'canonicalid1;' > 'canonicalid1..|..bucketname1' + } else if (this.onlyBlacklistAccounts) { + marker = `${canonicalId};`; } if (nEnqueued % BUCKET_CHECKPOINT_PUSH_NUMBER_BUCKETD === 0) { needCheckpoint = true; diff --git a/tests/unit/lifecycle/LifecycleConductor.spec.js b/tests/unit/lifecycle/LifecycleConductor.spec.js index 5c373e25b..09c65c636 100644 --- a/tests/unit/lifecycle/LifecycleConductor.spec.js +++ b/tests/unit/lifecycle/LifecycleConductor.spec.js @@ -3,6 +3,7 @@ const assert = require('assert'); const sinon = require('sinon'); const fakeLogger = require('../../utils/fakeLogger'); +const { splitter } = require('arsenal').constants; const LifecycleConductor = require( '../../../extensions/lifecycle/conductor/LifecycleConductor'); @@ -59,7 +60,7 @@ class Queue { } } -function makeLifecycleConductorWithFilters(options) { +function makeLifecycleConductorWithFilters(options, markers) { const { bucketsDenied, accountsDenied, bucketSource } = options; const lifecycleConfig = { ...lcConfig, @@ -90,8 +91,9 @@ function makeLifecycleConductorWithFilters(options) { }; lcConductor._bucketClient = { - listObject: (a, b, c, cb) => { - if (c.marker) { + listObject: (a, b, { marker }, cb) => { + if (marker) { + markers.push(marker); return cb(null, JSON.stringify({ Contents: [ { @@ -424,9 +426,10 @@ describe('Lifecycle Conductor', () => { }); it('should list buckets from bucketd', done => { + const markers = []; const lcConductor = makeLifecycleConductorWithFilters({ bucketSource: 'bucketd', - }); + }, markers); lcConductor.listBuckets(queue, fakeLogger, (err, length) => { assert.strictEqual(length, 2); assert.strictEqual(queue.length(), 2); @@ -442,6 +445,8 @@ describe('Lifecycle Conductor', () => { }, ]; assert.deepStrictEqual(queue.list(), expectedQueue); + // test the listing optimization was not applied. + assert.deepStrictEqual(markers, [`${account1}${splitter}${bucket1}`]); done(); }); }); @@ -494,10 +499,11 @@ describe('Lifecycle Conductor', () => { }); it('should filter by bucket when listing from bucketd', done => { + const markers = []; const lcConductor = makeLifecycleConductorWithFilters({ bucketsDenied: [bucket1], bucketSource: 'bucketd', - }); + }, markers); lcConductor.listBuckets(queue, fakeLogger, (err, length) => { assert.strictEqual(length, 1); assert.strictEqual(queue.length(), 1); @@ -508,6 +514,8 @@ describe('Lifecycle Conductor', () => { }, ]; assert.deepStrictEqual(queue.list(), expectedQueue); + // test the listing optimization was not applied. + assert.deepStrictEqual(markers, [`${account1}${splitter}${bucket1}`]); done(); }); }); @@ -532,10 +540,11 @@ describe('Lifecycle Conductor', () => { }); it('should filter by account when listing from bucketd', done => { + const markers = []; const lcConductor = makeLifecycleConductorWithFilters({ accountsDenied: [`${accountName1}:${account1}`], bucketSource: 'bucketd', - }); + }, markers); lcConductor.listBuckets(queue, fakeLogger, (err, length) => { assert.strictEqual(length, 1); assert.strictEqual(queue.length(), 1); @@ -546,6 +555,8 @@ describe('Lifecycle Conductor', () => { }, ]; assert.deepStrictEqual(queue.list(), expectedQueue); + // test the listing optimization was applied + assert.deepStrictEqual(markers, [`${account1};`]); done(); }); }); @@ -566,16 +577,19 @@ describe('Lifecycle Conductor', () => { }); it('should filter by account and bucket when listing from bucketd', done => { + const markers = []; const lcConductor = makeLifecycleConductorWithFilters({ accountsDenied: [`${accountName1}:${account1}`], bucketsDenied: [bucket2], bucketSource: 'bucketd', - }); + }, markers); lcConductor.listBuckets(queue, fakeLogger, (err, length) => { assert.strictEqual(length, 0); assert.strictEqual(queue.length(), 0); const expectedQueue = []; assert.deepStrictEqual(queue.list(), expectedQueue); + // test the listing optimization was not applied. + assert.deepStrictEqual(markers, [`${account1}${splitter}${bucket1}`]); done(); }); }); From 711b074e2dda5cac9ac5fe8fd0c742e17bda5d50 Mon Sep 17 00:00:00 2001 From: Kerkesni Date: Wed, 9 Oct 2024 18:13:32 +0200 Subject: [PATCH 3/3] blacklist lifecycle buckets and accounts when listing from mongodb Issue: BB-537 --- .../lifecycle/conductor/LifecycleConductor.js | 24 +++- .../unit/lifecycle/LifecycleConductor.spec.js | 133 ++++++++++++++++++ 2 files changed, 152 insertions(+), 5 deletions(-) diff --git a/extensions/lifecycle/conductor/LifecycleConductor.js b/extensions/lifecycle/conductor/LifecycleConductor.js index b8cb9ee33..6456d032f 100644 --- a/extensions/lifecycle/conductor/LifecycleConductor.js +++ b/extensions/lifecycle/conductor/LifecycleConductor.js @@ -705,11 +705,25 @@ class LifecycleConductor { lastSentVersion = stat.version; } + const filter = {}; + if (entry) { + filter._id = { $gt: entry }; + } + + if (this.accountsBlacklisted.size > 0) { + filter['value.owner'] = { $nin: Array.from(this.accountsBlacklisted) }; + } + + if (this.bucketsBlacklisted.size > 0) { + filter._id = { + ...filter._id, + $nin: Array.from(this.bucketsBlacklisted), + }; + } + const cursor = this._mongodbClient .getCollection('__metastore') - .find( - entry ? { _id: { $gt: entry } } : {} - ) + .find(filter) .project({ '_id': 1, 'value.owner': 1, 'value.lifecycleConfiguration': 1 }); return done(null, cursor); @@ -718,8 +732,8 @@ class LifecycleConductor { async.during( test => process.nextTick( () => cursor.hasNext() - .then(res => test(null, res)) - .catch(test)), + .then(res => test(null, res)) + .catch(test)), next => { const breakerState = this._circuitBreaker.state; const queueInfo = { diff --git a/tests/unit/lifecycle/LifecycleConductor.spec.js b/tests/unit/lifecycle/LifecycleConductor.spec.js index 09c65c636..1b82032f7 100644 --- a/tests/unit/lifecycle/LifecycleConductor.spec.js +++ b/tests/unit/lifecycle/LifecycleConductor.spec.js @@ -593,5 +593,138 @@ describe('Lifecycle Conductor', () => { done(); }); }); + + it('should not use any filter when listing from from mongodb', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketSource: 'mongodb', + }, []); + const findStub = sinon.stub().returns({ + project: () => ({ + hasNext: sinon.stub().resolves(false), + }) + }); + lcConductor._mongodbClient = { + getIndexingJobs: (_, cb) => cb(null, ['job1', 'job2']), + getCollection: () => ({ find: findStub }) + }; + sinon.stub(lcConductor._zkClient, 'getData').yields(null, null, null); + lcConductor.listBuckets(queue, fakeLogger, err => { + assert.ifError(err); + assert.deepEqual(findStub.getCall(0).args[0], {}); + done(); + }); + }); + + it('should filter by account when listing from from mongodb', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketSource: 'mongodb', + }, []); + const findStub = sinon.stub().returns({ + project: () => ({ + hasNext: sinon.stub().resolves(false), + }) + }); + lcConductor._mongodbClient = { + getIndexingJobs: (_, cb) => cb(null, ['job1', 'job2']), + getCollection: () => ({ find: findStub }) + }; + sinon.stub(lcConductor._zkClient, 'getData').yields(null, null, null); + lcConductor.listBuckets(queue, fakeLogger, err => { + assert.ifError(err); + assert.deepEqual(findStub.getCall(0).args[0], { + 'value.owner': { + $nin: [ + 'ab288756448dc58f61482903131e7ae533553d20b52b0e2ef80235599a1b9143', + ], + }, + }); + done(); + }); + }); + + it('should filter by bucket when listing from from mongodb', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketsDenied: [bucket2], + bucketSource: 'mongodb', + }, []); + const findStub = sinon.stub().returns({ + project: () => ({ + hasNext: sinon.stub().resolves(false), + }) + }); + lcConductor._mongodbClient = { + getIndexingJobs: (_, cb) => cb(null, ['job1', 'job2']), + getCollection: () => ({ find: findStub }) + }; + sinon.stub(lcConductor._zkClient, 'getData').yields(null, null, null); + lcConductor.listBuckets(queue, fakeLogger, err => { + assert.ifError(err); + assert.deepEqual(findStub.getCall(0).args[0], { + _id: { + $nin: ['bucket2'], + }, + }); + done(); + }); + }); + + it('should use the resume marker when listing from from mongodb', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + bucketSource: 'mongodb', + }, []); + const findStub = sinon.stub().returns({ + project: () => ({ + hasNext: sinon.stub().resolves(false), + }) + }); + lcConductor._mongodbClient = { + getIndexingJobs: (_, cb) => cb(null, ['job1', 'job2']), + getCollection: () => ({ find: findStub }) + }; + sinon.stub(lcConductor._zkClient, 'getData').yields(null, Buffer.from('bucket1'), null); + lcConductor.listBuckets(queue, fakeLogger, err => { + assert.ifError(err); + assert.deepEqual(findStub.getCall(0).args[0], { + _id: { + $gt: 'bucket1', + }, + }); + done(); + }); + }); + + it('should filter by account and bucket when listing from from mongodb', done => { + const lcConductor = makeLifecycleConductorWithFilters({ + accountsDenied: [`${accountName1}:${account1}`], + bucketsDenied: [bucket2], + bucketSource: 'mongodb', + }, []); + const findStub = sinon.stub().returns({ + project: () => ({ + hasNext: sinon.stub().resolves(false), + }) + }); + lcConductor._mongodbClient = { + getIndexingJobs: (_, cb) => cb(null, ['job1', 'job2']), + getCollection: () => ({ find: findStub }) + }; + sinon.stub(lcConductor._zkClient, 'getData').yields(null, Buffer.from('bucket1'), null); + lcConductor.listBuckets(queue, fakeLogger, err => { + assert.ifError(err); + assert.deepEqual(findStub.getCall(0).args[0], { + '_id': { + $gt: 'bucket1', + $nin: ['bucket2'], + }, + 'value.owner': { + $nin: [ + 'ab288756448dc58f61482903131e7ae533553d20b52b0e2ef80235599a1b9143', + ], + }, + }); + done(); + }); + }); }); });