From 6b638e8c150b0da6831a1727556f3d8511c88eba Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:05:31 -0700 Subject: [PATCH 01/11] Added worker to kill old instances. --- index.js | 5 ++-- lib/tasks/instances/cleanup.js | 43 ++++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 lib/tasks/instances/cleanup.js diff --git a/index.js b/index.js index 28bd497..b0acd44 100644 --- a/index.js +++ b/index.js @@ -10,6 +10,8 @@ var subscribedEvents = [ 'context-version.deleted' ] var queues = { + 'context-version.deleted': require('tasks/context-versions/deleted'), + 'khronos.context-version.deleted': require('tasks/context-versions/deleted'), 'khronos:canary:build': require('tasks/canary/build'), 'khronos:canary:failover': require('tasks/canary/failover'), 'khronos:canary:github-branch': require('tasks/canary/github-branch'), @@ -23,8 +25,6 @@ var queues = { 'khronos:containers:orphan:prune': require('tasks/containers/prune-orphans'), 'khronos:containers:orphan:prune-dock': require('tasks/containers/prune-orphans-dock'), 'khronos:containers:remove': require('tasks/containers/remove'), - 'khronos.context-version.deleted': require('tasks/context-versions/deleted'), - 'context-version.deleted': require('tasks/context-versions/deleted'), 'khronos:context-versions:check-recent-usage': require('tasks/context-versions/check-recent-usage'), 'khronos:context-versions:prune-expired': require('tasks/context-versions/prune-expired'), 'khronos:context-versions:remove-and-protect-instances': require('tasks/context-versions/remove-and-protect-instances'), @@ -33,6 +33,7 @@ var queues = { 'khronos:images:prune': require('tasks/images/prune'), 'khronos:images:prune-dock': require('tasks/images/prune-dock'), 'khronos:images:remove': require('tasks/images/remove'), + 'khronos:instances:cleanup': require('tasks/instances/cleanup'), 'khronos:metrics:container-status': require('tasks/metrics/container-status'), 'khronos:metrics:report-org-container-status': require('tasks/metrics/report-org-container-status'), 'khronos:weave:prune': require('tasks/weave/prune'), diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js new file mode 100644 index 0000000..5703b06 --- /dev/null +++ b/lib/tasks/instances/cleanup.js @@ -0,0 +1,43 @@ +'use strict' + +// external +const Promise = require('bluebird') +const rabbitmqHelper = require('tasks/utils/rabbitmq') +const mongodbHelper = require('tasks/utils/mongodb') +const moment = require('moment') + +// internal +const logger = require('logger').getChild(__filename) + +module.exports = CleanupInstances + +function CleanupInstances () { + var log = logger.child({ + tx: true + }) + log.info('CleanupInstances') + + return Promise.using(mongodbHelper(), function (mongoClient) { + return mongoClient.fetchInstancesAsync({ + masterPod: false, + 'contextVersion.created': { $lt: moment().subtract(7, 'days').toDate() }, + $or: [ + { isolated: { $exists: false } }, + { isIsolationGroupMaster: true } + ] + }) + }) + .then(function (instances) { + log.info({ + instanceCount: instances.length() + }, 'Found instances to cleanup') + return Promise.using(rabbitmqHelper(['instance.delete']), + function (rabbitmq) { + instances.forEach(function (instance) { + rabbitmq.publish('instance.delete', { + instanceId: instance._id + }) + }) + }) + }) +} From b079bdc04a108f2644df9f400b93dbf581479e71 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:14:39 -0700 Subject: [PATCH 02/11] Added test. --- test/unit/tasks/instances/cleanup.js | 71 ++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) create mode 100644 test/unit/tasks/instances/cleanup.js diff --git a/test/unit/tasks/instances/cleanup.js b/test/unit/tasks/instances/cleanup.js new file mode 100644 index 0000000..429cfbf --- /dev/null +++ b/test/unit/tasks/instances/cleanup.js @@ -0,0 +1,71 @@ +'use strict' + +require('loadenv')({ debugName: 'khronos:test' }) + +var chai = require('chai') +var assert = chai.assert +chai.use(require('chai-as-promised')) + +// external +var rabbitmq = require('models/rabbitmq') +var sinon = require('sinon') +require('sinon-as-promised')(require('bluebird')) + +// Internal +const MongoDB = require('models/mongodb') + +// internal (being tested) +var CleanupInstances = require('tasks/instances/cleanup') + +describe('khronos:instances:cleanup', function () { + var mockInstances + beforeEach(function () { + mockInstances = [ + { + id: '1234' + }, + { + id: '5678' + } + ] + sinon.stub(MongoDB.prototype, 'close').yieldsAsync() + sinon.stub(MongoDB.prototype, 'connect').yieldsAsync() + sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync() + sinon.stub(rabbitmq, 'publishTask').resolves() + }) + + afterEach(function () { + MongoDB.prototype.close.restore() + MongoDB.prototype.connect.restore() + MongoDB.prototype.fetchInstances.restore() + rabbitmq.publishTask.restore() + }) + + describe('when there are instances to cleanup', function () { + beforeEach(function () { + MongoDB.prototype.fetchInstances.yieldsAsync(null, mockInstances) + }) + + it('should cleanup the old instances', function (done) { + return assert.isFulfilled(CleanupInstances({})) + .then(function () { + sinon.assert.calledTwice(rabbitmq.publishTask) + sinon.assert.calledWithExactly( + rabbitmq.publishTask, + 'instance.delete', + { + instanceId: '1234' + } + ) + sinon.assert.calledWithExactly( + rabbitmq.publishTask, + 'instance.delete', + { + instanceId: '5678' + } + ) + }) + .asCallback(done) + }) + }) +}) From 8ea0795a1953ecc7e0b003b169f323fdb94bd015 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:20:41 -0700 Subject: [PATCH 03/11] Fixed tests! --- lib/tasks/instances/cleanup.js | 15 ++++++-------- test/unit/tasks/instances/cleanup.js | 30 +++++++++++++++++++++------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js index 5703b06..41de3c6 100644 --- a/lib/tasks/instances/cleanup.js +++ b/lib/tasks/instances/cleanup.js @@ -2,7 +2,7 @@ // external const Promise = require('bluebird') -const rabbitmqHelper = require('tasks/utils/rabbitmq') +const rabbitmq = require('models/rabbitmq') const mongodbHelper = require('tasks/utils/mongodb') const moment = require('moment') @@ -29,15 +29,12 @@ function CleanupInstances () { }) .then(function (instances) { log.info({ - instanceCount: instances.length() + instanceCount: instances.length }, 'Found instances to cleanup') - return Promise.using(rabbitmqHelper(['instance.delete']), - function (rabbitmq) { - instances.forEach(function (instance) { - rabbitmq.publish('instance.delete', { - instanceId: instance._id - }) - }) + instances.forEach(function (instance) { + rabbitmq.publishTask('instance.delete', { + instanceId: instance._id }) + }) }) } diff --git a/test/unit/tasks/instances/cleanup.js b/test/unit/tasks/instances/cleanup.js index 429cfbf..37097a4 100644 --- a/test/unit/tasks/instances/cleanup.js +++ b/test/unit/tasks/instances/cleanup.js @@ -19,18 +19,19 @@ var CleanupInstances = require('tasks/instances/cleanup') describe('khronos:instances:cleanup', function () { var mockInstances + beforeEach(function () { mockInstances = [ { - id: '1234' + _id: '1234' }, { - id: '5678' + _id: '5678' } ] sinon.stub(MongoDB.prototype, 'close').yieldsAsync() sinon.stub(MongoDB.prototype, 'connect').yieldsAsync() - sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync() + sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync(null, mockInstances) sinon.stub(rabbitmq, 'publishTask').resolves() }) @@ -42,22 +43,37 @@ describe('khronos:instances:cleanup', function () { }) describe('when there are instances to cleanup', function () { - beforeEach(function () { - MongoDB.prototype.fetchInstances.yieldsAsync(null, mockInstances) + it('should fetch instances with the propery query parameters', function (done) { + return assert.isFulfilled(CleanupInstances({})) + .then(function () { + sinon.assert.calledOnce(MongoDB.prototype.fetchInstances) + sinon.assert.calledWith( + MongoDB.prototype.fetchInstances, + { + masterPod: false, + 'contextVersion.created': { $lt: sinon.match.date }, + $or: [ + { isolated: { $exists: false } }, + { isIsolationGroupMaster: true } + ] + } + ) + }) + .asCallback(done) }) it('should cleanup the old instances', function (done) { return assert.isFulfilled(CleanupInstances({})) .then(function () { sinon.assert.calledTwice(rabbitmq.publishTask) - sinon.assert.calledWithExactly( + sinon.assert.calledWith( rabbitmq.publishTask, 'instance.delete', { instanceId: '1234' } ) - sinon.assert.calledWithExactly( + sinon.assert.calledWith( rabbitmq.publishTask, 'instance.delete', { From b4bbec4e89f3df606d9c5fa01c7b84268a4dfc72 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:36:14 -0700 Subject: [PATCH 04/11] Added instancesCleanup task to rabbitMQ. --- lib/models/rabbitmq.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/models/rabbitmq.js b/lib/models/rabbitmq.js index 5c872da..97a9c10 100644 --- a/lib/models/rabbitmq.js +++ b/lib/models/rabbitmq.js @@ -39,6 +39,7 @@ const publisher = new RabbitMQ({ 'khronos:images:prune', 'khronos:images:prune-dock', 'khronos:images:remove', + 'khronos:instances:cleanup', 'khronos:metrics:container-status', 'khronos:metrics:report-org-container-status', 'khronos:weave:prune', From 7517a84d150a2f3ca7c262ef47db874883938146 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:48:07 -0700 Subject: [PATCH 05/11] Updated logging. --- lib/tasks/instances/cleanup.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js index 41de3c6..2729f63 100644 --- a/lib/tasks/instances/cleanup.js +++ b/lib/tasks/instances/cleanup.js @@ -15,12 +15,13 @@ function CleanupInstances () { var log = logger.child({ tx: true }) - log.info('CleanupInstances') + const cleanupDate = moment().subtract(7, 'days').toDate() + log.info({cleanupCutoff: cleanupDate}, 'CleanupInstances') return Promise.using(mongodbHelper(), function (mongoClient) { return mongoClient.fetchInstancesAsync({ masterPod: false, - 'contextVersion.created': { $lt: moment().subtract(7, 'days').toDate() }, + 'contextVersion.created': { $lt: cleanupDate }, $or: [ { isolated: { $exists: false } }, { isIsolationGroupMaster: true } @@ -29,7 +30,10 @@ function CleanupInstances () { }) .then(function (instances) { log.info({ - instanceCount: instances.length + instanceCount: instances.length, + instanceIds: instances.map(function (instance) { + return instance._id; + }) }, 'Found instances to cleanup') instances.forEach(function (instance) { rabbitmq.publishTask('instance.delete', { From a9064cc0f3b4dbae816a828681d72846615628fb Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:54:44 -0700 Subject: [PATCH 06/11] Removed semicolon. --- lib/tasks/instances/cleanup.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js index 2729f63..16d10bb 100644 --- a/lib/tasks/instances/cleanup.js +++ b/lib/tasks/instances/cleanup.js @@ -32,7 +32,7 @@ function CleanupInstances () { log.info({ instanceCount: instances.length, instanceIds: instances.map(function (instance) { - return instance._id; + return instance._id }) }, 'Found instances to cleanup') instances.forEach(function (instance) { From cb1745f49ca5ee8016e6a2264ad444098f27775e Mon Sep 17 00:00:00 2001 From: Myztiq Date: Fri, 26 Aug 2016 16:56:41 -0700 Subject: [PATCH 07/11] Added moment.js --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 2024cac..610cc64 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "keypather": "^1.10.1", "loadenv": "^2.2.0", "lodash": "^4.14.0", + "moment": "^2.14.1", "mongodb": "2.1.7", "monitor-dog": "^1.5.0", "ponos": "^4.6.0", From a3ff89e3dcb51225c14c787ff615064175a1eeb7 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Mon, 29 Aug 2016 11:24:51 -0700 Subject: [PATCH 08/11] Updated so we publish instance.expired event. --- lib/models/rabbitmq.js | 3 ++- lib/tasks/instances/cleanup.js | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/models/rabbitmq.js b/lib/models/rabbitmq.js index 97a9c10..679db46 100644 --- a/lib/models/rabbitmq.js +++ b/lib/models/rabbitmq.js @@ -16,7 +16,8 @@ const publisher = new RabbitMQ({ username: process.env.RABBITMQ_USERNAME, password: process.env.RABBITMQ_PASSWORD, events: [ - 'instance.container.health-check.failed' + 'instance.container.health-check.failed', + 'instance.expired' ], tasks: [ 'khronos:canary:build', diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js index 16d10bb..7652fc9 100644 --- a/lib/tasks/instances/cleanup.js +++ b/lib/tasks/instances/cleanup.js @@ -36,7 +36,7 @@ function CleanupInstances () { }) }, 'Found instances to cleanup') instances.forEach(function (instance) { - rabbitmq.publishTask('instance.delete', { + rabbitmq.publishEvent('instance.expired', { instanceId: instance._id }) }) From 11b92dc77eb35af1cdc960b67c9ba4330cafbcce Mon Sep 17 00:00:00 2001 From: Myztiq Date: Mon, 29 Aug 2016 11:31:36 -0700 Subject: [PATCH 09/11] Removed tx from logger. --- lib/tasks/instances/cleanup.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/tasks/instances/cleanup.js b/lib/tasks/instances/cleanup.js index 7652fc9..36b002e 100644 --- a/lib/tasks/instances/cleanup.js +++ b/lib/tasks/instances/cleanup.js @@ -12,11 +12,11 @@ const logger = require('logger').getChild(__filename) module.exports = CleanupInstances function CleanupInstances () { + const cleanupDate = moment().subtract(7, 'days').toDate() var log = logger.child({ - tx: true + cleanupCutoff: cleanupDate }) - const cleanupDate = moment().subtract(7, 'days').toDate() - log.info({cleanupCutoff: cleanupDate}, 'CleanupInstances') + log.info('CleanupInstances') return Promise.using(mongodbHelper(), function (mongoClient) { return mongoClient.fetchInstancesAsync({ From bbb65bf86277880b86d132cedb4115ffbac0d60e Mon Sep 17 00:00:00 2001 From: Myztiq Date: Mon, 29 Aug 2016 11:32:25 -0700 Subject: [PATCH 10/11] publishTask -> publishEvent --- test/unit/tasks/instances/cleanup.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/unit/tasks/instances/cleanup.js b/test/unit/tasks/instances/cleanup.js index 37097a4..17156d0 100644 --- a/test/unit/tasks/instances/cleanup.js +++ b/test/unit/tasks/instances/cleanup.js @@ -32,14 +32,14 @@ describe('khronos:instances:cleanup', function () { sinon.stub(MongoDB.prototype, 'close').yieldsAsync() sinon.stub(MongoDB.prototype, 'connect').yieldsAsync() sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync(null, mockInstances) - sinon.stub(rabbitmq, 'publishTask').resolves() + sinon.stub(rabbitmq, 'publishEvent').resolves() }) afterEach(function () { MongoDB.prototype.close.restore() MongoDB.prototype.connect.restore() MongoDB.prototype.fetchInstances.restore() - rabbitmq.publishTask.restore() + rabbitmq.publishEvent.restore() }) describe('when there are instances to cleanup', function () { @@ -65,16 +65,16 @@ describe('khronos:instances:cleanup', function () { it('should cleanup the old instances', function (done) { return assert.isFulfilled(CleanupInstances({})) .then(function () { - sinon.assert.calledTwice(rabbitmq.publishTask) + sinon.assert.calledTwice(rabbitmq.publishEvent) sinon.assert.calledWith( - rabbitmq.publishTask, + rabbitmq.publishEvent, 'instance.delete', { instanceId: '1234' } ) sinon.assert.calledWith( - rabbitmq.publishTask, + rabbitmq.publishEvent, 'instance.delete', { instanceId: '5678' From 944dbfe6df51dd04baac4e8a080699ccbdd1dc85 Mon Sep 17 00:00:00 2001 From: Myztiq Date: Mon, 29 Aug 2016 11:49:38 -0700 Subject: [PATCH 11/11] instance.delete -> instance.expired --- test/unit/tasks/instances/cleanup.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/unit/tasks/instances/cleanup.js b/test/unit/tasks/instances/cleanup.js index 17156d0..429bb72 100644 --- a/test/unit/tasks/instances/cleanup.js +++ b/test/unit/tasks/instances/cleanup.js @@ -68,14 +68,14 @@ describe('khronos:instances:cleanup', function () { sinon.assert.calledTwice(rabbitmq.publishEvent) sinon.assert.calledWith( rabbitmq.publishEvent, - 'instance.delete', + 'instance.expired', { instanceId: '1234' } ) sinon.assert.calledWith( rabbitmq.publishEvent, - 'instance.delete', + 'instance.expired', { instanceId: '5678' }