SAN-4866 Cleanup Old Instances

index.js

@@ -31,6 +31,7 @@ const tasks = {
   'khronos:images:prune': require('tasks/images/prune'),
   'khronos:images:prune-dock': require('tasks/images/prune-dock'),
   'khronos:images:remove': require('tasks/images/remove'),
+  'khronos:instances:cleanup': require('tasks/instances/cleanup'),
   'khronos:metrics:container-status': require('tasks/metrics/container-status'),
   'khronos:metrics:report-org-container-status': require('tasks/metrics/report-org-container-status'),
   'khronos:weave:prune': require('tasks/weave/prune'),

lib/models/rabbitmq.js

@@ -16,7 +16,8 @@ const publisher = new RabbitMQ({
   username: process.env.RABBITMQ_USERNAME,
   password: process.env.RABBITMQ_PASSWORD,
   events: [
-    'instance.container.health-check.failed'
+    'instance.container.health-check.failed',
+    'instance.expired'
   ],
   tasks: [
     'khronos:canary:build',
@@ -39,6 +40,7 @@ const publisher = new RabbitMQ({
     'khronos:images:prune',
     'khronos:images:prune-dock',
     'khronos:images:remove',
+    'khronos:instances:cleanup',
     'khronos:metrics:container-status',
     'khronos:metrics:report-org-container-status',
     'khronos:weave:prune',

lib/tasks/instances/cleanup.js

@@ -0,0 +1,44 @@
+'use strict'
+
+// external
+const Promise = require('bluebird')
+const rabbitmq = require('models/rabbitmq')
+const mongodbHelper = require('tasks/utils/mongodb')
+const moment = require('moment')
+
+// internal
+const logger = require('logger').getChild(__filename)
+
+module.exports = CleanupInstances
+
+function CleanupInstances () {
+  const cleanupDate = moment().subtract(7, 'days').toDate()
+  var log = logger.child({
+    cleanupCutoff: cleanupDate
+  })
+  log.info('CleanupInstances')
+
+  return Promise.using(mongodbHelper(), function (mongoClient) {
+    return mongoClient.fetchInstancesAsync({
+      masterPod: false,
+      'contextVersion.created': { $lt: cleanupDate },
+      $or: [
+        { isolated: { $exists: false } },
+        { isIsolationGroupMaster: true }
+      ]
+    })
+  })
+    .then(function (instances) {
+      log.info({
+        instanceCount: instances.length,
+        instanceIds: instances.map(function (instance) {
+          return instance._id
+        })
+      }, 'Found instances to cleanup')
+      instances.forEach(function (instance) {
+        rabbitmq.publishEvent('instance.expired', {
+          instanceId: instance._id
+        })
+      })
+    })
+}

package.json

@@ -64,6 +64,7 @@
     "keypather": "^1.10.1",
     "loadenv": "^2.2.0",
     "lodash": "^4.14.0",
+    "moment": "^2.14.1",
     "mongodb": "2.1.7",
     "monitor-dog": "^1.5.0",
     "ponos": "^4.6.0",

test/unit/tasks/instances/cleanup.js

@@ -0,0 +1,87 @@
+'use strict'
+
+require('loadenv')({ debugName: 'khronos:test' })
+
+var chai = require('chai')
+var assert = chai.assert
+chai.use(require('chai-as-promised'))
+
+// external
+var rabbitmq = require('models/rabbitmq')
+var sinon = require('sinon')
+require('sinon-as-promised')(require('bluebird'))
+
+// Internal
+const MongoDB = require('models/mongodb')
+
+// internal (being tested)
+var CleanupInstances = require('tasks/instances/cleanup')
+
+describe('khronos:instances:cleanup', function () {
+  var mockInstances
+
+  beforeEach(function () {
+    mockInstances = [
+      {
+        _id: '1234'
+      },
+      {
+        _id: '5678'
+      }
+    ]
+    sinon.stub(MongoDB.prototype, 'close').yieldsAsync()
+    sinon.stub(MongoDB.prototype, 'connect').yieldsAsync()
+    sinon.stub(MongoDB.prototype, 'fetchInstances').yieldsAsync(null, mockInstances)
+    sinon.stub(rabbitmq, 'publishEvent').resolves()
+  })
+
+  afterEach(function () {
+    MongoDB.prototype.close.restore()
+    MongoDB.prototype.connect.restore()
+    MongoDB.prototype.fetchInstances.restore()
+    rabbitmq.publishEvent.restore()
+  })
+
+  describe('when there are instances to cleanup', function () {
+    it('should fetch instances with the propery query parameters', function (done) {
+      return assert.isFulfilled(CleanupInstances({}))
+        .then(function () {
+          sinon.assert.calledOnce(MongoDB.prototype.fetchInstances)
+          sinon.assert.calledWith(
+            MongoDB.prototype.fetchInstances,
+            {
+              masterPod: false,
+              'contextVersion.created': { $lt: sinon.match.date },
+              $or: [
+                { isolated: { $exists: false } },
+                { isIsolationGroupMaster: true }
+              ]
+            }
+          )
+        })
+        .asCallback(done)
+    })
+
+    it('should cleanup the old instances', function (done) {
+      return assert.isFulfilled(CleanupInstances({}))
+        .then(function () {
+          sinon.assert.calledTwice(rabbitmq.publishEvent)
+          sinon.assert.calledWith(
+            rabbitmq.publishEvent,
+            'instance.expired',
+            {
+              instanceId: '1234'
+            }
+          )
+          sinon.assert.calledWith(
+            rabbitmq.publishEvent,
+            'instance.expired',
+            {
+              instanceId: '5678'
+            }
+          )
+        })
+        .asCallback(done)
+    })
+  })
+})