-
Notifications
You must be signed in to change notification settings - Fork 9
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Implement statusSync and statusDefault endpoint support
In order to reduce code duplication, we reuse the syncBucket controller with a few value tweaks so that it can handle both endpoint types. We also move the sync* controller functions to the sync controller in order to reduce controller pollution of the large object and bucket controllers, and invoke these functions directly through the router instead. Lastly, some database layer modifications are done in order to allow bucketId to accept null fields. Signed-off-by: Jeremy Ho <jujaga@gmail.com>
- Loading branch information
Showing
14 changed files
with
201 additions
and
74 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,92 @@ | ||
const { NIL: SYSTEM_USER } = require('uuid'); | ||
|
||
const errorToProblem = require('../components/errorToProblem'); | ||
const { addDashesToUuid, getCurrentIdentity, isTruthy } = require('../components/utils'); | ||
const { objectService, storageService, objectQueueService, userService } = require('../services'); | ||
|
||
const SERVICE = 'ObjectQueueService'; | ||
|
||
/** | ||
* The Sync Controller | ||
*/ | ||
const controller = { | ||
/** | ||
* @function syncBucket | ||
* Synchronizes a bucket | ||
* @param {object} req Express request object | ||
* @param {object} res Express response object | ||
* @param {function} next The next callback function | ||
* @returns {function} Express middleware function | ||
*/ | ||
async syncBucket(req, res, next) { | ||
try { | ||
const allMode = isTruthy(req.query.all); | ||
const bucketId = addDashesToUuid(req.params.bucketId); | ||
const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); | ||
|
||
const dbParams = {}; | ||
if (!allMode) dbParams.bucketId = bucketId; | ||
|
||
const [dbResponse, s3Response] = await Promise.all([ | ||
objectService.searchObjects(dbParams), | ||
storageService.listAllObjectVersions({ bucketId: bucketId, filterLatest: true }) | ||
]); | ||
|
||
// Aggregate and dedupe all file paths to consider | ||
const jobs = [...new Set([ | ||
...dbResponse.map(object => object.path), | ||
...s3Response.DeleteMarkers.map(object => object.Key), | ||
...s3Response.Versions.map(object => object.Key) | ||
])].map(path => ({ path: path, bucketId: bucketId })); | ||
|
||
const response = await objectQueueService.enqueue({ jobs: jobs, full: isTruthy(req.query.full), createdBy: userId }); | ||
res.status(202).json(response); | ||
} catch (e) { | ||
next(errorToProblem(SERVICE, e)); | ||
} | ||
}, | ||
|
||
/** | ||
* @function syncObject | ||
* Synchronizes an object | ||
* @param {object} req Express request object | ||
* @param {object} res Express response object | ||
* @param {function} next The next callback function | ||
* @returns {function} Express middleware function | ||
*/ | ||
async syncObject(req, res, next) { | ||
try { | ||
const bucketId = req.currentObject?.bucketId; | ||
const path = req.currentObject?.path; | ||
const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); | ||
|
||
const response = await objectQueueService.enqueue({ | ||
jobs: [{ path: path, bucketId: bucketId }], | ||
full: isTruthy(req.query.full), | ||
createdBy: userId | ||
}); | ||
res.status(202).json(response); | ||
} catch (e) { | ||
next(errorToProblem(SERVICE, e)); | ||
} | ||
}, | ||
|
||
/** | ||
* @function syncStatus | ||
* Reports on current sync queue size | ||
* @param {object} req Express request object | ||
* @param {object} res Express response object | ||
* @param {function} next The next callback function | ||
* @returns {function} Express middleware function | ||
*/ | ||
async syncStatus(_req, res, next) { | ||
try { | ||
const response = await objectQueueService.queueSize(); | ||
res.status(200).json(response); | ||
} catch (e) { | ||
next(errorToProblem(SERVICE, e)); | ||
} | ||
} | ||
}; | ||
|
||
module.exports = controller; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,22 @@ | ||
const router = require('express').Router(); | ||
|
||
const { syncController } = require('../../controllers'); | ||
const { syncValidator } = require('../../validators'); | ||
const { checkAppMode } = require('../../middleware/authorization'); | ||
const { requireBasicAuth, requireSomeAuth } = require('../../middleware/featureToggle'); | ||
|
||
router.use(checkAppMode); | ||
router.use(requireSomeAuth); | ||
|
||
/** Synchronizes the default bucket */ | ||
router.get('/', requireBasicAuth, syncValidator.syncDefault, (req, res, next) => { | ||
req.params.bucketId = null; | ||
syncController.syncBucket(req, res, next); | ||
}); | ||
|
||
/** Check sync queue size */ | ||
router.get('/status', (req, res, next) => { | ||
syncController.syncStatus(req, res, next); | ||
}); | ||
|
||
module.exports = router; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,18 @@ | ||
const { validate, Joi } = require('express-validation'); | ||
const { type } = require('./common'); | ||
|
||
|
||
const schema = { | ||
syncDefault: { | ||
query: Joi.object({ | ||
full: type.truthy | ||
}) | ||
} | ||
}; | ||
|
||
const validator = { | ||
syncDefault: validate(schema.syncDefault, { statusCode: 422 }) | ||
}; | ||
|
||
module.exports = validator; | ||
module.exports.schema = schema; |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,38 @@ | ||
const jestJoi = require('jest-joi'); | ||
expect.extend(jestJoi.matchers); | ||
|
||
const { schema } = require('../../../src/validators/sync'); | ||
|
||
describe('syncDefault', () => { | ||
|
||
describe('query', () => { | ||
const query = schema.syncDefault.query.describe(); | ||
|
||
describe('full', () => { | ||
const full = query.keys.full; | ||
|
||
it('is a boolean', () => { | ||
expect(full).toBeTruthy(); | ||
expect(full.type).toEqual('boolean'); | ||
}); | ||
|
||
it('contains truthy array', () => { | ||
expect(Array.isArray(full.truthy)).toBeTruthy(); | ||
expect(full.truthy).toHaveLength(12); | ||
}); | ||
|
||
it.each([ | ||
true, 1, 'true', 'TRUE', 't', 'T', 'yes', 'yEs', 'y', 'Y', '1', | ||
false, 0, 'false', 'FALSE', 'f', 'F', 'no', 'nO', 'n', 'N', '0' | ||
])('accepts the schema given %j', (value) => { | ||
const req = { | ||
query: { | ||
full: value | ||
} | ||
}; | ||
|
||
expect(req).toMatchSchema(schema.syncDefault); | ||
}); | ||
}); | ||
}); | ||
}); |