From 129e59cb3327a44e7273a1101d163ce39c231736 Mon Sep 17 00:00:00 2001 From: Jeremy Ho Date: Mon, 24 Jul 2023 15:47:06 -0700 Subject: [PATCH 1/4] Implement bucketSync endpoint Signed-off-by: Jeremy Ho --- app/src/controllers/bucket.js | 34 +++++++++++++++++++++++- app/src/routes/v1/bucket.js | 5 ++++ app/src/validators/bucket.js | 10 +++++++ app/tests/unit/components/utils.spec.js | 2 ++ app/tests/unit/validators/bucket.spec.js | 30 ++++++++++++++++++++- 5 files changed, 79 insertions(+), 2 deletions(-) diff --git a/app/src/controllers/bucket.js b/app/src/controllers/bucket.js index 1458a7e2..5fd8aab1 100644 --- a/app/src/controllers/bucket.js +++ b/app/src/controllers/bucket.js @@ -15,7 +15,7 @@ const { } = require('../components/utils'); const { redactSecrets } = require('../db/models/utils'); -const { bucketService, storageService, userService } = require('../services'); +const { bucketService, objectQueueService, objectService, storageService, userService } = require('../services'); const SERVICE = 'BucketService'; const secretFields = ['accessKeyId', 'secretAccessKey']; @@ -212,6 +212,38 @@ const controller = { } }, + /** + * @function syncBucket + * Synchronizes a bucket + * @param {object} req Express request object + * @param {object} res Express response object + * @param {function} next The next callback function + * @returns {function} Express middleware function + */ + async syncBucket(req, res, next) { + try { + const bucketId = addDashesToUuid(req.params.bucketId); + const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); + + const [dbResponse, s3Response] = await Promise.all([ + objectService.searchObjects({ bucketId: bucketId }), + storageService.listAllObjectVersions({ bucketId: bucketId, filterLatest: true }) + ]); + + // Aggregate and dedupe all file paths to consider + const jobs = [...new Set([ + ...dbResponse.map(object => object.path), + ...s3Response.DeleteMarkers.map(object => object.Key), + ...s3Response.Versions.map(object => object.Key) + ])].map(path => ({ path: path, bucketId: bucketId })); + + const response = await objectQueueService.enqueue({ jobs: jobs, full: isTruthy(req.query.full), createdBy: userId }); + res.status(202).json(response); + } catch (e) { + next(errorToProblem(SERVICE, e)); + } + }, + /** * @function updateBucket * Updates a bucket diff --git a/app/src/routes/v1/bucket.js b/app/src/routes/v1/bucket.js index 726960c8..15379d40 100644 --- a/app/src/routes/v1/bucket.js +++ b/app/src/routes/v1/bucket.js @@ -44,4 +44,9 @@ router.delete('/:bucketId', bucketValidator.deleteBucket, hasPermission(Permissi bucketController.deleteBucket(req, res, next); }); +/** Synchronizes a bucket */ +router.get('/:bucketId/sync', bucketValidator.syncBucket, hasPermission(Permissions.READ), (req, res, next) => { + bucketController.syncBucket(req, res, next); +}); + module.exports = router; diff --git a/app/src/validators/bucket.js b/app/src/validators/bucket.js index d9586a03..83b729e8 100644 --- a/app/src/validators/bucket.js +++ b/app/src/validators/bucket.js @@ -43,6 +43,15 @@ const schema = { }) }, + syncBucket: { + params: Joi.object({ + bucketId: type.uuidv4.required() + }), + query: Joi.object({ + full: type.truthy + }) + }, + updateBucket: { body: Joi.object().keys({ bucketName: Joi.string().max(255), @@ -64,6 +73,7 @@ const validator = { deleteBucket: validate(schema.deleteBucket, { statusCode: 422 }), headBucket: validate(schema.headBucket, { statusCode: 422 }), readBucket: validate(schema.readBucket, { statusCode: 422 }), + syncBucket: validate(schema.readBucket, { statusCode: 422 }), searchBuckets: validate(schema.searchBuckets, { statusCode: 422 }), updateBucket: validate(schema.updateBucket, { statusCode: 422 }) }; diff --git a/app/tests/unit/components/utils.spec.js b/app/tests/unit/components/utils.spec.js index 01cdc1ac..776e837f 100644 --- a/app/tests/unit/components/utils.spec.js +++ b/app/tests/unit/components/utils.spec.js @@ -435,6 +435,8 @@ describe('isAtPath', () => { [false, undefined, undefined], [false, null, null], [false, '', ''], + [true, '', 'file'], + [false, '', 'file/bleep'], [true, '/', 'file'], [false, '/', 'file/bleep'], [true, 'foo', 'foo/bar'], diff --git a/app/tests/unit/validators/bucket.spec.js b/app/tests/unit/validators/bucket.spec.js index 613c1b19..9708295e 100644 --- a/app/tests/unit/validators/bucket.spec.js +++ b/app/tests/unit/validators/bucket.spec.js @@ -94,7 +94,7 @@ describe('createBucket', () => { bucket: 'ccc', endpoint: 'https://s3.ca', bucketName: 'My Bucket', - active : true + active: true } }; expect(value).toMatchSchema(schema.createBucket); @@ -214,6 +214,34 @@ describe('searchBuckets', () => { }); }); +describe('syncBucket', () => { + + describe('params', () => { + const params = schema.syncBucket.params.describe(); + + describe('bucketId', () => { + const bucketId = params.keys.bucketId; + + it('is the expected schema', () => { + expect(bucketId).toEqual(type.uuidv4.required().describe()); + }); + }); + }); + + describe('query', () => { + const query = schema.syncBucket.query.describe(); + + describe('full', () => { + const full = query.keys.full; + + it('is the expected schema', () => { + expect(full).toEqual(type.truthy.describe()); + }); + }); + }); + +}); + describe('updateBucket', () => { describe('body', () => { From 6b109d054b172896fa6cf76c390a6ee46c6840a5 Mon Sep 17 00:00:00 2001 From: Jeremy Ho Date: Mon, 24 Jul 2023 15:53:36 -0700 Subject: [PATCH 2/4] Implement syncObject endpoint Also fix some minor unit test issues Signed-off-by: Jeremy Ho --- app/src/controllers/object.js | 27 ++++++++++++++++++++ app/src/routes/v1/object.js | 5 ++++ app/src/validators/object.js | 10 ++++++++ app/tests/unit/controllers/object.spec.js | 1 + app/tests/unit/services/objectQueue.spec.js | 19 +++++++++++--- app/tests/unit/validators/object.spec.js | 28 +++++++++++++++++++++ 6 files changed, 86 insertions(+), 4 deletions(-) diff --git a/app/src/controllers/object.js b/app/src/controllers/object.js index 694a8281..723907b3 100644 --- a/app/src/controllers/object.js +++ b/app/src/controllers/object.js @@ -33,6 +33,8 @@ const { bucketPermissionService, metadataService, objectService, + objectQueueService, + syncService, storageService, tagService, userService, @@ -889,6 +891,31 @@ const controller = { } }, + /** + * @function syncObject + * Synchronizes an object + * @param {object} req Express request object + * @param {object} res Express response object + * @param {function} next The next callback function + * @returns {function} Express middleware function + */ + async syncObject(req, res, next) { + try { + const bucketId = req.currentObject?.bucketId; + const path = req.currentObject?.path; + const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); + + const response = await objectQueueService.enqueue({ + jobs: [{ path: path, bucketId: bucketId }], + full: isTruthy(req.query.full), + createdBy: userId + }); + res.status(202).json(response); + } catch (e) { + next(errorToProblem(SERVICE, e)); + } + }, + /** * @function togglePublic * Sets the public flag of an object diff --git a/app/src/routes/v1/object.js b/app/src/routes/v1/object.js index a0a147c2..d6f4fe33 100644 --- a/app/src/routes/v1/object.js +++ b/app/src/routes/v1/object.js @@ -74,6 +74,11 @@ router.delete('/:objectId/metadata', objectValidator.deleteMetadata, requireSome objectController.deleteMetadata(req, res, next); }); +/** Synchronizes an object */ +router.get('/:objectId/sync', objectValidator.syncObject, requireSomeAuth, currentObject, hasPermission(Permissions.READ), (req, res, next) => { + objectController.syncObject(req, res, next); +}); + /** Add tags to an object */ router.patch('/:objectId/tagging', objectValidator.addTags, requireSomeAuth, currentObject, hasPermission(Permissions.UPDATE), (req, res, next) => { objectController.addTags(req, res, next); diff --git a/app/src/validators/object.js b/app/src/validators/object.js index 3e6a4cac..bedd62c5 100644 --- a/app/src/validators/object.js +++ b/app/src/validators/object.js @@ -150,6 +150,15 @@ const schema = { }) }, + syncObject: { + params: Joi.object({ + objectId: type.uuidv4.required() + }), + query: Joi.object({ + full: type.truthy + }) + }, + togglePublic: { params: Joi.object({ objectId: type.uuidv4 @@ -185,6 +194,7 @@ const validator = { replaceMetadata: validate(schema.replaceMetadata, { statusCode: 422 }), replaceTags: validate(schema.replaceTags, { statusCode: 422 }), searchObjects: validate(schema.searchObjects, { statusCode: 422 }), + syncObject: validate(schema.searchObjects, { statusCode: 422 }), togglePublic: validate(schema.togglePublic, { statusCode: 422 }), updateObject: validate(schema.updateObject, { statusCode: 422 }) }; diff --git a/app/tests/unit/controllers/object.spec.js b/app/tests/unit/controllers/object.spec.js index 62af7f94..29cd89ba 100644 --- a/app/tests/unit/controllers/object.spec.js +++ b/app/tests/unit/controllers/object.spec.js @@ -331,6 +331,7 @@ describe('deleteObject', () => { expect(versionCreateSpy).toHaveBeenCalledTimes(1); expect(versionCreateSpy).toHaveBeenCalledWith({ id: 'xyz-789', + isLatest: true, deleteMarker: true, s3VersionId: '1234', }, 'user-123'); diff --git a/app/tests/unit/services/objectQueue.spec.js b/app/tests/unit/services/objectQueue.spec.js index fd5b1091..9d29abfe 100644 --- a/app/tests/unit/services/objectQueue.spec.js +++ b/app/tests/unit/services/objectQueue.spec.js @@ -44,21 +44,32 @@ describe('dequeue', () => { describe('enqueue', () => { it('Inserts a job into the object queue only if it is not already present', async () => { + ObjectQueue.ignore.mockReturnValue([]); const data = { - bucketId: BUCKET_ID, - path: 'path', + jobs: [{ + path: 'path', + bucketId: BUCKET_ID + }], full: true, retries: 0, createdBy: SYSTEM_USER }; - await service.enqueue(data.path, data.bucketId, data.full, data.retries, data.createdBy); + await service.enqueue(data); expect(ObjectQueue.startTransaction).toHaveBeenCalledTimes(1); expect(ObjectQueue.query).toHaveBeenCalledTimes(1); expect(ObjectQueue.query).toBeCalledWith(expect.anything()); expect(ObjectQueue.insert).toHaveBeenCalledTimes(1); - expect(ObjectQueue.insert).toBeCalledWith(expect.objectContaining(data)); + expect(ObjectQueue.insert).toBeCalledWith(expect.arrayContaining([ + expect.objectContaining({ + bucketId: data.jobs[0].bucketId, + createdBy: data.createdBy, + full: data.full, + path: data.jobs[0].path, + retries: data.retries + }) + ])); expect(ObjectQueue.onConflict).toHaveBeenCalledTimes(1); expect(ObjectQueue.ignore).toHaveBeenCalledTimes(1); expect(objectQueueTrx.commit).toHaveBeenCalledTimes(1); diff --git a/app/tests/unit/validators/object.spec.js b/app/tests/unit/validators/object.spec.js index b547460f..1a8757f5 100644 --- a/app/tests/unit/validators/object.spec.js +++ b/app/tests/unit/validators/object.spec.js @@ -530,6 +530,34 @@ describe('searchObjects', () => { }); }); +describe('syncObject', () => { + + describe('params', () => { + const params = schema.syncObject.params.describe(); + + describe('bucketId', () => { + const objectId = params.keys.objectId; + + it('is the expected schema', () => { + expect(objectId).toEqual(type.uuidv4.required().describe()); + }); + }); + }); + + describe('query', () => { + const query = schema.syncObject.query.describe(); + + describe('full', () => { + const full = query.keys.full; + + it('is the expected schema', () => { + expect(full).toEqual(type.truthy.describe()); + }); + }); + }); + +}); + describe('togglePublic', () => { describe('params', () => { From 0baf1cac7e69b3e26d92a11d33014e85da38fa56 Mon Sep 17 00:00:00 2001 From: Jeremy Ho Date: Mon, 24 Jul 2023 16:03:55 -0700 Subject: [PATCH 3/4] Implement listAllObjectVersions storage service Modify listAllObjects to default to empty string when path is only the delimiter. This is needed as pattern matching does not handle the root '/' context well. Signed-off-by: Jeremy Ho --- app/src/services/storage.js | 67 +++++++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/app/src/services/storage.js b/app/src/services/storage.js index 93ecc76f..444f54d2 100644 --- a/app/src/services/storage.js +++ b/app/src/services/storage.js @@ -24,6 +24,8 @@ const { MetadataDirective, TaggingDirective } = require('../components/constants const log = require('../components/log')(module.filename); const utils = require('../components/utils'); +const DELIMITER = '/'; + // Get app configuration const defaultTempExpiresIn = parseInt(config.get('objectStorage.defaultTempExpiresIn'), 10); @@ -244,19 +246,21 @@ const objectStorageService = { */ async listAllObjects({ filePath = undefined, bucketId = undefined, precisePath = true } = {}) { const key = filePath ?? (await utils.getBucket(bucketId)).key; + const path = key !== DELIMITER ? key : ''; + const objects = []; let incomplete = false; let nextToken = undefined; do { const { Contents, IsTruncated, NextContinuationToken } = await this.listObjectsV2({ - filePath: key, + filePath: path, continuationToken: nextToken, bucketId: bucketId }); if (Contents) objects.push( - ...Contents.filter(object => !precisePath || utils.isAtPath(key, object.Key)) + ...Contents.filter(object => !precisePath || utils.isAtPath(path, object.Key)) ); incomplete = IsTruncated; nextToken = NextContinuationToken; @@ -265,6 +269,49 @@ const objectStorageService = { return Promise.resolve(objects); }, + /** + * @function listAllObjectVersions + * Lists all objects in the bucket with the prefix of `filePath`. + * Performs pagination behind the scenes if required. + * @param {string} [options.filePath=undefined] Optional filePath of the objects + * @param {string} [options.bucketId=undefined] Optional bucketId + * @param {boolean} [options.precisePath=true] Optional boolean for filtering results based on the precise path + * @param {boolean} [options.filterLatest=false] Optional boolean for filtering results to only entries with IsLatest being true + * @returns {Promise} An object containg an array of DeleteMarkers and Versions + */ + async listAllObjectVersions({ filePath = undefined, bucketId = undefined, precisePath = true, filterLatest = false } = {}) { + const key = filePath ?? (await utils.getBucket(bucketId)).key; + const path = key !== DELIMITER ? key : ''; + + const deleteMarkers = []; + const versions = []; + + let incomplete = false; + let nextKeyMarker = undefined; + do { + const { DeleteMarkers, Versions, IsTruncated, NextKeyMarker } = await this.listObjectVersion({ + filePath: path, + keyMarker: nextKeyMarker, + bucketId: bucketId + }); + + if (DeleteMarkers) deleteMarkers.push( + ...DeleteMarkers + .filter(object => !precisePath || utils.isAtPath(path, object.Key)) + .filter(object => !filterLatest || object.IsLatest === true) + ); + if (Versions) versions.push( + ...Versions + .filter(object => !precisePath || utils.isAtPath(path, object.Key)) + .filter(object => !filterLatest || object.IsLatest === true) + ); + incomplete = IsTruncated; + nextKeyMarker = NextKeyMarker; + } while (incomplete); + + return Promise.resolve({ DeleteMarkers: deleteMarkers, Versions: versions }); + }, + /** * @deprecated Use `listObjectsV2` instead * @function listObjects @@ -296,11 +343,12 @@ const objectStorageService = { */ async listObjectsV2({ filePath = undefined, continuationToken = undefined, maxKeys = undefined, bucketId = undefined } = {}) { const data = await utils.getBucket(bucketId); + const prefix = data.key !== DELIMITER ? data.key : ''; const params = { Bucket: data.bucket, ContinuationToken: continuationToken, MaxKeys: maxKeys, - Prefix: filePath ?? data.key // Must filter via "prefix" - https://stackoverflow.com/a/56569856 + Prefix: filePath ?? prefix // Must filter via "prefix" - https://stackoverflow.com/a/56569856 }; return this._getS3Client(data).send(new ListObjectsV2Command(params)); @@ -309,15 +357,20 @@ const objectStorageService = { /** * @function ListObjectVersion * Lists the versions for the object at `filePath` - * @param {string} options.filePath The filePath of the object - * @param {string} [options.bucketId] Optional bucketId + * @param {string} [options.filePath=undefined] Optional filePath of the objects + * @param {string} [options.keyMarker=undefined] Optional keyMarker for pagination + * @param {number} [options.maxKeys=undefined] Optional maximum number of keys to return + * @param {string} [options.bucketId=undefined] Optional bucketId * @returns {Promise} The response of the list object version operation */ - async listObjectVersion({ filePath, bucketId = undefined }) { + async listObjectVersion({ filePath = undefined, keyMarker = undefined, maxKeys = undefined, bucketId = undefined } = {}) { const data = await utils.getBucket(bucketId); + const prefix = data.key !== DELIMITER ? data.key : ''; const params = { Bucket: data.bucket, - Prefix: filePath // Must filter via "prefix" - https://stackoverflow.com/a/56569856 + KeyMarker: keyMarker, + MaxKeys: maxKeys, + Prefix: filePath ?? prefix // Must filter via "prefix" - https://stackoverflow.com/a/56569856 }; return this._getS3Client(data).send(new ListObjectVersionsCommand(params)); From 7e2fc688230b32d44a2dcdbe38df26bb0b782252 Mon Sep 17 00:00:00 2001 From: Jeremy Ho Date: Wed, 26 Jul 2023 16:58:35 -0700 Subject: [PATCH 4/4] Implement statusSync and statusDefault endpoint support In order to reduce code duplication, we reuse the syncBucket controller with a few value tweaks so that it can handle both endpoint types. We also move the sync* controller functions to the sync controller in order to reduce controller pollution of the large object and bucket controllers, and invoke these functions directly through the router instead. Lastly, some database layer modifications are done in order to allow bucketId to accept null fields. Signed-off-by: Jeremy Ho --- app/src/controllers/bucket.js | 34 +-------- app/src/controllers/index.js | 1 + app/src/controllers/object.js | 27 -------- app/src/controllers/sync.js | 92 +++++++++++++++++++++++++ app/src/db/models/tables/objectModel.js | 8 ++- app/src/db/models/tables/objectQueue.js | 2 +- app/src/routes/v1/bucket.js | 4 +- app/src/routes/v1/index.js | 5 ++ app/src/routes/v1/object.js | 4 +- app/src/routes/v1/sync.js | 22 ++++++ app/src/validators/index.js | 1 + app/src/validators/sync.js | 18 +++++ app/tests/unit/routes/v1.spec.js | 19 +++-- app/tests/unit/validators/sync.spec.js | 38 ++++++++++ 14 files changed, 201 insertions(+), 74 deletions(-) create mode 100644 app/src/controllers/sync.js create mode 100644 app/src/routes/v1/sync.js create mode 100644 app/src/validators/sync.js create mode 100644 app/tests/unit/validators/sync.spec.js diff --git a/app/src/controllers/bucket.js b/app/src/controllers/bucket.js index 5fd8aab1..1458a7e2 100644 --- a/app/src/controllers/bucket.js +++ b/app/src/controllers/bucket.js @@ -15,7 +15,7 @@ const { } = require('../components/utils'); const { redactSecrets } = require('../db/models/utils'); -const { bucketService, objectQueueService, objectService, storageService, userService } = require('../services'); +const { bucketService, storageService, userService } = require('../services'); const SERVICE = 'BucketService'; const secretFields = ['accessKeyId', 'secretAccessKey']; @@ -212,38 +212,6 @@ const controller = { } }, - /** - * @function syncBucket - * Synchronizes a bucket - * @param {object} req Express request object - * @param {object} res Express response object - * @param {function} next The next callback function - * @returns {function} Express middleware function - */ - async syncBucket(req, res, next) { - try { - const bucketId = addDashesToUuid(req.params.bucketId); - const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); - - const [dbResponse, s3Response] = await Promise.all([ - objectService.searchObjects({ bucketId: bucketId }), - storageService.listAllObjectVersions({ bucketId: bucketId, filterLatest: true }) - ]); - - // Aggregate and dedupe all file paths to consider - const jobs = [...new Set([ - ...dbResponse.map(object => object.path), - ...s3Response.DeleteMarkers.map(object => object.Key), - ...s3Response.Versions.map(object => object.Key) - ])].map(path => ({ path: path, bucketId: bucketId })); - - const response = await objectQueueService.enqueue({ jobs: jobs, full: isTruthy(req.query.full), createdBy: userId }); - res.status(202).json(response); - } catch (e) { - next(errorToProblem(SERVICE, e)); - } - }, - /** * @function updateBucket * Updates a bucket diff --git a/app/src/controllers/index.js b/app/src/controllers/index.js index fd772e33..7e629bad 100644 --- a/app/src/controllers/index.js +++ b/app/src/controllers/index.js @@ -4,6 +4,7 @@ module.exports = { metadataController: require('./metadata'), objectController: require('./object'), objectPermissionController: require('./objectPermission'), + syncController: require('./sync'), userController: require('./user'), tagController: require('./tag'), versionController: require('./version') diff --git a/app/src/controllers/object.js b/app/src/controllers/object.js index 723907b3..694a8281 100644 --- a/app/src/controllers/object.js +++ b/app/src/controllers/object.js @@ -33,8 +33,6 @@ const { bucketPermissionService, metadataService, objectService, - objectQueueService, - syncService, storageService, tagService, userService, @@ -891,31 +889,6 @@ const controller = { } }, - /** - * @function syncObject - * Synchronizes an object - * @param {object} req Express request object - * @param {object} res Express response object - * @param {function} next The next callback function - * @returns {function} Express middleware function - */ - async syncObject(req, res, next) { - try { - const bucketId = req.currentObject?.bucketId; - const path = req.currentObject?.path; - const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); - - const response = await objectQueueService.enqueue({ - jobs: [{ path: path, bucketId: bucketId }], - full: isTruthy(req.query.full), - createdBy: userId - }); - res.status(202).json(response); - } catch (e) { - next(errorToProblem(SERVICE, e)); - } - }, - /** * @function togglePublic * Sets the public flag of an object diff --git a/app/src/controllers/sync.js b/app/src/controllers/sync.js new file mode 100644 index 00000000..d9b9cb69 --- /dev/null +++ b/app/src/controllers/sync.js @@ -0,0 +1,92 @@ +const { NIL: SYSTEM_USER } = require('uuid'); + +const errorToProblem = require('../components/errorToProblem'); +const { addDashesToUuid, getCurrentIdentity, isTruthy } = require('../components/utils'); +const { objectService, storageService, objectQueueService, userService } = require('../services'); + +const SERVICE = 'ObjectQueueService'; + +/** + * The Sync Controller + */ +const controller = { + /** + * @function syncBucket + * Synchronizes a bucket + * @param {object} req Express request object + * @param {object} res Express response object + * @param {function} next The next callback function + * @returns {function} Express middleware function + */ + async syncBucket(req, res, next) { + try { + const allMode = isTruthy(req.query.all); + const bucketId = addDashesToUuid(req.params.bucketId); + const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); + + const dbParams = {}; + if (!allMode) dbParams.bucketId = bucketId; + + const [dbResponse, s3Response] = await Promise.all([ + objectService.searchObjects(dbParams), + storageService.listAllObjectVersions({ bucketId: bucketId, filterLatest: true }) + ]); + + // Aggregate and dedupe all file paths to consider + const jobs = [...new Set([ + ...dbResponse.map(object => object.path), + ...s3Response.DeleteMarkers.map(object => object.Key), + ...s3Response.Versions.map(object => object.Key) + ])].map(path => ({ path: path, bucketId: bucketId })); + + const response = await objectQueueService.enqueue({ jobs: jobs, full: isTruthy(req.query.full), createdBy: userId }); + res.status(202).json(response); + } catch (e) { + next(errorToProblem(SERVICE, e)); + } + }, + + /** + * @function syncObject + * Synchronizes an object + * @param {object} req Express request object + * @param {object} res Express response object + * @param {function} next The next callback function + * @returns {function} Express middleware function + */ + async syncObject(req, res, next) { + try { + const bucketId = req.currentObject?.bucketId; + const path = req.currentObject?.path; + const userId = await userService.getCurrentUserId(getCurrentIdentity(req.currentUser, SYSTEM_USER), SYSTEM_USER); + + const response = await objectQueueService.enqueue({ + jobs: [{ path: path, bucketId: bucketId }], + full: isTruthy(req.query.full), + createdBy: userId + }); + res.status(202).json(response); + } catch (e) { + next(errorToProblem(SERVICE, e)); + } + }, + + /** + * @function syncStatus + * Reports on current sync queue size + * @param {object} req Express request object + * @param {object} res Express response object + * @param {function} next The next callback function + * @returns {function} Express middleware function + */ + async syncStatus(_req, res, next) { + try { + const response = await objectQueueService.queueSize(); + res.status(200).json(response); + } catch (e) { + next(errorToProblem(SERVICE, e)); + } + } +}; + +module.exports = controller; diff --git a/app/src/db/models/tables/objectModel.js b/app/src/db/models/tables/objectModel.js index 7afd81fe..01232709 100644 --- a/app/src/db/models/tables/objectModel.js +++ b/app/src/db/models/tables/objectModel.js @@ -60,7 +60,11 @@ class ObjectModel extends Timestamps(Model) { filterOneOrMany(query, value, 'object.id'); }, filterBucketIds(query, value) { - filterOneOrMany(query, value, 'object.bucketId'); + if (value === null) { + query.whereNull('object.bucketId'); + } else { + filterOneOrMany(query, value, 'object.bucketId'); + } }, filterName(query, value) { filterILike(query, value, 'object.name'); @@ -192,7 +196,7 @@ class ObjectModel extends Timestamps(Model) { path: { type: 'string', minLength: 1, maxLength: 1024 }, public: { type: 'boolean' }, active: { type: 'boolean' }, - bucketId: { type: 'string', maxLength: 255 }, + bucketId: { type: 'string', maxLength: 255, nullable: true }, name: { type: 'string', maxLength: 1024 }, ...stamps }, diff --git a/app/src/db/models/tables/objectQueue.js b/app/src/db/models/tables/objectQueue.js index d5bcf2d1..d0205eb5 100644 --- a/app/src/db/models/tables/objectQueue.js +++ b/app/src/db/models/tables/objectQueue.js @@ -39,7 +39,7 @@ class ObjectModel extends Timestamps(Model) { required: ['path', 'full', 'retries'], properties: { id: { type: 'integer' }, - bucketId: { type: 'string', maxLength: 255 }, + bucketId: { type: 'string', maxLength: 255, nullable: true }, path: { type: 'string', minLength: 1, maxLength: 1024 }, full: { type: 'boolean' }, retries: { type: 'integer' }, diff --git a/app/src/routes/v1/bucket.js b/app/src/routes/v1/bucket.js index 15379d40..cdbac940 100644 --- a/app/src/routes/v1/bucket.js +++ b/app/src/routes/v1/bucket.js @@ -1,7 +1,7 @@ const router = require('express').Router(); const { Permissions } = require('../../components/constants'); -const { bucketController } = require('../../controllers'); +const { bucketController, syncController } = require('../../controllers'); const { bucketValidator } = require('../../validators'); const { requireSomeAuth } = require('../../middleware/featureToggle'); const { checkAppMode, hasPermission } = require('../../middleware/authorization'); @@ -46,7 +46,7 @@ router.delete('/:bucketId', bucketValidator.deleteBucket, hasPermission(Permissi /** Synchronizes a bucket */ router.get('/:bucketId/sync', bucketValidator.syncBucket, hasPermission(Permissions.READ), (req, res, next) => { - bucketController.syncBucket(req, res, next); + syncController.syncBucket(req, res, next); }); module.exports = router; diff --git a/app/src/routes/v1/index.js b/app/src/routes/v1/index.js index f22b00d6..98b3b3a9 100644 --- a/app/src/routes/v1/index.js +++ b/app/src/routes/v1/index.js @@ -12,6 +12,8 @@ router.get('/', (_req, res) => { '/metadata', '/object', '/permission', + '/sync', + '/tagging', '/user', '/version' ] @@ -33,6 +35,9 @@ router.use('/object', require('./object')); /** Permission Router */ router.use('/permission', require('./permission')); +/** Sync Router */ +router.use('/sync', require('./sync')); + /** Tagging Router */ router.use('/tagging', require('./tag')); diff --git a/app/src/routes/v1/object.js b/app/src/routes/v1/object.js index d6f4fe33..d904cf20 100644 --- a/app/src/routes/v1/object.js +++ b/app/src/routes/v1/object.js @@ -1,7 +1,7 @@ const router = require('express').Router(); const { Permissions } = require('../../components/constants'); -const { objectController } = require('../../controllers'); +const { objectController, syncController } = require('../../controllers'); const { objectValidator } = require('../../validators'); const { requireSomeAuth } = require('../../middleware/featureToggle'); const { checkAppMode, currentObject, hasPermission } = require('../../middleware/authorization'); @@ -76,7 +76,7 @@ router.delete('/:objectId/metadata', objectValidator.deleteMetadata, requireSome /** Synchronizes an object */ router.get('/:objectId/sync', objectValidator.syncObject, requireSomeAuth, currentObject, hasPermission(Permissions.READ), (req, res, next) => { - objectController.syncObject(req, res, next); + syncController.syncObject(req, res, next); }); /** Add tags to an object */ diff --git a/app/src/routes/v1/sync.js b/app/src/routes/v1/sync.js new file mode 100644 index 00000000..7630260b --- /dev/null +++ b/app/src/routes/v1/sync.js @@ -0,0 +1,22 @@ +const router = require('express').Router(); + +const { syncController } = require('../../controllers'); +const { syncValidator } = require('../../validators'); +const { checkAppMode } = require('../../middleware/authorization'); +const { requireBasicAuth, requireSomeAuth } = require('../../middleware/featureToggle'); + +router.use(checkAppMode); +router.use(requireSomeAuth); + +/** Synchronizes the default bucket */ +router.get('/', requireBasicAuth, syncValidator.syncDefault, (req, res, next) => { + req.params.bucketId = null; + syncController.syncBucket(req, res, next); +}); + +/** Check sync queue size */ +router.get('/status', (req, res, next) => { + syncController.syncStatus(req, res, next); +}); + +module.exports = router; diff --git a/app/src/validators/index.js b/app/src/validators/index.js index 4af176bd..e19e9e7f 100644 --- a/app/src/validators/index.js +++ b/app/src/validators/index.js @@ -4,6 +4,7 @@ module.exports = { metadataValidator: require('./metadata'), objectValidator: require('./object'), objectPermissionValidator: require('./objectPermission'), + syncValidator: require('./sync'), tagValidator: require('./tag'), userValidator: require('./user'), versionValidator: require('./version'), diff --git a/app/src/validators/sync.js b/app/src/validators/sync.js new file mode 100644 index 00000000..e36d4f95 --- /dev/null +++ b/app/src/validators/sync.js @@ -0,0 +1,18 @@ +const { validate, Joi } = require('express-validation'); +const { type } = require('./common'); + + +const schema = { + syncDefault: { + query: Joi.object({ + full: type.truthy + }) + } +}; + +const validator = { + syncDefault: validate(schema.syncDefault, { statusCode: 422 }) +}; + +module.exports = validator; +module.exports.schema = schema; diff --git a/app/tests/unit/routes/v1.spec.js b/app/tests/unit/routes/v1.spec.js index 5438e635..69145064 100644 --- a/app/tests/unit/routes/v1.spec.js +++ b/app/tests/unit/routes/v1.spec.js @@ -17,13 +17,18 @@ describe(`GET ${basePath}`, () => { expect(response.statusCode).toBe(200); expect(response.body).toBeTruthy(); expect(Array.isArray(response.body.endpoints)).toBeTruthy(); - expect(response.body.endpoints).toHaveLength(7); - expect(response.body.endpoints).toContain('/bucket'); - expect(response.body.endpoints).toContain('/docs'); - expect(response.body.endpoints).toContain('/metadata'); - expect(response.body.endpoints).toContain('/object'); - expect(response.body.endpoints).toContain('/permission'); - expect(response.body.endpoints).toContain('/user'); + expect(response.body.endpoints).toHaveLength(9); + expect(response.body.endpoints).toEqual(expect.arrayContaining([ + '/bucket', + '/docs', + '/metadata', + '/object', + '/permission', + '/sync', + '/tagging', + '/user', + '/version' + ])); }); }); diff --git a/app/tests/unit/validators/sync.spec.js b/app/tests/unit/validators/sync.spec.js new file mode 100644 index 00000000..e1b4dbb6 --- /dev/null +++ b/app/tests/unit/validators/sync.spec.js @@ -0,0 +1,38 @@ +const jestJoi = require('jest-joi'); +expect.extend(jestJoi.matchers); + +const { schema } = require('../../../src/validators/sync'); + +describe('syncDefault', () => { + + describe('query', () => { + const query = schema.syncDefault.query.describe(); + + describe('full', () => { + const full = query.keys.full; + + it('is a boolean', () => { + expect(full).toBeTruthy(); + expect(full.type).toEqual('boolean'); + }); + + it('contains truthy array', () => { + expect(Array.isArray(full.truthy)).toBeTruthy(); + expect(full.truthy).toHaveLength(12); + }); + + it.each([ + true, 1, 'true', 'TRUE', 't', 'T', 'yes', 'yEs', 'y', 'Y', '1', + false, 0, 'false', 'FALSE', 'f', 'F', 'no', 'nO', 'n', 'N', '0' + ])('accepts the schema given %j', (value) => { + const req = { + query: { + full: value + } + }; + + expect(req).toMatchSchema(schema.syncDefault); + }); + }); + }); +});