From c851901855422e627606e13c1e3f346ca1294c48 Mon Sep 17 00:00:00 2001 From: tholulomo Date: Tue, 18 Jul 2023 14:48:51 -0400 Subject: [PATCH] feat(#416): Enabling minio s3 bucket storage --- .../src/controllers/fileController.js | 85 ++++++++++++++----- resfulservice/src/middlewares/fileStorage.js | 35 +++++++- resfulservice/src/middlewares/index.js | 3 +- resfulservice/src/routes/files.js | 5 +- 4 files changed, 101 insertions(+), 27 deletions(-) diff --git a/resfulservice/src/controllers/fileController.js b/resfulservice/src/controllers/fileController.js index 9f174ef0..c76f019b 100644 --- a/resfulservice/src/controllers/fileController.js +++ b/resfulservice/src/controllers/fileController.js @@ -3,19 +3,18 @@ const { PassThrough } = require('stream'); const fsFiles = require('../models/fsFiles'); const latency = require('../middlewares/latencyTimer'); const { errorWriter, successWriter } = require('../utils/logWriter'); -const { deleteFile, findFile } = require('../utils/fileManager'); -const { SupportFileResponseHeaders } = require('../../config/constant'); +const FileManager = require('../utils/fileManager'); +const { SupportedFileResponseHeaders } = require('../../config/constant'); +const minioClient = require('../utils/minio'); +const { MinioBucket } = require('../../config/constant'); -const _createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end(); +exports._createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end(); exports.imageMigration = async (req, res, next) => { const { imageType } = req.params; try { - const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { - bucketName: 'fs' - }); - + const bucket = this.connectToMongoBucket(); const files = await bucket .find({ filename: { $regex: imageType } }) .limit(10) @@ -29,37 +28,52 @@ exports.imageMigration = async (req, res, next) => { }; exports.fileContent = async (req, res, next) => { + const { fileId } = req.params; try { - if (req.query.isDirectory) { - const { fileStream, ext } = await findFile(req); + if (req.query.isFileStore) { + const { fileStream, ext } = await FileManager.findFile(req); if (!fileStream) { + // TODO (@TOLU): Refactor later as this is duplicated below Ln 67, also used in Ln 51 res.setHeader('Content-Type', 'image/png'); latency.latencyCalculator(res); - return _createEmptyStream().pipe(res); + return this._createEmptyStream().pipe(res); } - res.setHeader('Content-Type', SupportFileResponseHeaders[ext]); latency.latencyCalculator(res); + res.setHeader('Content-Type', SupportedFileResponseHeaders[ext]); return fileStream.pipe(res); } - const { fileId } = req.params; - const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { - bucketName: 'fs' - }); + if (req.query.isStore) { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const dataStream = await minioClient.getObject(bucketName, fileId); + + if (!dataStream) { + res.setHeader('Content-Type', 'image/png'); + latency.latencyCalculator(res); + return this._createEmptyStream().pipe(res); + } + + const { ext } = FileManager.getFileExtension(fileId); + res.setHeader('Content-Type', SupportedFileResponseHeaders[ext ?? 'image/png']); + latency.latencyCalculator(res); + return dataStream.pipe(res); + } + + const bucket = this.connectToMongoBucket(); const _id = new mongoose.Types.ObjectId(fileId); const exist = await fsFiles.findById(_id).limit(1); if (!exist) { res.setHeader('Content-Type', 'image/png'); latency.latencyCalculator(res); - return _createEmptyStream().pipe(res); + return this._createEmptyStream().pipe(res); } const downloadStream = bucket.openDownloadStream(_id); latency.latencyCalculator(res); downloadStream.pipe(res); } catch (error) { - next(errorWriter(req, 'Error fetching file', 'fileContent', 500)); + next(errorWriter(req, `${error.message ?? 'Error fetching file'}`, 'fileContent', 500)); } }; @@ -75,15 +89,46 @@ exports.uploadFile = async (req, res, next) => { } }; -exports.deleteFile = (req, res, next) => { +exports.deleteFile = async (req, res, next) => { const filesDirectory = req.env?.FILES_DIRECTORY; const { fileId } = req.params; const filePath = `${filesDirectory}/${fileId}`; try { - deleteFile(filePath, req); + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + + await minioClient.removeObject(bucketName, fileId); + FileManager.deleteFile(filePath, req); latency.latencyCalculator(res); return res.sendStatus(200); } catch (err) { - next(errorWriter(req, 'Error deleting files', 'deleteFile', 500)); + next(errorWriter(req, `${err.message ?? 'Error deleting files'}`, 'deleteFile', 500)); } }; + +exports.findFiles = (req, res) => { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const query = req.query.filename; + + const objectsStream = minioClient.listObjects(bucketName, query, true); + const foundFiles = []; + objectsStream.on('data', (obj) => { + foundFiles.push(obj.name); + }); + + objectsStream.on('end', () => { + res.json({ files: foundFiles }); + }); + + objectsStream.on('error', (err) => { + req.logger.error(err); + res.status(500).json({ error: 'Error finding files in Minio' }); + }); +}; + +// TODO (@TOLU): Move to utils folder +exports.connectToMongoBucket = () => { + const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, { + bucketName: 'fs' + }); + return bucket; +}; diff --git a/resfulservice/src/middlewares/fileStorage.js b/resfulservice/src/middlewares/fileStorage.js index ee4e36d3..f4c6f61d 100644 --- a/resfulservice/src/middlewares/fileStorage.js +++ b/resfulservice/src/middlewares/fileStorage.js @@ -1,7 +1,8 @@ -const path = require('path'); -const express = require('express'); const multer = require('multer'); const { uniqueNamesGenerator, adjectives, names, animals } = require('unique-names-generator'); +const minioClient = require('../utils/minio'); +const { deleteFile } = require('../utils/fileManager'); +const { MinioBucket } = require('../../config/constant'); const shortName = uniqueNamesGenerator({ dictionaries: [adjectives, animals, names], @@ -39,9 +40,35 @@ const fileFilter = (req, file, cb) => { const fileMgr = multer({ storage: fileStorage, fileFilter }).fields([{ name: 'uploadfile', maxCount: 20 }]); -const fileServer = express.static(path.join(__dirname, 'filestore')); +const minioUpload = (req, res, next) => { + const files = req.files?.uploadfile; + if (!files) { + return next(); + } + + files.forEach(file => { + minioPutObject(file, req); + }); + next(); +}; + +const minioPutObject = (file, req) => { + const bucketName = req.env.MINIO_BUCKET ?? MinioBucket; + const metaData = { + 'Content-Type': file.mimetype, + 'X-Amz-Meta-Testing': '1234' + }; + minioClient.fPutObject(bucketName, file.filename, file.path, metaData, (err, objInfo) => { + if (err) { + console.log(err); + } + + deleteFile(file.path, req); + }); +}; module.exports = { fileMgr, - fileServer + minioUpload, + minioPutObject }; diff --git a/resfulservice/src/middlewares/index.js b/resfulservice/src/middlewares/index.js index 084ac43e..260a4286 100644 --- a/resfulservice/src/middlewares/index.js +++ b/resfulservice/src/middlewares/index.js @@ -1,7 +1,7 @@ const express = require('express'); const acceptedHeaders = require('./accept'); const getEnv = require('./parseEnv'); -const { fileMgr, fileServer } = require('./fileStorage'); +const { fileMgr } = require('./fileStorage'); const { logParser, mmLogger } = require('./loggerService'); const swaggerService = require('./swagger-service'); @@ -17,7 +17,6 @@ const globalMiddleWare = async (app) => { app.use(express.urlencoded({ extended: true })); app.use((req, res, next) => logParser(log, req, next)); app.use(fileMgr); - app.use('/mm_files', fileServer); app.use(acceptedHeaders); app.use(getEnv); }; diff --git a/resfulservice/src/routes/files.js b/resfulservice/src/routes/files.js index 9674aab4..cdda1b03 100644 --- a/resfulservice/src/routes/files.js +++ b/resfulservice/src/routes/files.js @@ -3,12 +3,15 @@ const router = express.Router(); const fileController = require('../controllers/fileController'); const isAuth = require('../middlewares/isAuth'); const { latencyTimer } = require('../middlewares/latencyTimer'); +const { minioUpload } = require('../middlewares/fileStorage'); const { validateImageType, validateFileId, validateFileDownload } = require('../middlewares/validations'); +// Todo: Contemplating if this is needed - Will remove if router.route('/:fileId([^/]*)') works fine along with its controller +// router.route('/').get(latencyTimer, fileController.findFiles); router.route('/:fileId([^/]*)') .get(validateFileDownload, latencyTimer, fileController.fileContent) .delete(isAuth, validateFileId, latencyTimer, fileController.deleteFile); router.route('/image_migration/:imageType').get(validateImageType, latencyTimer, fileController.imageMigration); -router.route('/upload').post(latencyTimer, fileController.uploadFile); +router.route('/upload').post(latencyTimer, minioUpload, fileController.uploadFile); module.exports = router;