feat(#416): Enabling minio s3 bucket storage

resfulservice/src/controllers/fileController.js

@@ -3,19 +3,18 @@ const { PassThrough } = require('stream');
 const fsFiles = require('../models/fsFiles');
 const latency = require('../middlewares/latencyTimer');
 const { errorWriter, successWriter } = require('../utils/logWriter');
-const { deleteFile, findFile } = require('../utils/fileManager');
-const { SupportFileResponseHeaders } = require('../../config/constant');
+const FileManager = require('../utils/fileManager');
+const { SupportedFileResponseHeaders } = require('../../config/constant');
+const minioClient = require('../utils/minio');
+const { MinioBucket } = require('../../config/constant');
 
-const _createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end();
+exports._createEmptyStream = () => new PassThrough('data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAIAQMAAAD+wSzIAAAABlBMVEX///+/v7+jQ3Y5AAAADklEQVQI12P4AIX8EAgALgAD/aNpbtEAAAAASUVORK5CYII').end();
 
 exports.imageMigration = async (req, res, next) => {
   const { imageType } = req.params;
 
   try {
-    const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
-      bucketName: 'fs'
-    });
-
+    const bucket = this.connectToMongoBucket();
     const files = await bucket
       .find({ filename: { $regex: imageType } })
       .limit(10)
@@ -29,37 +28,52 @@ exports.imageMigration = async (req, res, next) => {
 };
 
 exports.fileContent = async (req, res, next) => {
+  const { fileId } = req.params;
   try {
-    if (req.query.isDirectory) {
-      const { fileStream, ext } = await findFile(req);
+    if (req.query.isFileStore) {
+      const { fileStream, ext } = await FileManager.findFile(req);
 
       if (!fileStream) {
+        // TODO (@TOLU): Refactor later as this is duplicated below Ln 67, also used in Ln 51
         res.setHeader('Content-Type', 'image/png');
         latency.latencyCalculator(res);
-        return _createEmptyStream().pipe(res);
+        return this._createEmptyStream().pipe(res);
       }
-      res.setHeader('Content-Type', SupportFileResponseHeaders[ext]);
       latency.latencyCalculator(res);
+      res.setHeader('Content-Type', SupportedFileResponseHeaders[ext]);
       return fileStream.pipe(res);
     }
 
-    const { fileId } = req.params;
-    const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
-      bucketName: 'fs'
-    });
+    if (req.query.isStore) {
+      const bucketName = req.env.MINIO_BUCKET ?? MinioBucket;
+      const dataStream = await minioClient.getObject(bucketName, fileId);
+
+      if (!dataStream) {
+        res.setHeader('Content-Type', 'image/png');
+        latency.latencyCalculator(res);
+        return this._createEmptyStream().pipe(res);
+      }
+
+      const { ext } = FileManager.getFileExtension(fileId);
+      res.setHeader('Content-Type', SupportedFileResponseHeaders[ext ?? 'image/png']);
+      latency.latencyCalculator(res);
+      return dataStream.pipe(res);
+    }
+
+    const bucket = this.connectToMongoBucket();
 
     const _id = new mongoose.Types.ObjectId(fileId);
     const exist = await fsFiles.findById(_id).limit(1);
     if (!exist) {
       res.setHeader('Content-Type', 'image/png');
       latency.latencyCalculator(res);
-      return _createEmptyStream().pipe(res);
+      return this._createEmptyStream().pipe(res);
     }
     const downloadStream = bucket.openDownloadStream(_id);
     latency.latencyCalculator(res);
     downloadStream.pipe(res);
   } catch (error) {
-    next(errorWriter(req, 'Error fetching file', 'fileContent', 500));
+    next(errorWriter(req, `${error.message ?? 'Error fetching file'}`, 'fileContent', 500));
   }
 };
 
@@ -75,15 +89,46 @@ exports.uploadFile = async (req, res, next) => {
   }
 };
 
-exports.deleteFile = (req, res, next) => {
+exports.deleteFile = async (req, res, next) => {
   const filesDirectory = req.env?.FILES_DIRECTORY;
   const { fileId } = req.params;
   const filePath = `${filesDirectory}/${fileId}`;
   try {
-    deleteFile(filePath, req);
+    const bucketName = req.env.MINIO_BUCKET ?? MinioBucket;
+
+    await minioClient.removeObject(bucketName, fileId);
+    FileManager.deleteFile(filePath, req);
     latency.latencyCalculator(res);
     return res.sendStatus(200);
   } catch (err) {
-    next(errorWriter(req, 'Error deleting files', 'deleteFile', 500));
+    next(errorWriter(req, `${err.message ?? 'Error deleting files'}`, 'deleteFile', 500));
   }
 };
+
+exports.findFiles = (req, res) => {
+  const bucketName = req.env.MINIO_BUCKET ?? MinioBucket;
+  const query = req.query.filename;
+
+  const objectsStream = minioClient.listObjects(bucketName, query, true);
+  const foundFiles = [];
+  objectsStream.on('data', (obj) => {
+    foundFiles.push(obj.name);
+  });
+
+  objectsStream.on('end', () => {
+    res.json({ files: foundFiles });
+  });
+
+  objectsStream.on('error', (err) => {
+    req.logger.error(err);
+    res.status(500).json({ error: 'Error finding files in Minio' });
+  });
+};
+
+// TODO (@TOLU): Move to utils folder
+exports.connectToMongoBucket = () => {
+  const bucket = new mongoose.mongo.GridFSBucket(mongoose.connection.db, {
+    bucketName: 'fs'
+  });
+  return bucket;
+};

resfulservice/src/middlewares/fileStorage.js

@@ -1,7 +1,8 @@
-const path = require('path');
-const express = require('express');
 const multer = require('multer');
 const { uniqueNamesGenerator, adjectives, names, animals } = require('unique-names-generator');
+const minioClient = require('../utils/minio');
+const { deleteFile } = require('../utils/fileManager');
+const { MinioBucket } = require('../../config/constant');
 
 const shortName = uniqueNamesGenerator({
   dictionaries: [adjectives, animals, names],
@@ -39,9 +40,35 @@ const fileFilter = (req, file, cb) => {
 
 const fileMgr = multer({ storage: fileStorage, fileFilter }).fields([{ name: 'uploadfile', maxCount: 20 }]);
 
-const fileServer = express.static(path.join(__dirname, 'filestore'));
+const minioUpload = (req, res, next) => {
+  const files = req.files?.uploadfile;
+  if (!files) {
+    return next();
+  }
+
+  files.forEach(file => {
+    minioPutObject(file, req);
+  });
+  next();
+};
+
+const minioPutObject = (file, req) => {
+  const bucketName = req.env.MINIO_BUCKET ?? MinioBucket;
+  const metaData = {
+    'Content-Type': file.mimetype,
+    'X-Amz-Meta-Testing': '1234'
+  };
+  minioClient.fPutObject(bucketName, file.filename, file.path, metaData, (err, objInfo) => {
+    if (err) {
+      console.log(err);
+    }
+
+    deleteFile(file.path, req);
+  });
+};
 
 module.exports = {
   fileMgr,
-  fileServer
+  minioUpload,
+  minioPutObject
 };

resfulservice/src/middlewares/index.js

@@ -1,7 +1,7 @@
 const express = require('express');
 const acceptedHeaders = require('./accept');
 const getEnv = require('./parseEnv');
-const { fileMgr, fileServer } = require('./fileStorage');
+const { fileMgr } = require('./fileStorage');
 const { logParser, mmLogger } = require('./loggerService');
 const swaggerService = require('./swagger-service');
 
@@ -17,7 +17,6 @@ const globalMiddleWare = async (app) => {
   app.use(express.urlencoded({ extended: true }));
   app.use((req, res, next) => logParser(log, req, next));
   app.use(fileMgr);
-  app.use('/mm_files', fileServer);
   app.use(acceptedHeaders);
   app.use(getEnv);
 };

resfulservice/src/routes/files.js

@@ -3,12 +3,15 @@ const router = express.Router();
 const fileController = require('../controllers/fileController');
 const isAuth = require('../middlewares/isAuth');
 const { latencyTimer } = require('../middlewares/latencyTimer');
+const { minioUpload } = require('../middlewares/fileStorage');
 const { validateImageType, validateFileId, validateFileDownload } = require('../middlewares/validations');
 
+// Todo: Contemplating if this is needed - Will remove if router.route('/:fileId([^/]*)') works fine along with its controller
+// router.route('/').get(latencyTimer, fileController.findFiles);
 router.route('/:fileId([^/]*)')
   .get(validateFileDownload, latencyTimer, fileController.fileContent)
   .delete(isAuth, validateFileId, latencyTimer, fileController.deleteFile);
 router.route('/image_migration/:imageType').get(validateImageType, latencyTimer, fileController.imageMigration);
-router.route('/upload').post(latencyTimer, fileController.uploadFile);
+router.route('/upload').post(latencyTimer, minioUpload, fileController.uploadFile);
 
 module.exports = router;