From 6ff51ad44e106f07b6ac1ac95bb5356a2092e886 Mon Sep 17 00:00:00 2001 From: Hannah Wolfe Date: Wed, 10 Dec 2014 13:28:16 +0000 Subject: [PATCH] Add support for zip import fixes #4607 - moves file checks from db and upload API endpoints to api utils - adds code to accept and then extract a zip and pull out a JSON file - zip handling requires a lot of dependencies - this needs a good refactor --- core/server/api/db.js | 84 +++++++++++++++++++++++++++------------ core/server/api/upload.js | 32 +++++---------- core/server/api/utils.js | 13 ++++++ package.json | 2 + 4 files changed, 84 insertions(+), 47 deletions(-) diff --git a/core/server/api/db.js b/core/server/api/db.js index a63309c53b85..21f4638711f4 100644 --- a/core/server/api/db.js +++ b/core/server/api/db.js @@ -7,19 +7,51 @@ var dataExport = require('../data/export'), Promise = require('bluebird'), _ = require('lodash'), path = require('path'), + os = require('os'), + glob = require('glob'), + uuid = require('node-uuid'), + extract = require('extract-zip'), errors = require('../../server/errors'), canThis = require('../permissions').canThis, + utils = require('./utils'), api = {}, - db; + db, + types = ['application/octet-stream', 'application/json', 'application/zip', 'application/x-zip-compressed'], + extensions = ['.json', '.zip']; api.settings = require('./settings'); -function isValidFile(ext) { - if (ext === '.json') { - return true; +// TODO refactor this out of here +function isJSON(ext) { + return ext === '.json'; +} + +function isZip(ext) { + return ext === '.zip'; +} + +function getJSONFileContents(filepath, ext) { + if (isJSON(ext)) { + // if it's just a JSON file, read it + return Promise.promisify(fs.readFile)(filepath); + } else if (isZip(ext)) { + var tmpdir = path.join(os.tmpdir(), uuid.v4()); + + return Promise.promisify(extract)(filepath, {dir: tmpdir}).then(function () { + return Promise.promisify(glob)('**/*.json', {cwd: tmpdir}).then(function (files) { + if (files[0]) { + // @TODO: handle multiple JSON files + return Promise.promisify(fs.readFile)(path.join(tmpdir, files[0])); + } else { + return Promise.reject(new errors.UnsupportedMediaTypeError( + 'Zip did not include any content to import.' + )); + } + }); + }); } - return false; } + /** * ## DB API Methods * @@ -59,35 +91,37 @@ db = { importContent: function (options) { options = options || {}; var databaseVersion, - type, ext, filepath; - return canThis(options.context).importContent.db().then(function () { - if (!options.importfile || !options.importfile.type || !options.importfile.path) { - return Promise.reject(new errors.NoPermissionError('Please select a file to import.')); - } + // Check if a file was provided + if (!utils.checkFileExists(options, 'importfile')) { + return Promise.reject(new errors.NoPermissionError('Please select a file to import.')); + } - type = options.importfile.type; - ext = path.extname(options.importfile.name).toLowerCase(); - filepath = options.importfile.path; + // Check if the file is valid + if (!utils.checkFileIsValid(options.importfile, types, extensions)) { + return Promise.reject(new errors.UnsupportedMediaTypeError( + 'Please select either a .json or .zip file to import.' + )); + } - return Promise.resolve(isValidFile(ext)).then(function (result) { - if (!result) { - return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a .json file to import.')); - } - }).then(function () { - return api.settings.read( - {key: 'databaseVersion', context: {internal: true}} - ).then(function (response) { - var setting = response.settings[0]; + // TODO refactor this out of here + filepath = options.importfile.path; + ext = path.extname(options.importfile.name).toLowerCase(); - return setting.value; - }); + // Permissions check + return canThis(options.context).importContent.db().then(function () { + return api.settings.read( + {key: 'databaseVersion', context: {internal: true}} + ).then(function (response) { + var setting = response.settings[0]; + + return setting.value; }).then(function (version) { databaseVersion = version; // Read the file contents - return Promise.promisify(fs.readFile)(filepath); + return getJSONFileContents(filepath, ext); }).then(function (fileContents) { var importData; diff --git a/core/server/api/upload.js b/core/server/api/upload.js index cb297a19c9ca..acac1e2ebae9 100644 --- a/core/server/api/upload.js +++ b/core/server/api/upload.js @@ -1,20 +1,12 @@ -var _ = require('lodash'), - config = require('../config'), +var config = require('../config'), Promise = require('bluebird'), - path = require('path'), fs = require('fs-extra'), storage = require('../storage'), errors = require('../errors'), + utils = require('./utils'), upload; -function isImage(type, ext) { - if (_.contains(config.uploads.contentTypes, type) && _.contains(config.uploads.extensions, ext)) { - return true; - } - return false; -} - /** * ## Upload API Methods * @@ -31,25 +23,21 @@ upload = { */ add: function (options) { var store = storage.getStorage(), - type, - ext, filepath; - if (!options.uploadimage || !options.uploadimage.type || !options.uploadimage.path) { + // Check if a file was provided + if (!utils.checkFileExists(options, 'uploadimage')) { return Promise.reject(new errors.NoPermissionError('Please select an image.')); } - type = options.uploadimage.type; - ext = path.extname(options.uploadimage.name).toLowerCase(); + // Check if the file is valid + if (!utils.checkFileIsValid(options.uploadimage, config.uploads.contentTypes, config.uploads.extensions)) { + return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a valid image.')); + } + filepath = options.uploadimage.path; - return Promise.resolve(isImage(type, ext)).then(function (result) { - if (!result) { - return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a valid image.')); - } - }).then(function () { - return store.save(options.uploadimage); - }).then(function (url) { + return store.save(options.uploadimage).then(function (url) { return url; }).finally(function () { // Remove uploaded file from tmp location diff --git a/core/server/api/utils.js b/core/server/api/utils.js index cfc77bc75de6..39414ae2f973 100644 --- a/core/server/api/utils.js +++ b/core/server/api/utils.js @@ -2,6 +2,7 @@ // Shared helpers for working with the API var Promise = require('bluebird'), _ = require('lodash'), + path = require('path'), errors = require('../errors'), utils; @@ -28,6 +29,18 @@ utils = { } } return Promise.resolve(object); + }, + checkFileExists: function (options, filename) { + return options[filename] && options[filename].type && options[filename].path; + }, + checkFileIsValid: function (file, types, extensions) { + var type = file.type, + ext = path.extname(file.name).toLowerCase(); + + if (_.contains(types, type) && _.contains(extensions, ext)) { + return true; + } + return false; } }; diff --git a/package.json b/package.json index b6575d55b56d..be667ba8c0e9 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,9 @@ "downsize": "0.0.5", "express": "4.10.2", "express-hbs": "0.7.11", + "extract-zip": "1.0.3", "fs-extra": "0.12.0", + "glob": "4.3.1", "html-to-text": "1.0.0", "knex": "0.7.3", "lodash": "2.4.1",