Add support for zip import

fixes TryGhost#4607

- moves file checks from db and upload API endpoints to api utils
- adds code to accept and then extract a zip and pull out a JSON file
- zip handling requires a lot of dependencies - this needs a good refactor

core/server/api/db.js

@@ -7,19 +7,51 @@ var dataExport       = require('../data/export'),
     Promise          = require('bluebird'),
     _                = require('lodash'),
     path             = require('path'),
+    os               = require('os'),
+    glob             = require('glob'),
+    uuid             = require('node-uuid'),
+    extract          = require('extract-zip'),
     errors           = require('../../server/errors'),
     canThis          = require('../permissions').canThis,
+    utils            = require('./utils'),
     api              = {},
-    db;
+    db,
+    types = ['application/octet-stream', 'application/json', 'application/zip', 'application/x-zip-compressed'],
+    extensions = ['.json', '.zip'];
 
 api.settings         = require('./settings');
 
-function isValidFile(ext) {
-    if (ext === '.json') {
-        return true;
+// TODO refactor this out of here
+function isJSON(ext) {
+    return ext === '.json';
+}
+
+function isZip(ext) {
+    return ext === '.zip';
+}
+
+function getJSONFileContents(filepath, ext) {
+    if (isJSON(ext)) {
+    // if it's just a JSON file, read it
+        return Promise.promisify(fs.readFile)(filepath);
+    } else if (isZip(ext)) {
+        var tmpdir = path.join(os.tmpdir(), uuid.v4());
+
+        return Promise.promisify(extract)(filepath, {dir: tmpdir}).then(function () {
+            return Promise.promisify(glob)('**/*.json', {cwd: tmpdir}).then(function (files) {
+                if (files[0]) {
+                    // @TODO: handle multiple JSON files
+                    return Promise.promisify(fs.readFile)(path.join(tmpdir, files[0]));
+                } else {
+                    return Promise.reject(new errors.UnsupportedMediaTypeError(
+                        'Zip did not include any content to import.'
+                    ));
+                }
+            });
+        });
     }
-    return false;
 }
+
 /**
  * ## DB API Methods
  *
@@ -59,35 +91,37 @@ db = {
     importContent: function (options) {
         options = options || {};
         var databaseVersion,
-            type,
             ext,
             filepath;
 
-        return canThis(options.context).importContent.db().then(function () {
-            if (!options.importfile || !options.importfile.type || !options.importfile.path) {
-                return Promise.reject(new errors.NoPermissionError('Please select a file to import.'));
-            }
+        // Check if a file was provided
+        if (!utils.checkFileExists(options, 'importfile')) {
+            return Promise.reject(new errors.NoPermissionError('Please select a file to import.'));
+        }
 
-            type = options.importfile.type;
-            ext = path.extname(options.importfile.name).toLowerCase();
-            filepath = options.importfile.path;
+        // Check if the file is valid
+        if (!utils.checkFileIsValid(options.importfile, types, extensions)) {
+            return Promise.reject(new errors.UnsupportedMediaTypeError(
+                'Please select either a .json or .zip file to import.'
+            ));
+        }
 
-            return Promise.resolve(isValidFile(ext)).then(function (result) {
-                if (!result) {
-                    return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a .json file to import.'));
-                }
-            }).then(function () {
-                return api.settings.read(
-                    {key: 'databaseVersion', context: {internal: true}}
-                ).then(function (response) {
-                    var setting = response.settings[0];
+        // TODO refactor this out of here
+        filepath = options.importfile.path;
+        ext = path.extname(options.importfile.name).toLowerCase();
 
-                    return setting.value;
-                });
+        // Permissions check
+        return canThis(options.context).importContent.db().then(function () {
+            return api.settings.read(
+                {key: 'databaseVersion', context: {internal: true}}
+            ).then(function (response) {
+                var setting = response.settings[0];
+
+                return setting.value;
             }).then(function (version) {
                 databaseVersion = version;
                 // Read the file contents
-                return Promise.promisify(fs.readFile)(filepath);
+                return getJSONFileContents(filepath, ext);
             }).then(function (fileContents) {
                 var importData;
 

core/server/api/upload.js

@@ -1,20 +1,12 @@
-var _       = require('lodash'),
-    config  = require('../config'),
+var config  = require('../config'),
     Promise = require('bluebird'),
-    path    = require('path'),
     fs      = require('fs-extra'),
     storage = require('../storage'),
     errors  = require('../errors'),
+    utils   = require('./utils'),
 
     upload;
 
-function isImage(type, ext) {
-    if (_.contains(config.uploads.contentTypes, type) && _.contains(config.uploads.extensions, ext)) {
-        return true;
-    }
-    return false;
-}
-
 /**
  * ## Upload API Methods
  *
@@ -31,25 +23,21 @@ upload = {
      */
     add: function (options) {
         var store = storage.getStorage(),
-            type,
-            ext,
             filepath;
 
-        if (!options.uploadimage || !options.uploadimage.type || !options.uploadimage.path) {
+        // Check if a file was provided
+        if (!utils.checkFileExists(options, 'uploadimage')) {
             return Promise.reject(new errors.NoPermissionError('Please select an image.'));
         }
 
-        type = options.uploadimage.type;
-        ext = path.extname(options.uploadimage.name).toLowerCase();
+        // Check if the file is valid
+        if (!utils.checkFileIsValid(options.uploadimage, config.uploads.contentTypes, config.uploads.extensions)) {
+            return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a valid image.'));
+        }
+
         filepath = options.uploadimage.path;
 
-        return Promise.resolve(isImage(type, ext)).then(function (result) {
-            if (!result) {
-                return Promise.reject(new errors.UnsupportedMediaTypeError('Please select a valid image.'));
-            }
-        }).then(function () {
-            return store.save(options.uploadimage);
-        }).then(function (url) {
+        return store.save(options.uploadimage).then(function (url) {
             return url;
         }).finally(function () {
             // Remove uploaded file from tmp location

core/server/api/utils.js

@@ -2,6 +2,7 @@
 // Shared helpers for working with the API
 var Promise = require('bluebird'),
     _       = require('lodash'),
+    path    = require('path'),
     errors  = require('../errors'),
     utils;
 
@@ -28,6 +29,18 @@ utils = {
             }
         }
         return Promise.resolve(object);
+    },
+    checkFileExists: function (options, filename)  {
+        return options[filename] && options[filename].type && options[filename].path;
+    },
+    checkFileIsValid: function (file, types, extensions) {
+        var type = file.type,
+            ext = path.extname(file.name).toLowerCase();
+
+        if (_.contains(types, type) && _.contains(extensions, ext)) {
+            return true;
+        }
+        return false;
     }
 };
 

package.json

@@ -43,7 +43,9 @@
         "downsize": "0.0.5",
         "express": "4.10.2",
         "express-hbs": "0.7.11",
+        "extract-zip": "1.0.3",
         "fs-extra": "0.12.0",
+        "glob": "4.3.1",
         "html-to-text": "1.0.0",
         "knex": "0.7.3",
         "lodash": "2.4.1",