diff --git a/.gitignore b/.gitignore index e920c16..1856faf 100644 --- a/.gitignore +++ b/.gitignore @@ -31,3 +31,5 @@ node_modules # Optional REPL history .node_repl_history + +.DS_Store \ No newline at end of file diff --git a/README.md b/README.md index 47f54ca..7366bfa 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,51 @@ # parse-files-utils -Utilities to list and migrate Parse files +Utilities to list and migrate Parse files. -This utility will print in the terminal all the files URL's from the parse server +This utility will do the following: -This can be really useful when you migrate your files and want to move the files from the Parse S3 host to you own. +1. Get all files across all classess in a Parse database. +2. Print file URLs to console OR transfer to S3, GCS, or filesystem. +3. Rename files so that [Parse Server](https://github.com/ParsePlatform/parse-server) no longer detects that they are hosted by Parse. +4. Update MongoDB with new file names. -This utility won't save the files anywhere else. You can save the results to a file or pipe the results to another program: +#### \*WARNING\* +As soon as this script transfers files away from Parse.com hosted files (and renames them in the database) +any clients that use api.parse.com will no longer be able to access the files. +See the section titled "5. Files" in the [Parse Migration Guide](https://parse.com/migration) +and Parse Server [issue #1582](https://github.com/ParsePlatform/parse-server/issues/1582). -## usage +## Installation +1. Clone the repo: `git clone git@github.com:parse-server-modules/parse-files-utils.git` +2. cd into repo: `cd parse-file-utils` +3. Install dependencies: `npm install` + +## Usage + +The quickest way to get started is to run `npm start` and follow the command prompts. + +You can optionally specify a js/json configuration file (see [config.example.js](./config.example.js)). ``` -$ node index.js MY_APP_ID MY_MASTER_KEY +$ npm start config.js ``` -you can optionally specify a server URL +### Available configuration options -``` -$ node index.js MY_APP_ID MY_MASTER_KEY MY_SERVER_URL -``` \ No newline at end of file +* `applicationId`: Parse application id. +* `masterKey`: Parse master key. +* `mongoURL`: MongoDB connection url. +* `serverURL`: The URL for the Parse server (default: http://api.parse.com/1). +* `filesToTransfer`: Which files to transfer. Accepted options: `parseOnly`, `parseServerOnly`, `all`. +* `renameInDatabase` (boolean): Whether or not to rename files in MongoDB. +* `filesAdapter`: A Parse Server file adapter with a function for `createFile(filename, data)` +(ie. [parse-server-fs-adapter](https://github.com/parse-server-modules/parse-server-fs-adapter), +[parse-server-s3-adapter](https://github.com/parse-server-modules/parse-server-s3-adapter), +[parse-server-gcs-adapter](https://github.com/parse-server-modules/parse-server-gcs-adapter)). +* `filesystemPath`: The path/directory to save files to when transfering to filesystem. +* `aws_accessKeyId`: AWS access key id. +* `aws_secretAccessKey`: AWS secret access key. +* `aws_bucket`: S3 bucket name. +* `gcs_projectId`: GCS project id. +* `gcs_keyFilename`: GCS key filename (ie. `credentials.json`). +* `gcs_bucket`: GCS bucket name. +* `asyncLimit`: The number of files to process at the same time (default: 5). \ No newline at end of file diff --git a/config.example.js b/config.example.js new file mode 100644 index 0000000..7ceaeab --- /dev/null +++ b/config.example.js @@ -0,0 +1,40 @@ +var FileAdapter = require('parse-server-fs-adapter'); +var S3Adapter = require('parse-server-s3-adapter'); +var GCSAdapter = require('parse-server-gcs-adapter'); + +module.exports = { + applicationId: "PARSE_APPLICATION_ID", + masterKey: "PARSE_MASTER_KEY", + mongoURL: "mongodb://:@mongourl.com:27017/database_name", + serverURL: "https://api.customparseserver.com/parse", + filesToTransfer: 'parseOnly', + renameInDatabase: false, + + // For filesystem configuration + filesystemPath: './downloaded_files', + + // For S3 configuration + aws_accessKeyId: "ACCESS_KEY_ID", + aws_secretAccessKey: "SECRET_ACCESS_KEY", + aws_bucket: "BUCKET_NAME", + + // For GCS configuration + gcs_projectId: "GCS_PROJECT_ID", + gcs_keyFilename: "credentials.json", + gcs_bucket: "BUCKET_NAME", + + // Or set filesAdapter to a Parse Server file adapter + // filesAdapter: new FileAdapter({ + // filesSubDirectory: './downloaded_files' + // }), + // filesAdapter: new S3Adapter({ + // accessKey: 'ACCESS_KEY_ID', + // secretKey: 'SECRET_ACCESS_KEY', + // bucket: 'BUCKET_NAME' + // }), + // filesAdapter: new GCSAdapter({ + // projectId: "GCS_PROJECT_ID", + // keyFilename: "credentials.json", + // bucket: "BUCKET_NAME", + // }), +}; \ No newline at end of file diff --git a/index.js b/index.js index 43c72ec..37ed6f2 100644 --- a/index.js +++ b/index.js @@ -1,10 +1,16 @@ -var appID = process.argv[2]; -var masterKey = process.argv[3]; -var serverURL = process.argv[4]; +var path = require('path'); +var configFilePath = process.argv[2]; +var config = {}; -if (!appID || !masterKey) { - process.stderr.write('An appId and a masterKey are required\n'); - process.exit(1); +if (configFilePath) { + configFilePath = path.resolve(configFilePath); + + try { + config = require(configFilePath); + } catch(e) { + console.log('Cannot load '+configFilePath); + process.exit(1); + } } -var utils = require('./lib')(appID, masterKey, serverURL); +var utils = require('./lib')(config); diff --git a/lib/index.js b/lib/index.js index 132a3a3..13233ba 100644 --- a/lib/index.js +++ b/lib/index.js @@ -1,5 +1,56 @@ var Parse = require('parse/node'); +var inquirer = require('inquirer'); + var schemas = require('./schemas'); +var transfer = require('./transfer'); +var questions = require('./questions.js'); + +module.exports = initialize; + +function initialize(config) { + questions(config).then(function (answers) { + config = Object.assign(config, answers); + console.log(JSON.stringify(config, null, 2)); + return inquirer.prompt({ + type: 'confirm', + name: 'next', + message: 'About to start the file transfer. Does the above look correct?', + default: true, + }); + }).then(function(answers) { + if (!answers.next) { + console.log('Aborted!'); + process.exit(); + } + Parse.initialize(config.applicationId, null, config.masterKey); + Parse.serverURL = config.serverURL; + return transfer.init(config); + }).then(function() { + return getAllFileObjects(); + }).then(function(objects) { + return transfer.run(objects); + }).then(function() { + console.log('Complete!'); + process.exit(); + }).catch(function(error) { + console.log(error); + process.exit(1); + }); +} + +function getAllFileObjects() { + console.log("Fetching schema..."); + return schemas.get().then(function(res){ + console.log("Fetching all objects with files..."); + var schemasWithFiles = onlyFiles(res); + return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema)); + }).then(function(results) { + var files = results.reduce(function(c, r) { + return c.concat(r); + }, []); + return Promise.resolve(files); + }); +} function onlyFiles(schemas) { return schemas.map(function(schema) { @@ -18,53 +69,43 @@ function onlyFiles(schemas) { function getAllObjects(baseQuery) { var allObjects = []; - var next = function(startIndex) { - baseQuery.skip(startIndex); + var next = function() { + if (allObjects.length) { + baseQuery.greaterThan('createdAt', allObjects[allObjects.length-1].createdAt); + } return baseQuery.find({useMasterKey: true}).then(function(r){ allObjects = allObjects.concat(r); if (r.length == 0) { return Promise.resolve(allObjects); } else { - return next(startIndex+r.length); + return next(); } }); } - return next(0); + return next(); } -function getFilesFromSchema(schema) { +function getObjectsWithFilesFromSchema(schema) { var query = new Parse.Query(schema.className); - query.select(schema.fields); + query.select(schema.fields.concat('createdAt')); + query.ascending('createdAt'); + query.limit(1000); schema.fields.forEach(function(field) { query.exists(field); - }) + }); return getAllObjects(query).then(function(results) { return results.reduce(function(current, result){ - return current.concat(schema.fields.map(function(field){ - return result.get(field).url(); - })) + return current.concat( + schema.fields.map(function(field){ + return { + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: result.get(field).name(), + url: result.get(field).url() + } + }) + ); }, []); }); -} - -module.exports = function(applicationId, masterKey, serverURL) { - Parse.initialize(applicationId, null, masterKey); - Parse.serverURL = serverURL || "https://api.parse.com/1"; - schemas.get().then(function(res){ - var schemasWithFiles = onlyFiles(res); - return Promise.all(schemasWithFiles.map(getFilesFromSchema)); - }).then(function(results) { - var files = results.reduce(function(c, r) { - return c.concat(r); - }, []); - files.forEach(function(file) { - process.stdout.write(file); - process.stdout.write("\n"); - }); - process.exit(0); - }).catch(function(err){ - process.stderr.write(err); - process.stderr.write("\n"); - process.exit(1); - }) -} +} \ No newline at end of file diff --git a/lib/questions.js b/lib/questions.js new file mode 100644 index 0000000..3e92a11 --- /dev/null +++ b/lib/questions.js @@ -0,0 +1,145 @@ +/** + * Uses command line prompts to collect necessary info + */ + +var inquirer = require('inquirer'); +module.exports = questions; + +function questions(config) { + return inquirer.prompt([ + // Collect Parse info + { + type: 'input', + name: 'applicationId', + message: 'The applicationId', + when: !config.applicationId + }, { + type: 'input', + name: 'masterKey', + message: 'The masterKey', + when: !config.masterKey + }, { + type: 'input', + name: 'serverURL', + message: 'The Parse serverURL', + when: !config.serverURL, + default: 'https://api.parse.com/1' + }, { + type: 'list', + name: 'filesToTransfer', + message: 'What files would you like to transfer?', + choices: [ + {name: 'Only parse.com hosted files', value: 'parseOnly'}, + {name: 'Only Parse Server (self hosted server) files', value: 'parseServerOnly'}, + {name: 'All files', value: 'all'} + ], + when: (['parseOnly','parseServerOnly', 'all'].indexOf(config.filesToTransfer) == -1) + }, { + type: 'confirm', + name: 'renameInDatabase', + message: 'Rename Parse hosted files in the database after transfer?', + default: false, + when: function(answers) { + return !config.renameInDatabase && + (answers.filesToTransfer == 'all' || config.filesToTransfer == 'all' || + config.filesToTransfer == 'parseOnly' || answers.filesToTransfer == 'parseOnly'); + } + }, { + type: 'input', + name: 'mongoURL', + message: 'MongoDB URL', + default: 'mongodb://localhost:27017/database', + when: function(answers) { + return (config.renameInDatabase || answers.renameInDatabase) && + !config.mongoURL; + } + }, + + // Where to transfer to + { + type: 'list', + name: 'transferTo', + message: 'Where would you like to transfer files to?', + choices: [ + {name: 'Print List of URLs', value: 'print'}, + {name: 'Local File System', value: 'filesystem'}, + {name: 'AWS S3', value: 's3'}, + {name: 'Google Cloud Storage', value: 'gcs'}, + ], + when: function() { + return (['print','filesystem','s3','gcs'].indexOf(config.transferTo) == -1) && + !config.filesAdapter + } + }, + + // filesystem settings + { + type: 'input', + name: 'filesystemPath', + message: 'Local filesystem path to save files to', + when: function(answers) { + return !config.filesystemPath && + (config.transferTo == 'filesystem' || + answers.transferTo == 'filesystem'); + }, + default: './downloaded_files' + }, + + // S3 settings + { + type: 'input', + name: 'aws_accessKeyId', + message: 'AWS access key id', + when: function(answers) { + return (answers.transferTo == 's3' || config.transferTo == 's3') && + !config.aws_accessKeyId && + !config.aws_profile; + } + }, { + type: 'input', + name: 'aws_secretAccessKey', + message: 'AWS secret access key', + when: function(answers) { + return (answers.transferTo == 's3' || config.transferTo == 's3') && + !config.aws_secretAccessKey && + !config.aws_profile; + } + }, { + type: 'input', + name: 'aws_bucket', + message: 'S3 bucket name', + when: function(answers) { + return (answers.transferTo == 's3' || config.transferTo == 's3') && + !config.aws_bucket; + } + }, + + // GCS settings + { + type: 'input', + name: 'gcs_projectId', + message: 'GCS project id', + when: function(answers) { + return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && + !config.gcs_projectId; + } + }, { + type: 'input', + name: 'gcs_keyFilename', + message: 'GCS key filename', + when: function(answers) { + return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && + !config.gcs_keyFilename; + }, + default: 'credentials.json' + }, { + type: 'input', + name: 'gcs_bucket', + message: 'GCS bucket name', + when: function(answers) { + return (answers.transferTo == 'gcs' || config.transferTo == 'gcs') && + !config.gcs_bucket; + } + }, + ]); +} \ No newline at end of file diff --git a/lib/transfer.js b/lib/transfer.js new file mode 100644 index 0000000..acecaab --- /dev/null +++ b/lib/transfer.js @@ -0,0 +1,196 @@ +var request = require('request'); +var crypto = require('crypto'); +var async = require('async'); +var FilesystemAdapter = require('parse-server-fs-adapter'); +var S3Adapter = require('parse-server-s3-adapter'); +var GCSAdapter = require('parse-server-gcs-adapter'); +var MongoClient = require('mongodb').MongoClient; + +// regex that matches old legacy Parse hosted files +var legacyFilesPrefixRegex = new RegExp("^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}-"); + +var db, config; + +module.exports.init = init; +module.exports.run = run; + +function init(options) { + console.log('Initializing transfer configuration...'); + config = options; + return new Promise(function(resolve, reject) { + if (config.renameInDatabase) { + console.log('Connecting to MongoDB'); + MongoClient.connect(config.mongoURL, function(error, database) { + if (error) { + return reject(error); + } + console.log('Successfully connected to MongoDB'); + db = database; + _setup().then(resolve, reject); + }); + } else { + _setup().then(resolve, reject); + } + }); +} + +function _setup() { + config.adapterName = config.transferTo || config.filesAdapter.constructor.name; + console.log('Initializing '+config.adapterName+' adapter'); + if (config.filesAdapter && config.filesAdapter.createFile) { + return Promise.resolve(); + } else if (config.transferTo == 'print') { + return Promise.resolve(); + } else if (config.transferTo == 'filesystem') { + config.filesAdapter = new FilesystemAdapter({ + filesSubDirectory: config.filesystemPath + }); + } else if (config.transferTo == 's3') { + config.filesAdapter = new S3Adapter({ + accessKey: config.aws_secretAccessKey, + secretKey: config.aws_secretAccessKey, + bucket: config.aws_bucket, + directAccess: true + }); + } else if (config.transferTo == 'gcs') { + config.filesAdapter = new GCSAdapter({ + projectId: config.gcs_projectId, + keyFilename: config.gcs_keyFilename, + bucket: config.gcs_bucket, + directAccess: true + }); + } else { + return Promise.reject('Invalid files adapter'); + } + return Promise.resolve(); +} + +function run(files) { + console.log('Processing '+files.length+' files'); + console.log('Saving files with '+config.adapterName); + return _processFiles(files); +} + +/** + * Handle error from requests + */ +function _requestErrorHandler(error, response) { + if (error) { + return error; + } else if (response.statusCode >= 300) { + console.log('Failed request ('+response.statusCode+') skipping: '+response.request.href); + return true; + } + return false; +} + +/** + * Converts a file into a non Parse file name + * @param {String} fileName + * @return {String} + */ +function _nonParseFileName(fileName) { + if (fileName.indexOf('tfss-') === 0) { + return fileName.replace('tfss-', ''); + } else if (legacyFilesPrefixRegex.test(fileName)) { + var newPrefix = crypto.randomBytes(32/2).toString('hex'); + return newPrefix + fileName.replace(legacyFilesPrefixRegex, ''); + } else { + return fileName; + } +} + +/** + * Loops through n files at a time and calls handler + * @param {Array} files Array of files + * @param {Function} handler handler function for file + * @return {Promise} + */ +function _processFiles(files, handler) { + var asyncLimit = config.asyncLimit || 5; + return new Promise(function(resolve, reject) { + async.eachOfLimit(files, asyncLimit, function(file, index, callback) { + process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); + file.newFileName = _nonParseFileName(file.fileName); + if (_shouldTransferFile(file)) { + _transferFile(file).then(callback, callback); + } else { + callback(); + } + }, function(error) { + if (error) { + return reject('\nError!', error); + } + resolve('\nComplete!'); + }); + }) +} + +/** + * Changes the file name that is saved in MongoDB + * @param {Object} file the file info + */ +function _changeDBFileField(file) { + return new Promise(function(resolve, reject) { + if (file.fileName == file.newFileName || !config.renameInDatabase) { + return resolve(); + } + var update = {$set:{}}; + update.$set[file.fieldName] = file.newFileName; + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result ) { + if (error) { + return reject(error); + } + resolve(); + } + ); + }); +} + +/** + * Determines if a file should be transferred based on configuration + * @param {Object} file the file info + */ +function _shouldTransferFile(file) { + if (config.filesToTransfer == 'all') { + return true; + } else if ( + config.filesToTransfer == 'parseOnly' && + file.fileName != file.newFileName + ) { + return true; + } else if ( + config.filesToTransfer == 'parseServerOnly' && + file.fileName == file.newFileName + ) { + return true; + } + return false; +} + +/** + * Request file from URL and upload with filesAdapter + * @param {Ibject} file the file info object + */ +function _transferFile(file) { + return new Promise(function(resolve, reject) { + if (config.transferTo == 'print') { + console.log(file.url); + // Use process.nextTick to avoid max call stack error + return process.nextTick(resolve); + } + request({ url: file.url, encoding: null }, function(error, response, body) { + if (_requestErrorHandler(error, response)) { + return reject(error); + } + config.filesAdapter.createFile( + file.newFileName, body, response.headers['content-type'] + ).then(function() { + return _changeDBFileField(file); + }).then(resolve, reject); + }); + }); +} \ No newline at end of file diff --git a/package.json b/package.json index bf6c969..9fd53f7 100644 --- a/package.json +++ b/package.json @@ -23,7 +23,12 @@ }, "homepage": "https://github.com/parse-server-modules/parse-files-utils#readme", "dependencies": { + "async": "^2.0.0", + "inquirer": "^1.1.2", "parse": "^1.8.5", + "parse-server-fs-adapter": "^1.0.0", + "parse-server-gcs-adapter": "^1.0.0", + "parse-server-s3-adapter": "^1.0.4", "request": "^2.72.0" } }