From 68c08edd8e763309ed935bfbfbd42341c8d44730 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 3 Jun 2015 17:06:29 -0700 Subject: [PATCH 1/5] Move to streams Start of moving to supporting streams, also now support folder uploading. --- index.js | 74 +++++++++++++++----------------- multipartdir.js | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- test/test.js | 12 +++++- 4 files changed, 158 insertions(+), 43 deletions(-) create mode 100644 multipartdir.js diff --git a/index.js b/index.js index e0176280b..6cd7a035c 100644 --- a/index.js +++ b/index.js @@ -2,9 +2,11 @@ var fs = require('fs') var http = require('http') -var Multipart = require('multipart-stream') var qs = require('querystring') var multiaddr = require('multiaddr') +var File = require('vinyl') +var MultipartDir = require('./multipartdir.js') +var stream = require('stream') try { var pkg = JSON.parse(fs.readFileSync(__dirname + '/package.json')) @@ -28,18 +30,21 @@ module.exports = function (host_or_multiaddr, port) { if (!port) port = 5001 function send (path, args, opts, files, buffer, cb) { + var query, stream, contentType = 'application/json' + if (Array.isArray(path)) path = path.join('/') opts = opts || {} + if (args && !Array.isArray(args)) args = [args] if (args) opts.arg = args + opts['stream-channels'] = true - var query = qs.stringify(opts) + query = qs.stringify(opts) - var contentType = 'application/json' if (files) { - var boundary = randomString() - contentType = 'multipart/form-data; boundary=' + boundary + stream = getFileStream(files) + contentType = 'multipart/form-data; boundary=' + stream.boundary } if (typeof buffer === 'function') { @@ -58,14 +63,12 @@ module.exports = function (host_or_multiaddr, port) { }, withCredentials: false }, function (res) { + var data = '', objects = [] var stream = !!res.headers['x-stream-output'] - if (stream && !buffer) return cb(null, res) - var chunkedObjects = !!res.headers['x-chunked-output'] - if (chunkedObjects && buffer) return cb(null, res) - var data = '' - var objects = [] + if (stream && !buffer) return cb(null, res) + if (chunkedObjects && buffer) return cb(null, res) res.on('data', function (chunk) { if (!chunkedObjects) { @@ -82,9 +85,11 @@ module.exports = function (host_or_multiaddr, port) { } }) res.on('end', function () { + var parsed + if (!chunkedObjects) { try { - var parsed = JSON.parse(data) + parsed = JSON.parse(data) data = parsed } catch (e) {} } else { @@ -102,8 +107,7 @@ module.exports = function (host_or_multiaddr, port) { }) }) - if (files) { - var stream = getFileStream(files, boundary) + if (stream) { stream.pipe(req) } else { req.end() @@ -112,37 +116,31 @@ module.exports = function (host_or_multiaddr, port) { return req } - function getFileStream (files, boundary) { + function getFileStream (files) { if (!files) return null - - var mp = new Multipart(boundary) if (!Array.isArray(files)) files = [files] - for (var i in files) { - var file = files[i] - - if (typeof file === 'string') { - // TODO: get actual content type - mp.addPart({ - body: fs.createReadStream(file), - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': 'file; name="file"; filename="' + file + '"' - } - }) + var file - } else if (Buffer.isBuffer(file)) { - mp.addPart({ - body: file, - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': 'file; name="file"; filename=""' - } + for (var i = 0; i < files.length; i++) { + file = files[i] + if (file instanceof stream.Stream || Buffer.isBuffer(file)) { + file = new File({ + cwd: '/', + base: '/', + path: '/', + contents: file }) } + + if (!file instanceof File) { + return null + } + + files[i] = file } - return mp + return MultipartDir(files) } function command (name) { @@ -294,7 +292,3 @@ module.exports = function (host_or_multiaddr, port) { } } } - -function randomString () { - return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2) -} diff --git a/multipartdir.js b/multipartdir.js new file mode 100644 index 000000000..b4a257fb3 --- /dev/null +++ b/multipartdir.js @@ -0,0 +1,112 @@ +var Multipart = require('multipart-stream') +var Path = require('path') +var stream = require('stream') + +module.exports = function MultipartDir (files) { + if (files.length === 0) return null + + var base = files[0].base + var root = { + files: new Multipart(randomString()), + folders: {} + } + + for (var i = 0; i < files.length; i++) { + var file = files[i] + + // All files must share the same base + if (file.base !== base) return null + + addFile(root, file) + } + + collapse(root) + + return root.files +} + +function resolve (curr, path) { + var prev + path = path.split(Path.sep) + + if (path[0] === '') path.shift() + + while (curr && path.length) { + prev = curr + curr = curr.folders[path[0]] + + if (curr) path.shift() + } + + return { + curr: curr || prev, + path: path + } +} + +function constructPath (curr, path) { + while (path.length) { + var folder = path.shift() + curr.folders[folder] = { + files: new Multipart(randomString()), + folders: {} + } + + curr = curr.folders[folder] + } + + return curr +} + +function addFile (root, file) { + var relative = Path.relative(file.cwd, file.path) + var relative_dir = Path.dirname(relative) + var folder, info + + if (relative_dir === '.') relative_dir = '' + + info = resolve(root, relative_dir) + + if (info.path.length > 0) { + folder = constructPath(info.curr, info.path) + } else { + folder = info.curr + } + + if (file.isDirectory()) return + + folder.files.addPart({ + body: file.contents, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': 'file; name="file"; filename="' + relative + '"' + } + }) +} + +function collapse (curr, loc) { + var key + var folders = Object.keys(curr.folders) + + if (!loc) loc = '' + for (var i = 0; i < folders.length; i++) { + key = folders[i] + collapse(curr.folders[key], loc + key + '/') + + if (!(curr.folders[key].files instanceof stream.Stream)) { + return + } + + curr.files.addPart({ + body: curr.folders[key].files, + headers: { + 'Content-Type': 'multipart/form-data; boundary=' + curr.folders[key].files.boundary, + 'Content-Disposition': 'file; name="folder"; filename="' + loc + key + '"' + } + }) + } +} + +function randomString () { + return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2) +} diff --git a/package.json b/package.json index 439b66baf..de5fa3e3b 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "index.js", "dependencies": { "multiaddr": "^0.1.2", - "multipart-stream": "^1.0.0" + "multipart-stream": "^1.0.0", + "vinyl": "^0.4.6" }, "repository": { "type": "git", diff --git a/test/test.js b/test/test.js index d44fa77df..86a947a69 100644 --- a/test/test.js +++ b/test/test.js @@ -2,6 +2,8 @@ var ipfsd = require('ipfsd-ctl') var ipfsApi = require('../index.js') var assert = require('assert') var fs = require('fs') +var path = require('path') +var File = require('vinyl') /*global describe, before, it*/ @@ -25,7 +27,13 @@ describe('ipfs node api', function () { var fileName = __dirname + '/testfile.txt' before(function (done) { - ipfs.add(fileName, function (err, res) { + var file = new File({ + cwd: path.dirname(fileName), + base: path.dirname(fileName), + path: fileName, + contents: fs.createReadStream(fileName) + }) + ipfs.add(file, function (err, res) { if (err) throw err fileAdded = res done() @@ -34,7 +42,7 @@ describe('ipfs node api', function () { it('add file', function () { assert.equal(fileAdded[0].Hash, 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - assert.equal(fileAdded[0].Name, fileName) + assert.equal(fileAdded[0].Name, path.basename(fileName)) }) var bufferAdded From f92078fad0f174e32882acfc160c6d709d6fe14b Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 3 Jun 2015 17:09:35 -0700 Subject: [PATCH 2/5] Update to vinyl for new dirname/basename/extname features --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index de5fa3e3b..e8ccbe01b 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "dependencies": { "multiaddr": "^0.1.2", "multipart-stream": "^1.0.0", - "vinyl": "^0.4.6" + "vinyl": "^0.5.0" }, "repository": { "type": "git", From 0cb1d20c424162efe6885a721455f1a72fb19b94 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 3 Jun 2015 17:06:29 -0700 Subject: [PATCH 3/5] Move to streams Start of moving to supporting streams, also now support folder uploading. --- index.js | 74 +++++++++++++++----------------- multipartdir.js | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 3 +- test/test.js | 12 +++++- 4 files changed, 158 insertions(+), 43 deletions(-) create mode 100644 multipartdir.js diff --git a/index.js b/index.js index e0176280b..6cd7a035c 100644 --- a/index.js +++ b/index.js @@ -2,9 +2,11 @@ var fs = require('fs') var http = require('http') -var Multipart = require('multipart-stream') var qs = require('querystring') var multiaddr = require('multiaddr') +var File = require('vinyl') +var MultipartDir = require('./multipartdir.js') +var stream = require('stream') try { var pkg = JSON.parse(fs.readFileSync(__dirname + '/package.json')) @@ -28,18 +30,21 @@ module.exports = function (host_or_multiaddr, port) { if (!port) port = 5001 function send (path, args, opts, files, buffer, cb) { + var query, stream, contentType = 'application/json' + if (Array.isArray(path)) path = path.join('/') opts = opts || {} + if (args && !Array.isArray(args)) args = [args] if (args) opts.arg = args + opts['stream-channels'] = true - var query = qs.stringify(opts) + query = qs.stringify(opts) - var contentType = 'application/json' if (files) { - var boundary = randomString() - contentType = 'multipart/form-data; boundary=' + boundary + stream = getFileStream(files) + contentType = 'multipart/form-data; boundary=' + stream.boundary } if (typeof buffer === 'function') { @@ -58,14 +63,12 @@ module.exports = function (host_or_multiaddr, port) { }, withCredentials: false }, function (res) { + var data = '', objects = [] var stream = !!res.headers['x-stream-output'] - if (stream && !buffer) return cb(null, res) - var chunkedObjects = !!res.headers['x-chunked-output'] - if (chunkedObjects && buffer) return cb(null, res) - var data = '' - var objects = [] + if (stream && !buffer) return cb(null, res) + if (chunkedObjects && buffer) return cb(null, res) res.on('data', function (chunk) { if (!chunkedObjects) { @@ -82,9 +85,11 @@ module.exports = function (host_or_multiaddr, port) { } }) res.on('end', function () { + var parsed + if (!chunkedObjects) { try { - var parsed = JSON.parse(data) + parsed = JSON.parse(data) data = parsed } catch (e) {} } else { @@ -102,8 +107,7 @@ module.exports = function (host_or_multiaddr, port) { }) }) - if (files) { - var stream = getFileStream(files, boundary) + if (stream) { stream.pipe(req) } else { req.end() @@ -112,37 +116,31 @@ module.exports = function (host_or_multiaddr, port) { return req } - function getFileStream (files, boundary) { + function getFileStream (files) { if (!files) return null - - var mp = new Multipart(boundary) if (!Array.isArray(files)) files = [files] - for (var i in files) { - var file = files[i] - - if (typeof file === 'string') { - // TODO: get actual content type - mp.addPart({ - body: fs.createReadStream(file), - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': 'file; name="file"; filename="' + file + '"' - } - }) + var file - } else if (Buffer.isBuffer(file)) { - mp.addPart({ - body: file, - headers: { - 'Content-Type': 'application/octet-stream', - 'Content-Disposition': 'file; name="file"; filename=""' - } + for (var i = 0; i < files.length; i++) { + file = files[i] + if (file instanceof stream.Stream || Buffer.isBuffer(file)) { + file = new File({ + cwd: '/', + base: '/', + path: '/', + contents: file }) } + + if (!file instanceof File) { + return null + } + + files[i] = file } - return mp + return MultipartDir(files) } function command (name) { @@ -294,7 +292,3 @@ module.exports = function (host_or_multiaddr, port) { } } } - -function randomString () { - return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2) -} diff --git a/multipartdir.js b/multipartdir.js new file mode 100644 index 000000000..b4a257fb3 --- /dev/null +++ b/multipartdir.js @@ -0,0 +1,112 @@ +var Multipart = require('multipart-stream') +var Path = require('path') +var stream = require('stream') + +module.exports = function MultipartDir (files) { + if (files.length === 0) return null + + var base = files[0].base + var root = { + files: new Multipart(randomString()), + folders: {} + } + + for (var i = 0; i < files.length; i++) { + var file = files[i] + + // All files must share the same base + if (file.base !== base) return null + + addFile(root, file) + } + + collapse(root) + + return root.files +} + +function resolve (curr, path) { + var prev + path = path.split(Path.sep) + + if (path[0] === '') path.shift() + + while (curr && path.length) { + prev = curr + curr = curr.folders[path[0]] + + if (curr) path.shift() + } + + return { + curr: curr || prev, + path: path + } +} + +function constructPath (curr, path) { + while (path.length) { + var folder = path.shift() + curr.folders[folder] = { + files: new Multipart(randomString()), + folders: {} + } + + curr = curr.folders[folder] + } + + return curr +} + +function addFile (root, file) { + var relative = Path.relative(file.cwd, file.path) + var relative_dir = Path.dirname(relative) + var folder, info + + if (relative_dir === '.') relative_dir = '' + + info = resolve(root, relative_dir) + + if (info.path.length > 0) { + folder = constructPath(info.curr, info.path) + } else { + folder = info.curr + } + + if (file.isDirectory()) return + + folder.files.addPart({ + body: file.contents, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Disposition': 'file; name="file"; filename="' + relative + '"' + } + }) +} + +function collapse (curr, loc) { + var key + var folders = Object.keys(curr.folders) + + if (!loc) loc = '' + for (var i = 0; i < folders.length; i++) { + key = folders[i] + collapse(curr.folders[key], loc + key + '/') + + if (!(curr.folders[key].files instanceof stream.Stream)) { + return + } + + curr.files.addPart({ + body: curr.folders[key].files, + headers: { + 'Content-Type': 'multipart/form-data; boundary=' + curr.folders[key].files.boundary, + 'Content-Disposition': 'file; name="folder"; filename="' + loc + key + '"' + } + }) + } +} + +function randomString () { + return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2) +} diff --git a/package.json b/package.json index c0a4810b6..ac932698a 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,8 @@ "main": "index.js", "dependencies": { "multiaddr": "^0.1.2", - "multipart-stream": "^1.0.0" + "multipart-stream": "^1.0.0", + "vinyl": "^0.4.6" }, "repository": { "type": "git", diff --git a/test/test.js b/test/test.js index d44fa77df..86a947a69 100644 --- a/test/test.js +++ b/test/test.js @@ -2,6 +2,8 @@ var ipfsd = require('ipfsd-ctl') var ipfsApi = require('../index.js') var assert = require('assert') var fs = require('fs') +var path = require('path') +var File = require('vinyl') /*global describe, before, it*/ @@ -25,7 +27,13 @@ describe('ipfs node api', function () { var fileName = __dirname + '/testfile.txt' before(function (done) { - ipfs.add(fileName, function (err, res) { + var file = new File({ + cwd: path.dirname(fileName), + base: path.dirname(fileName), + path: fileName, + contents: fs.createReadStream(fileName) + }) + ipfs.add(file, function (err, res) { if (err) throw err fileAdded = res done() @@ -34,7 +42,7 @@ describe('ipfs node api', function () { it('add file', function () { assert.equal(fileAdded[0].Hash, 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') - assert.equal(fileAdded[0].Name, fileName) + assert.equal(fileAdded[0].Name, path.basename(fileName)) }) var bufferAdded From 49841ebde2ff2e2006cc4cabf57344236cf6d4bc Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 3 Jun 2015 17:09:35 -0700 Subject: [PATCH 4/5] Update to vinyl for new dirname/basename/extname features --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index ac932698a..0d25314ab 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,7 @@ "dependencies": { "multiaddr": "^0.1.2", "multipart-stream": "^1.0.0", - "vinyl": "^0.4.6" + "vinyl": "^0.5.0" }, "repository": { "type": "git", From b9a0e96a60f08057c89d56cde22e69881f7894e2 Mon Sep 17 00:00:00 2001 From: Travis Person Date: Wed, 3 Jun 2015 20:27:19 -0700 Subject: [PATCH 5/5] Removed base restriction --- multipartdir.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/multipartdir.js b/multipartdir.js index b4a257fb3..aa852a54f 100644 --- a/multipartdir.js +++ b/multipartdir.js @@ -5,19 +5,13 @@ var stream = require('stream') module.exports = function MultipartDir (files) { if (files.length === 0) return null - var base = files[0].base var root = { files: new Multipart(randomString()), folders: {} } for (var i = 0; i < files.length; i++) { - var file = files[i] - - // All files must share the same base - if (file.base !== base) return null - - addFile(root, file) + addFile(root, files[i]) } collapse(root) @@ -59,7 +53,7 @@ function constructPath (curr, path) { } function addFile (root, file) { - var relative = Path.relative(file.cwd, file.path) + var relative = Path.relative(file.base, file.path) var relative_dir = Path.dirname(relative) var folder, info