Add streams and directory building

index.js

@@ -2,9 +2,11 @@
 
 var fs = require('fs')
 var http = require('http')
-var Multipart = require('multipart-stream')
 var qs = require('querystring')
 var multiaddr = require('multiaddr')
+var File = require('vinyl')
+var MultipartDir = require('./multipartdir.js')
+var stream = require('stream')
 
 try {
   var pkg = JSON.parse(fs.readFileSync(__dirname + '/package.json'))
@@ -28,18 +30,21 @@ module.exports = function (host_or_multiaddr, port) {
   if (!port) port = 5001
 
   function send (path, args, opts, files, buffer, cb) {
+    var query, stream, contentType = 'application/json'
+
     if (Array.isArray(path)) path = path.join('/')
 
     opts = opts || {}
+
     if (args && !Array.isArray(args)) args = [args]
     if (args) opts.arg = args
+
     opts['stream-channels'] = true
-    var query = qs.stringify(opts)
+    query = qs.stringify(opts)
 
-    var contentType = 'application/json'
     if (files) {
-      var boundary = randomString()
-      contentType = 'multipart/form-data; boundary=' + boundary
+      stream = getFileStream(files)
+      contentType = 'multipart/form-data; boundary=' + stream.boundary
     }
 
     if (typeof buffer === 'function') {
@@ -58,14 +63,12 @@ module.exports = function (host_or_multiaddr, port) {
       },
       withCredentials: false
     }, function (res) {
+      var data = '', objects = []
       var stream = !!res.headers['x-stream-output']
-      if (stream && !buffer) return cb(null, res)
-
       var chunkedObjects = !!res.headers['x-chunked-output']
-      if (chunkedObjects && buffer) return cb(null, res)
 
-      var data = ''
-      var objects = []
+      if (stream && !buffer) return cb(null, res)
+      if (chunkedObjects && buffer) return cb(null, res)
 
       res.on('data', function (chunk) {
         if (!chunkedObjects) {
@@ -82,9 +85,11 @@ module.exports = function (host_or_multiaddr, port) {
         }
       })
       res.on('end', function () {
+        var parsed
+
         if (!chunkedObjects) {
           try {
-            var parsed = JSON.parse(data)
+            parsed = JSON.parse(data)
             data = parsed
           } catch (e) {}
         } else {
@@ -102,8 +107,7 @@ module.exports = function (host_or_multiaddr, port) {
       })
     })
 
-    if (files) {
-      var stream = getFileStream(files, boundary)
+    if (stream) {
       stream.pipe(req)
     } else {
       req.end()
@@ -112,37 +116,31 @@ module.exports = function (host_or_multiaddr, port) {
     return req
   }
 
-  function getFileStream (files, boundary) {
+  function getFileStream (files) {
     if (!files) return null
-
-    var mp = new Multipart(boundary)
     if (!Array.isArray(files)) files = [files]
 
-    for (var i in files) {
-      var file = files[i]
-
-      if (typeof file === 'string') {
-        // TODO: get actual content type
-        mp.addPart({
-          body: fs.createReadStream(file),
-          headers: {
-            'Content-Type': 'application/octet-stream',
-            'Content-Disposition': 'file; name="file"; filename="' + file + '"'
-          }
-        })
+    var file
 
-      } else if (Buffer.isBuffer(file)) {
-        mp.addPart({
-          body: file,
-          headers: {
-            'Content-Type': 'application/octet-stream',
-            'Content-Disposition': 'file; name="file"; filename=""'
-          }
+    for (var i = 0; i < files.length; i++) {
+      file = files[i]
+      if (file instanceof stream.Stream || Buffer.isBuffer(file)) {
+        file = new File({
+          cwd: '/',
+          base: '/',
+          path: '/',
+          contents: file
         })
       }
+
+      if (!file instanceof File) {
+        return null
+      }
+
+      files[i] = file
     }
 
-    return mp
+    return MultipartDir(files)
   }
 
   function command (name) {
@@ -294,7 +292,3 @@ module.exports = function (host_or_multiaddr, port) {
     }
   }
 }
-
-function randomString () {
-  return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2)
-}

multipartdir.js

@@ -0,0 +1,106 @@
+var Multipart = require('multipart-stream')
+var Path = require('path')
+var stream = require('stream')
+
+module.exports = function MultipartDir (files) {
+  if (files.length === 0) return null
+
+  var root = {
+    files: new Multipart(randomString()),
+    folders: {}
+  }
+
+  for (var i = 0; i < files.length; i++) {
+    addFile(root, files[i])
+  }
+
+  collapse(root)
+
+  return root.files
+}
+
+function resolve (curr, path) {
+  var prev
+  path = path.split(Path.sep)
+
+  if (path[0] === '') path.shift()
+
+  while (curr && path.length) {
+    prev = curr
+    curr = curr.folders[path[0]]
+
+    if (curr) path.shift()
+  }
+
+  return {
+    curr: curr || prev,
+    path: path
+  }
+}
+
+function constructPath (curr, path) {
+  while (path.length) {
+    var folder = path.shift()
+    curr.folders[folder] = {
+      files: new Multipart(randomString()),
+      folders: {}
+    }
+
+    curr = curr.folders[folder]
+  }
+
+  return curr
+}
+
+function addFile (root, file) {
+  var relative = Path.relative(file.base, file.path)
+  var relative_dir = Path.dirname(relative)
+  var folder, info
+
+  if (relative_dir === '.') relative_dir = ''
+
+  info = resolve(root, relative_dir)
+
+  if (info.path.length > 0) {
+    folder = constructPath(info.curr, info.path)
+  } else {
+    folder = info.curr
+  }
+
+  if (file.isDirectory()) return
+
+  folder.files.addPart({
+    body: file.contents,
+    headers: {
+      'Content-Type': 'application/octet-stream',
+      'Content-Disposition': 'file; name="file"; filename="' + relative + '"'
+    }
+  })
+}
+
+function collapse (curr, loc) {
+  var key
+  var folders = Object.keys(curr.folders)
+
+  if (!loc) loc = ''
+  for (var i = 0; i < folders.length; i++) {
+    key = folders[i]
+    collapse(curr.folders[key], loc + key + '/')
+
+    if (!(curr.folders[key].files instanceof stream.Stream)) {
+      return
+    }
+
+    curr.files.addPart({
+      body: curr.folders[key].files,
+      headers: {
+        'Content-Type': 'multipart/form-data; boundary=' + curr.folders[key].files.boundary,
+        'Content-Disposition': 'file; name="folder"; filename="' + loc + key + '"'
+      }
+    })
+  }
+}
+
+function randomString () {
+  return Math.random().toString(36).slice(2) + Math.random().toString(36).slice(2)
+}

package.json

@@ -5,7 +5,8 @@
   "main": "index.js",
   "dependencies": {
     "multiaddr": "^0.1.2",
-    "multipart-stream": "^1.0.0"
+    "multipart-stream": "^1.0.0",
+    "vinyl": "^0.5.0"
   },
   "repository": {
     "type": "git",

test/test.js

@@ -2,6 +2,8 @@ var ipfsd = require('ipfsd-ctl')
 var ipfsApi = require('../index.js')
 var assert = require('assert')
 var fs = require('fs')
+var path = require('path')
+var File = require('vinyl')
 
 /*global describe, before, it*/
 
@@ -25,7 +27,13 @@ describe('ipfs node api', function () {
   var fileName = __dirname + '/testfile.txt'
 
   before(function (done) {
-    ipfs.add(fileName, function (err, res) {
+    var file = new File({
+      cwd: path.dirname(fileName),
+      base: path.dirname(fileName),
+      path: fileName,
+     contents: fs.createReadStream(fileName)
+    })
+    ipfs.add(file, function (err, res) {
       if (err) throw err
       fileAdded = res
       done()
@@ -34,7 +42,7 @@ describe('ipfs node api', function () {
 
   it('add file', function () {
     assert.equal(fileAdded[0].Hash, 'Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP')
-    assert.equal(fileAdded[0].Name, fileName)
+    assert.equal(fileAdded[0].Name, path.basename(fileName))
   })
 
   var bufferAdded