Merge pull request #25 from nginnever/master

Data Exporting

Author: daviddias

src/index.js

@@ -9,6 +9,9 @@ const FixedSizeChunker = require('./chunker-fixed-size')
 const through2 = require('through2')
 const UnixFS = require('ipfs-unixfs')
 const async = require('async')
+const events = require('events')
+const Readable = require('stream').Readable
+const pathj = require('path')
 
 exports = module.exports
 
@@ -206,7 +209,6 @@ exports.import = (target, dagService, options, callback) => {
             leafSize: raw.fileSize(),
             Name: ''
           })
-
           cb()
         })
       }, (cb) => {
@@ -249,6 +251,114 @@ exports.import = (target, dagService, options, callback) => {
   // function streamImporter (stream, callback) {}
 }
 
-exports.export = function () {
-  // export into files by hash
+exports.export = function (hash, dagService, options, callback) {
+  if (typeof options === 'function') {
+    callback = options
+    options = {}
+  }
+
+  const ee = new events.EventEmitter()
+  dagService.get(hash, (err, fetchedNode) => {
+    if (err) {
+      if (callback) {
+        return callback(err)
+      }
+      return
+    }
+    const data = UnixFS.unmarshal(fetchedNode.data)
+    const type = data.type
+    if (type === 'directory') {
+      dirExporter(fetchedNode, hash, callback)
+    }
+    if (type === 'file') {
+      fileExporter(fetchedNode, hash, false, callback)
+    }
+  })
+  return ee
+
+  function fileExporter (node, name, dir, callback) {
+    if (typeof dir === 'function') { callback = dir; dir = {} }
+    var rs = new Readable()
+    if (node.links.length === 0) {
+      const unmarshaledData = UnixFS.unmarshal(node.data)
+      ee.emit('file', { stream: rs, path: name, dir: dir })
+      rs.push(unmarshaledData.data)
+      rs.push(null)
+      if (callback) {
+        callback()
+      }
+      return
+    } else {
+      ee.emit('file', { stream: rs, path: name, dir: dir })
+      var init = false
+      rs._read = () => {
+        if (init) {
+          return
+        }
+        init = true
+        async.forEachSeries(node.links, (link, callback) => {
+          dagService.get(link.hash, (err, res) => {
+            if (err) {
+              callback(err)
+            }
+            var unmarshaledData = UnixFS.unmarshal(res.data)
+            rs.push(unmarshaledData.data)
+            callback()
+          })
+        }, (err) => {
+          if (err) {
+            if (callback) {
+              return callback(err)
+            }
+            return
+          }
+          rs.push(null)
+          if (callback) {
+            callback()
+          }
+          return
+        })
+      }
+    }
+  }
+
+  function dirExporter (node, name, callback) {
+    var rs = new Readable()
+    if (node.links.length === 0) {
+      rs.push(node.data)
+      rs.push(null)
+      ee.emit('file', {stream: rs, path: name})
+      if (callback) {
+        callback()
+      }
+      return
+    } else {
+      async.forEachSeries(node.links, (link, callback) => {
+        dagService.get(link.hash, (err, res) => {
+          if (err) {
+            callback(err)
+          }
+          var unmarshaledData = UnixFS.unmarshal(res.data)
+          if (unmarshaledData.type === 'file') {
+            return (fileExporter(res, pathj.join(name, link.name), callback))
+          }
+          if (unmarshaledData.type === 'directory') {
+            return (dirExporter(res, pathj.join(name, link.name), callback))
+          }
+          callback()
+        })
+      }, (err) => {
+        if (err) {
+          if (callback) {
+            return callback(err)
+          }
+          return
+        }
+        if (callback) {
+          callback()
+        }
+        return
+      })
+    }
+  }
 }

test/buffer-test.js

@@ -82,5 +82,60 @@ module.exports = function (repo) {
         })
       })
     })
+
+    it('export a file with no links', (done) => {
+      const hash = 'QmQmZQxSKQppbsWfVzBvg59Cn3DKtsNVQ94bjAxg2h3Lb8'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        ds.get(hash, (err, fetchedNode) => {
+          expect(err).to.not.exist
+          const unmarsh = UnixFS.unmarshal(fetchedNode.data)
+          expect(unmarsh.data).to.deep.equal(data.stream._readableState.buffer[0])
+          done()
+        })
+      })
+    })
+
+    it('export a small file with links', (done) => {
+      const hash = 'QmW7BDxEbGqxxSYVtn3peNPQgdDXbWkoQ6J1EFYAEuQV3Q'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        expect(data.stream).to.exist
+        done()
+      })
+    })
+
+    it('export a large file > 5mb', (done) => {
+      const hash = 'QmRQgufjp9vLE8XK2LGKZSsPCFCF6e4iynCQtNB5X2HBKE'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      testExport.on('file', (data) => {
+        expect(data.stream).to.exist
+        done()
+      })
+    })
+
+    it('export a directory', (done) => {
+      const hash = 'QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN'
+      const bs = new BlockService(repo)
+      const ds = new DAGService(bs)
+      const testExport = importer.export(hash, ds)
+      var fs = []
+      testExport.on('file', (data) => {
+        fs.push(data)
+      })
+      setTimeout(() => {
+        expect(fs[0].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/200Bytes.txt')
+        expect(fs[1].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/dir-another')
+        expect(fs[2].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/200Bytes.txt')
+        expect(fs[3].path).to.equal('QmWChcSFMNcFkfeJtNd8Yru1rE6PhtCRfewi1tMwjkwKjN/level-1/level-2')
+        done()
+      }, 1000)
+    })
   })
 }

test/repo-example/blocks/12200046/12200046ea932273bc69bf22ccd57a2837eee93896de64e70099724ae333c67eebae.data

@@ -0,0 +1,4 @@
+5
+" $��G�,�A�4{���x�Z/.����D`�200Bytes.txt�
/
+" Y��9_)a���˹2�R�m�Ŗke�9��level-2
+

test/repo-example/blocks/1220016d/1220016d22c790b4e50495df66eff6361276442eff53b5457cf8de5c2c6726325ca7.data

@@ -0,0 +1,5 @@
+5
+" $��G�,�A�4{���x�Z/.����D`�200Bytes.txt�
3
+" Y��9_)a���˹2�R�m�Ŗke�9��dir-another0
+" Ty�5;_9Yf�q��F�Lhyl���/��level-1�
+