refactor: fetch add (#1087)

License: MIT
Signed-off-by: Alan Shaw <alan.shaw@protocol.ai>

Author: Alan Shaw

.aegir.js

@@ -5,7 +5,7 @@ const createServer = require('ipfsd-ctl').createServer
 const server = createServer()
 
 module.exports = {
-  bundlesize: { maxSize: '237kB' },
+  bundlesize: { maxSize: '240kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main']

package.json

@@ -18,7 +18,9 @@
     "glob": false,
     "fs": false,
     "stream": "readable-stream",
-    "ky-universal": "ky/umd"
+    "ky-universal": "ky/umd",
+    "./src/add/form-data.js": "./src/add/form-data.browser.js",
+    "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js"
   },
   "repository": "github:ipfs/js-ipfs-http-client",
   "scripts": {
@@ -40,6 +42,8 @@
   "dependencies": {
     "abort-controller": "^3.0.0",
     "async": "^2.6.1",
+    "async-iterator-all": "^1.0.0",
+    "async-iterator-to-pull-stream": "^1.3.0",
     "bignumber.js": "^9.0.0",
     "bl": "^3.0.0",
     "bs58": "^4.0.1",
@@ -52,9 +56,10 @@
     "err-code": "^2.0.0",
     "explain-error": "^1.0.4",
     "flatmap": "0.0.3",
+    "fs-extra": "^8.1.0",
     "glob": "^7.1.3",
     "ipfs-block": "~0.8.1",
-    "ipfs-utils": "~0.0.3",
+    "ipfs-utils": "^0.1.0",
     "ipld-dag-cbor": "~0.15.0",
     "ipld-dag-pb": "~0.17.3",
     "ipld-raw": "^4.0.0",
@@ -63,6 +68,8 @@
     "is-stream": "^2.0.0",
     "iso-stream-http": "~0.1.2",
     "iso-url": "~0.4.6",
+    "it-glob": "0.0.4",
+    "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "just-kebab-case": "^1.1.0",
     "just-map-keys": "^1.1.0",
@@ -82,6 +89,7 @@
     "promisify-es6": "^1.0.3",
     "pull-defer": "~0.2.3",
     "pull-stream": "^3.6.9",
+    "pull-stream-to-async-iterator": "^1.0.2",
     "pull-to-stream": "~0.1.1",
     "pump": "^3.0.0",
     "qs": "^6.5.2",
@@ -98,7 +106,7 @@
     "cross-env": "^5.2.0",
     "dirty-chai": "^2.0.1",
     "go-ipfs-dep": "^0.4.22",
-    "interface-ipfs-core": "^0.111.0",
+    "interface-ipfs-core": "^0.112.0",
     "ipfsd-ctl": "~0.45.0",
     "nock": "^10.0.2",
     "stream-equal": "^1.1.1"

src/add-from-fs/index.browser.js

@@ -0,0 +1,3 @@
+'use strict'
+
+module.exports = () => () => { throw new Error('unavailable in the browser') }

src/add-from-fs/index.js

@@ -0,0 +1,9 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const globSource = require('ipfs-utils/src/files/glob-source')
+
+module.exports = configure(({ ky }) => {
+  const add = require('../add')({ ky })
+  return (path, options) => add(globSource(path, options), options)
+})

src/add-from-url.js

@@ -0,0 +1,22 @@
+'use strict'
+
+const kyDefault = require('ky-universal').default
+const configure = require('./lib/configure')
+const toIterable = require('./lib/stream-to-iterable')
+
+module.exports = configure(({ ky }) => {
+  const add = require('./add')({ ky })
+
+  return (url, options) => (async function * () {
+    options = options || {}
+
+    const { body } = await kyDefault.get(url)
+
+    const input = {
+      path: decodeURIComponent(new URL(url).pathname.split('/').pop() || ''),
+      content: toIterable(body)
+    }
+
+    yield * add(input, options)
+  })()
+})

src/add/form-data.browser.js

@@ -0,0 +1,31 @@
+'use strict'
+/* eslint-env browser */
+
+const { Buffer } = require('buffer')
+const normaliseInput = require('ipfs-utils/src/files/normalise-input')
+
+exports.toFormData = async input => {
+  const files = normaliseInput(input)
+  const formData = new FormData()
+  let i = 0
+
+  for await (const file of files) {
+    if (file.content) {
+      // In the browser there's _currently_ no streaming upload, buffer up our
+      // async iterator chunks and append a big Blob :(
+      // One day, this will be browser streams
+      const bufs = []
+      for await (const chunk of file.content) {
+        bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk)
+      }
+
+      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), file.path)
+    } else {
+      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), file.path)
+    }
+
+    i++
+  }
+
+  return formData
+}

src/add/form-data.js

@@ -0,0 +1,42 @@
+'use strict'
+
+const FormData = require('form-data')
+const { Buffer } = require('buffer')
+const toStream = require('it-to-stream')
+const normaliseInput = require('ipfs-utils/src/files/normalise-input')
+
+exports.toFormData = async input => {
+  const files = normaliseInput(input)
+  const formData = new FormData()
+  let i = 0
+
+  for await (const file of files) {
+    if (file.content) {
+      // In Node.js, FormData can be passed a stream so no need to buffer
+      formData.append(
+        `file-${i}`,
+        // FIXME: add a `path` property to the stream so `form-data` doesn't set
+        // a Content-Length header that is only the sum of the size of the
+        // header/footer when knownLength option (below) is null.
+        Object.assign(
+          toStream.readable(file.content),
+          { path: file.path || `file-${i}` }
+        ),
+        {
+          filepath: encodeURIComponent(file.path),
+          contentType: 'application/octet-stream',
+          knownLength: file.content.length // Send Content-Length header if known
+        }
+      )
+    } else {
+      formData.append(`dir-${i}`, Buffer.alloc(0), {
+        filepath: encodeURIComponent(file.path),
+        contentType: 'application/x-directory'
+      })
+    }
+
+    i++
+  }
+
+  return formData
+}

src/add/index.js

@@ -0,0 +1,54 @@
+'use strict'
+
+const ndjson = require('iterable-ndjson')
+const configure = require('../lib/configure')
+const toIterable = require('../lib/stream-to-iterable')
+const { toFormData } = require('./form-data')
+const toCamel = require('../lib/object-to-camel')
+
+module.exports = configure(({ ky }) => {
+  return (input, options) => (async function * () {
+    options = options || {}
+
+    const searchParams = new URLSearchParams(options.searchParams)
+
+    searchParams.set('stream-channels', true)
+    if (options.chunker) searchParams.set('chunker', options.chunker)
+    if (options.cidVersion) searchParams.set('cid-version', options.cidVersion)
+    if (options.cidBase) searchParams.set('cid-base', options.cidBase)
+    if (options.enableShardingExperiment != null) searchParams.set('enable-sharding-experiment', options.enableShardingExperiment)
+    if (options.hashAlg) searchParams.set('hash', options.hashAlg)
+    if (options.onlyHash != null) searchParams.set('only-hash', options.onlyHash)
+    if (options.pin != null) searchParams.set('pin', options.pin)
+    if (options.progress) searchParams.set('progress', true)
+    if (options.quiet != null) searchParams.set('quiet', options.quiet)
+    if (options.quieter != null) searchParams.set('quieter', options.quieter)
+    if (options.rawLeaves != null) searchParams.set('raw-leaves', options.rawLeaves)
+    if (options.shardSplitThreshold) searchParams.set('shard-split-threshold', options.shardSplitThreshold)
+    if (options.silent) searchParams.set('silent', options.silent)
+    if (options.trickle != null) searchParams.set('trickle', options.trickle)
+    if (options.wrapWithDirectory != null) searchParams.set('wrap-with-directory', options.wrapWithDirectory)
+
+    const res = await ky.post('add', {
+      timeout: options.timeout,
+      signal: options.signal,
+      headers: options.headers,
+      searchParams,
+      body: await toFormData(input)
+    })
+
+    for await (let file of ndjson(toIterable(res.body))) {
+      file = toCamel(file)
+      // console.log(file)
+      if (options.progress && file.bytes) {
+        options.progress(file.bytes)
+      } else {
+        yield toCoreInterface(file)
+      }
+    }
+  })()
+})
+
+function toCoreInterface ({ name, hash, size }) {
+  return { path: name, hash, size: parseInt(size) }
+}

src/files-mfs/write.js

@@ -3,7 +3,6 @@
 const promisify = require('promisify-es6')
 const concatStream = require('concat-stream')
 const once = require('once')
-const FileResultStreamConverter = require('../utils/file-result-stream-converter')
 const SendFilesStream = require('../utils/send-files-stream')
 
 module.exports = (send) => {
@@ -29,8 +28,7 @@ module.exports = (send) => {
 
     const options = {
       args: pathDst,
-      qs: opts,
-      converter: FileResultStreamConverter
+      qs: opts
     }
 
     const stream = sendFilesStream({ qs: options })