feat: add support for File DOM API to files-regular (ipfs#986)

* feat: add file dom api support to files api

* feat: add support for File DOM API to files-regular

* chore: fix package declaration cause npm is dumb

* chore: fix lint

* chore: add ipfs-utils

* fix: change the requires to ipfs-utils

* chore: increase max bundle size

Author: hugomrdias

.aegir.js

@@ -5,7 +5,7 @@ const createServer = require('ipfsd-ctl').createServer
 const server = createServer()
 
 module.exports = {
-  bundlesize: { maxSize: '231kB' },
+  bundlesize: { maxSize: '232kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main']

examples/upload-file-via-browser/.eslintrc

@@ -2,3 +2,4 @@ node_modules
 npm-debug.log
 .DS_Store
 dist
+yarn.lock

examples/upload-file-via-browser/.gitignore

@@ -3,7 +3,7 @@
   "version": "1.0.0",
   "description": "Upload file to IPFS via browser using js-ipfs-http-client with Webpack",
   "scripts": {
-    "start": "node server.js"
+    "start": "webpack-dev-server"
   },
   "author": "Harlan T Wood <code@harlantwood.net>",
   "contributors": [
@@ -12,12 +12,24 @@
   "license": "MIT",
   "devDependencies": {
     "@babel/core": "^7.4.3",
-    "ipfs-http-client": "../../",
-    "pull-file-reader": "~1.0.2",
-    "react": "~16.8.6",
-    "react-dom": "~16.8.6",
-    "react-hot-loader": "~4.8.4",
-    "webpack": "~4.31.0",
+    "@babel/preset-env": "^7.3.1",
+    "@babel/preset-react": "^7.0.0",
+    "eslint": "^5.16.0",
+    "eslint-plugin-react": "^7.11.1",
+    "react": "~16.6.3",
+    "react-dom": "~16.6.3",
+    "webpack": "~4.30.0",
     "webpack-dev-server": "~3.3.1"
+  },
+  "eslintConfig" : {
+    "extends": "standard",
+    "rules": {
+      "react/jsx-uses-react": 2,
+      "react/jsx-uses-vars": 2,
+      "react/react-in-jsx-scope": 2
+    },
+    "plugins": [
+      "react"
+    ]
   }
 }

examples/upload-file-via-browser/package.json

@@ -1,17 +1,14 @@
 'use strict'
 const React = require('react')
-const ipfsClient = require('ipfs-http-client')
-
-// create a stream from a file, which enables uploads of big files without allocating memory twice
-const fileReaderPullStream = require('pull-file-reader')
+const ipfsClient = require('../../../src')
 
 class App extends React.Component {
   constructor () {
     super()
     this.state = {
       added_file_hash: null
     }
-    this.ipfs = ipfsClient('localhost', '5001')
+    this.ipfs = ipfsClient('localhost', '58041')
 
     // bind methods
     this.captureFile = this.captureFile.bind(this)
@@ -22,38 +19,36 @@ class App extends React.Component {
   captureFile (event) {
     event.stopPropagation()
     event.preventDefault()
-    const file = event.target.files[0]
     if (document.getElementById('keepFilename').checked) {
-      this.saveToIpfsWithFilename(file)
+      this.saveToIpfsWithFilename(event.target.files)
     } else {
-      this.saveToIpfs(file)
+      this.saveToIpfs(event.target.files)
     }
   }
 
   // Example #1
   // Add file to IPFS and return a CID
-  saveToIpfs (file) {
+  saveToIpfs (files) {
     let ipfsId
-    const fileStream = fileReaderPullStream(file)
-    this.ipfs.add(fileStream, { progress: (prog) => console.log(`received: ${prog}`) })
+    this.ipfs.add([...files], { progress: (prog) => console.log(`received: ${prog}`) })
       .then((response) => {
         console.log(response)
         ipfsId = response[0].hash
         console.log(ipfsId)
-        this.setState({added_file_hash: ipfsId})
+        this.setState({ added_file_hash: ipfsId })
       }).catch((err) => {
         console.error(err)
       })
   }
 
   // Example #2
   // Add file to IPFS and wrap it in a directory to keep the original filename
-  saveToIpfsWithFilename (file) {
+  saveToIpfsWithFilename (files) {
+    const file = [...files][0]
     let ipfsId
-    const fileStream = fileReaderPullStream(file)
     const fileDetails = {
       path: file.name,
-      content: fileStream
+      content: file
     }
     const options = {
       wrapWithDirectory: true,
@@ -65,7 +60,7 @@ class App extends React.Component {
         // CID of wrapping directory is returned last
         ipfsId = response[response.length - 1].hash
         console.log(ipfsId)
-        this.setState({added_file_hash: ipfsId})
+        this.setState({ added_file_hash: ipfsId })
       }).catch((err) => {
         console.error(err)
       })

examples/upload-file-via-browser/server.js

@@ -1,24 +1,18 @@
 'use strict'
 
 const path = require('path')
-const webpack = require('webpack')
 
 module.exports = {
   mode: 'development',
   devtool: 'eval',
   entry: [
-    'webpack-dev-server/client?http://localhost:3000',
-    'webpack/hot/only-dev-server',
     './src/index'
   ],
   output: {
     path: path.join(__dirname, 'dist'),
     filename: 'bundle.js',
     publicPath: '/static/'
   },
-  plugins: [
-    new webpack.HotModuleReplacementPlugin()
-  ],
   module: {
     rules: [
       {
@@ -28,8 +22,7 @@ module.exports = {
           {
             loader: 'babel-loader',
             options: {
-              presets: ['@babel/preset-env', '@babel/preset-react'],
-              plugins: ['react-hot-loader/babel']
+              presets: ['@babel/preset-env', '@babel/preset-react']
             }
           }
         ]

examples/upload-file-via-browser/src/App.js

@@ -47,8 +47,10 @@
     "is-stream": "^2.0.0",
     "iso-stream-http": "~0.1.2",
     "iso-url": "~0.4.6",
+    "ipfs-utils": "~0.0.3",
     "just-kebab-case": "^1.1.0",
     "just-map-keys": "^1.1.0",
+    "kind-of": "^6.0.2",
     "lru-cache": "^5.1.1",
     "multiaddr": "^6.0.6",
     "multibase": "~0.6.0",
@@ -83,9 +85,8 @@
     "chai": "^4.2.0",
     "cross-env": "^5.2.0",
     "dirty-chai": "^2.0.1",
-    "eslint-plugin-react": "^7.11.1",
     "go-ipfs-dep": "0.4.19",
-    "interface-ipfs-core": "~0.101.1",
+    "interface-ipfs-core": "~0.102.0",
     "ipfsd-ctl": "~0.42.0",
     "nock": "^10.0.2",
     "stream-equal": "^1.1.1"

examples/upload-file-via-browser/webpack.config.js

@@ -3,10 +3,10 @@
 const promisify = require('promisify-es6')
 const ConcatStream = require('concat-stream')
 const once = require('once')
-const isStream = require('is-stream')
-const isSource = require('is-pull-stream').isSource
+const { isSource } = require('is-pull-stream')
 const FileResultStreamConverter = require('../utils/file-result-stream-converter')
 const SendFilesStream = require('../utils/send-files-stream')
+const validateAddInput = require('ipfs-utils/src/files/add-input-validation')
 
 module.exports = (send) => {
   const createAddStream = SendFilesStream(send, 'add')
@@ -16,31 +16,17 @@ module.exports = (send) => {
       _callback = options
       options = null
     }
-
     const callback = once(_callback)
 
     if (!options) {
       options = {}
     }
     options.converter = FileResultStreamConverter
 
-    // Buffer, pull stream or Node.js stream
-    const isBufferOrStream = obj => Buffer.isBuffer(obj) || isStream.readable(obj) || isSource(obj)
-    // An object like { content?, path? }, where content isBufferOrStream and path isString
-    const isContentObject = obj => {
-      if (typeof obj !== 'object') return false
-      // path is optional if content is present
-      if (obj.content) return isBufferOrStream(obj.content)
-      // path must be a non-empty string if no content
-      return Boolean(obj.path) && typeof obj.path === 'string'
-    }
-    // An input atom: a buffer, stream or content object
-    const isInput = obj => isBufferOrStream(obj) || isContentObject(obj)
-    // All is ok if data isInput or data is an array of isInput
-    const ok = isInput(_files) || (Array.isArray(_files) && _files.every(isInput))
-
-    if (!ok) {
-      return callback(new Error('invalid input: expected buffer, readable stream, pull stream, object or array of objects'))
+    try {
+      validateAddInput(_files)
+    } catch (err) {
+      return callback(err)
     }
 
     const files = [].concat(_files)

package.json

@@ -2,8 +2,6 @@
 
 const Transform = require('readable-stream').Transform
 const isNode = require('detect-node')
-const isSource = require('is-pull-stream').isSource
-const toStream = require('pull-to-stream')
 
 const PADDING = '--'
 const NEW_LINE = '\r\n'
@@ -77,12 +75,7 @@ class Multipart extends Transform {
       return callback() // early
     }
 
-    if (isSource(content)) {
-      content = toStream.readable(content)
-    }
-
     // From now on we assume content is a stream
-
     content.once('error', this.emit.bind(this, 'error'))
 
     content.once('end', () => {

src/files-regular/add.js

@@ -2,6 +2,13 @@
 
 const isNode = require('detect-node')
 const flatmap = require('flatmap')
+const { Readable } = require('readable-stream')
+const kindOf = require('kind-of')
+const { isSource } = require('is-pull-stream')
+const isStream = require('is-stream')
+const pullToStream = require('pull-to-stream')
+const { supportsFileReader } = require('ipfs-utils/src/supports')
+const streamFromFileReader = require('ipfs-utils/src/streams/stream-from-filereader')
 
 function loadPaths (opts, file) {
   const path = require('path')
@@ -73,10 +80,36 @@ function loadPaths (opts, file) {
   }
 }
 
+function contentToStream (content) {
+  if (supportsFileReader && kindOf(content) === 'file') {
+    return streamFromFileReader(content)
+  }
+
+  if (kindOf(content) === 'buffer') {
+    return new Readable({
+      read () {
+        this.push(content)
+        this.push(null)
+      }
+    })
+  }
+
+  if (isSource(content)) {
+    return pullToStream.readable(content)
+  }
+
+  if (isStream.readable(content)) {
+    return content
+  }
+
+  throw new Error(`Input not supported. Expected Buffer|ReadableStream|PullStream|File got ${kindOf(content)}. Check the documentation for more info https://github.com/ipfs/interface-js-ipfs-core/blob/master/SPEC/FILES.md#add`)
+}
+
 function prepareFile (file, opts) {
   let files = [].concat(file)
 
   return flatmap(files, (file) => {
+    // add from fs with file path
     if (typeof file === 'string') {
       if (!isNode) {
         throw new Error('Can only add file paths in node')
@@ -85,20 +118,34 @@ function prepareFile (file, opts) {
       return loadPaths(opts, file)
     }
 
-    if (file.path && !file.content) {
-      file.dir = true
-      return file
-    }
+    // add with object syntax { path : <string> , content: <Buffer|ReadableStream|PullStream|File }
+    if (kindOf(file) === 'object') {
+      // treat as an empty directory when path is a string and content undefined
+      if (file.path && kindOf(file.path) === 'string' && !file.content) {
+        file.dir = true
+        return file
+      }
 
-    if (file.content || file.dir) {
-      return file
+      // just return when directory
+      if (file.dir) {
+        return file
+      }
+
+      if (file.content) {
+        return {
+          path: file.path || '',
+          symlink: false,
+          dir: false,
+          content: contentToStream(file.content)
+        }
+      }
     }
 
     return {
       path: '',
       symlink: false,
       dir: false,
-      content: file
+      content: contentToStream(file)
     }
   })
 }