fix: fix electron renderer tests and a couple more bugs (#1105)

temp fix for electron renderer until ky-universal is fixed
Blob can accept Buffer directly
Buffer.buffer triggered a super weird bug in electron renderer
enable a bunch of tests in the browser
xvfb-run makes electron go crazy
detect-node needs to be removed everywhere and ipfs-utils/src/env.js should be used
If you mix promises and callbacks inside, call the callback with setImmediate

Author: hugomrdias

.aegir.js

@@ -1,9 +1,13 @@
 'use strict'
 
+const promisify = require('promisify-es6')
 const createServer = require('ipfsd-ctl').createServer
-
+const EchoServer = require('interface-ipfs-core/src/utils/echo-http-server')
 const server = createServer()
+const echoServer = EchoServer.createServer()
 
+const echoServerStart = promisify(echoServer.start)
+const echoServerStop = promisify(echoServer.stop)
 module.exports = {
   bundlesize: { maxSize: '245kB' },
   webpack: {
@@ -22,9 +26,23 @@ module.exports = {
     singleRun: true
   },
   hooks: {
+    node: {
+      pre: () => echoServerStart(),
+      post: () => echoServerStop()
+    },
     browser: {
-      pre: () => server.start(),
-      post: () => server.stop()
+      pre: () => {
+        return Promise.all([
+          server.start(),
+          echoServerStart()
+        ])
+      },
+      post: () => {
+        return Promise.all([
+          server.stop(),
+          echoServerStop()
+        ])
+      }
     }
   }
 }

.travis.yml

@@ -1,13 +1,14 @@
 language: node_js
 cache: npm
+
 stages:
   - check
   - test
   - cov
 
 node_js:
-  - '10'
   - '12'
+  - '10'
 
 os:
   - linux
@@ -18,11 +19,6 @@ script: npx nyc -s npm run test:node -- --bail
 after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov
 
 jobs:
-  allow_failures:
-    - name: electron-renderer
-
-  fast_finish: true
-
   include:
     - stage: check
       script:
@@ -44,13 +40,15 @@ jobs:
 
     - stage: test
       name: electron-main
+      os: osx
       script:
-        - xvfb-run npx aegir test -t electron-main -- --bail
+        - npx aegir test -t electron-main --bail
 
     - stage: test
       name: electron-renderer
+      os: osx
       script:
-        - xvfb-run npx aegir test -t electron-renderer -- --bail
+        - npx aegir test -t electron-renderer --bail
 
 notifications:
   email: false

package.json

@@ -51,6 +51,7 @@
     "cids": "~0.7.1",
     "concat-stream": "github:hugomrdias/concat-stream#feat/smaller",
     "debug": "^4.1.0",
+    "delay": "^4.3.0",
     "detect-node": "^2.0.4",
     "end-of-stream": "^1.4.1",
     "err-code": "^2.0.0",
@@ -77,6 +78,7 @@
     "ky": "^0.13.0",
     "ky-universal": "^0.3.0",
     "lru-cache": "^5.1.1",
+    "merge-options": "^1.0.1",
     "multiaddr": "^7.1.0",
     "multibase": "~0.6.0",
     "multicodec": "~0.5.1",
@@ -99,15 +101,15 @@
     "through2": "^3.0.1"
   },
   "devDependencies": {
-    "aegir": "^20.0.0",
+    "aegir": "^20.2.0",
     "browser-process-platform": "~0.1.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
-    "cross-env": "^5.2.0",
+    "cross-env": "^5.2.1",
     "dirty-chai": "^2.0.1",
     "go-ipfs-dep": "^0.4.22",
-    "interface-ipfs-core": "^0.112.0",
-    "ipfsd-ctl": "~0.45.0",
+    "interface-ipfs-core": "~0.114.0",
+    "ipfsd-ctl": "^0.46.2",
     "nock": "^11.3.2",
     "stream-equal": "^1.1.1"
   },

src/add/form-data.browser.js

@@ -1,7 +1,6 @@
 'use strict'
 /* eslint-env browser */
 
-const { Buffer } = require('buffer')
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
 
 exports.toFormData = async input => {
@@ -16,12 +15,12 @@ exports.toFormData = async input => {
       // One day, this will be browser streams
       const bufs = []
       for await (const chunk of file.content) {
-        bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk)
+        bufs.push(chunk)
       }
 
-      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), file.path)
+      formData.append(`file-${i}`, new Blob(bufs, { type: 'application/octet-stream' }), encodeURIComponent(file.path))
     } else {
-      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), file.path)
+      formData.append(`dir-${i}`, new Blob([], { type: 'application/x-directory' }), encodeURIComponent(file.path))
     }
 
     i++

src/add/form-data.js

@@ -4,6 +4,7 @@ const FormData = require('form-data')
 const { Buffer } = require('buffer')
 const toStream = require('it-to-stream')
 const normaliseInput = require('ipfs-utils/src/files/normalise-input')
+const { isElectronRenderer } = require('ipfs-utils/src/env')
 
 exports.toFormData = async input => {
   const files = normaliseInput(input)
@@ -40,3 +41,10 @@ exports.toFormData = async input => {
 
   return formData
 }
+
+// TODO remove this when upstream fix for ky-universal is merged
+// https://github.com/sindresorhus/ky-universal/issues/9
+// also this should only be necessary when nodeIntegration is false in electron renderer
+if (isElectronRenderer) {
+  exports.toFormData = require('./form-data.browser').toFormData
+}

src/dht/findpeer.js

@@ -32,7 +32,7 @@ module.exports = (send) => {
       // 2 = FinalPeer
       // https://github.com/libp2p/go-libp2p-core/blob/6e566d10f4a5447317a66d64c7459954b969bdab/routing/query.go#L18
       if (!res || res.Type !== 2) {
-        const errMsg = `key was not found (type 4)`
+        const errMsg = 'key was not found (type 4)'
         return callback(errcode(new Error(errMsg), 'ERR_KEY_TYPE_4_NOT_FOUND'))
       }
 

src/id.js

@@ -11,6 +11,7 @@ module.exports = (arg) => {
       callback = opts
       opts = undefined
     }
+
     send({
       path: 'id',
       args: opts

test/get.spec.js

@@ -9,7 +9,6 @@ const chaiAsPromised = require('chai-as-promised')
 const expect = chai.expect
 chai.use(dirtyChai)
 chai.use(chaiAsPromised)
-const isNode = require('detect-node')
 const loadFixture = require('aegir/fixtures')
 
 const ipfsClient = require('../src')
@@ -75,26 +74,19 @@ describe('.get (specific go-ipfs features)', function () {
   })
 
   it('add path containing "+"s (for testing get)', async () => {
-    if (!isNode) {
-      return
-    }
-
     const filename = 'ti,c64x+mega++mod-pic.txt'
     const subdir = 'tmp/c++files'
     const expectedCid = 'QmPkmARcqjo5fqK1V1o8cFsuaXxWYsnwCNLJUYS4KeZyff'
+    const path = `${subdir}/${filename}`
     const files = await ipfs.add([{
-      path: subdir + '/' + filename,
-      content: Buffer.from(subdir + '/' + filename, 'utf-8')
+      path,
+      content: Buffer.from(path)
     }])
 
     expect(files[2].hash).to.equal(expectedCid)
   })
 
   it('get path containing "+"s', async () => {
-    if (!isNode) {
-      return
-    }
-
     const cid = 'QmPkmARcqjo5fqK1V1o8cFsuaXxWYsnwCNLJUYS4KeZyff'
     let count = 0
     const files = await ipfs.get(cid)

test/interface.spec.js

@@ -4,7 +4,6 @@
 const tests = require('interface-ipfs-core')
 const isNode = require('detect-node')
 const CommonFactory = require('./utils/interface-common-factory')
-const ipfsClient = require('../src')
 const isWindows = process.platform && process.platform === 'win32'
 
 describe('interface-ipfs-core tests', () => {
@@ -102,40 +101,11 @@ describe('interface-ipfs-core tests', () => {
 
   tests.filesRegular(defaultCommonFactory, {
     skip: [
-      // .add
-      isNode ? null : {
-        name: 'should add a nested directory as array of tupples',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      isNode ? null : {
-        name: 'should add a nested directory as array of tupples with progress',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      // .addPullStream
-      isNode ? null : {
-        name: 'should add pull stream of valid files and dirs',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      // .addReadableStream
-      isNode ? null : {
-        name: 'should add readable stream of valid files and dirs',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
       // .addFromFs
       isNode ? null : {
         name: 'addFromFs',
         reason: 'Not designed to run in the browser'
       },
-      // .addFromURL
-      isNode ? null : {
-        name: 'addFromURL',
-        reason: 'Not designed to run in the browser'
-      },
-      // TODO: remove when interface-ipfs-core updated
-      isNode ? null : {
-        name: 'addFromUrl',
-        reason: 'Not designed to run in the browser'
-      },
       // .catPullStream
       {
         name: 'should export a chunk of a file',
@@ -148,26 +118,6 @@ describe('interface-ipfs-core tests', () => {
       {
         name: 'should export a chunk of a file in a Readable Stream',
         reason: 'TODO not implemented in go-ipfs yet'
-      },
-      // .get
-      isNode ? null : {
-        name: 'should get a directory',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      // .ls
-      isNode ? null : {
-        name: 'should ls with a base58 encoded CID',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      // .lsPullStream
-      isNode ? null : {
-        name: 'should pull stream ls with a base58 encoded CID',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
-      },
-      // .lsReadableStream
-      isNode ? null : {
-        name: 'should readable stream ls with a base58 encoded CID',
-        reason: 'FIXME https://github.com/ipfs/js-ipfs-http-client/issues/339'
       }
     ]
   })
@@ -277,31 +227,5 @@ describe('interface-ipfs-core tests', () => {
 
   tests.stats(defaultCommonFactory)
 
-  tests.swarm(CommonFactory.create({
-    createSetup ({ ipfsFactory, nodes }) {
-      return callback => {
-        callback(null, {
-          spawnNode (repoPath, config, cb) {
-            if (typeof repoPath === 'function') {
-              cb = repoPath
-              repoPath = undefined
-            }
-
-            if (typeof config === 'function') {
-              cb = config
-              config = undefined
-            }
-
-            const spawnOptions = { repoPath, config, initOptions: { bits: 1024, profile: 'test' } }
-
-            ipfsFactory.spawn(spawnOptions)
-              .then(ipfsd => {
-                nodes.push(ipfsd)
-                cb(null, ipfsClient(ipfsd.apiAddr))
-              }, cb)
-          }
-        })
-      }
-    }
-  }))
+  tests.swarm(CommonFactory.createAsync())
 })

test/utils/interface-common-factory.js

@@ -4,11 +4,12 @@
 const each = require('async/each')
 const IPFSFactory = require('ipfsd-ctl')
 const ipfsClient = require('../../src')
+const merge = require('merge-options')
 
 function createFactory (options) {
   options = options || {}
 
-  options.factoryOptions = options.factoryOptions || {}
+  options.factoryOptions = options.factoryOptions || { IpfsClient: ipfsClient }
   options.spawnOptions = options.spawnOptions || { initOptions: { bits: 1024, profile: 'test' } }
 
   const ipfsFactory = IPFSFactory.create(options.factoryOptions)
@@ -26,8 +27,11 @@ function createFactory (options) {
             ipfsFactory.spawn(options.spawnOptions)
               .then((ipfsd) => {
                 nodes.push(ipfsd)
-                cb(null, ipfsClient(ipfsd.apiAddr))
-              }, cb)
+                setImmediate(() => cb(null, ipfsd.api))
+              })
+              .catch(err => {
+                setImmediate(() => cb(err))
+              })
           }
         })
       }
@@ -36,11 +40,50 @@ function createFactory (options) {
     if (options.createTeardown) {
       teardown = options.createTeardown({ ipfsFactory, nodes }, options)
     } else {
-      teardown = callback => each(nodes, (node, cb) => node.stop().then(cb, cb), callback)
+      teardown = callback => each(nodes, (node, cb) => {
+        node
+          .stop()
+          .then(() => setImmediate(() => cb()))
+          .catch(err => setImmediate(() => cb(err)))
+      }, callback)
     }
 
     return { setup, teardown }
   }
 }
 
-exports.create = createFactory
+function createAsync (createFactoryOptions, createSpawnOptions) {
+  return () => {
+    const nodes = []
+    const setup = async (factoryOptions = {}, spawnOptions) => {
+      const ipfsFactory = IPFSFactory.create(merge(
+        { IpfsClient: ipfsClient },
+        factoryOptions,
+        createFactoryOptions
+      ))
+      const node = await ipfsFactory.spawn(merge(
+        { initOptions: { profile: 'test' } },
+        spawnOptions,
+        createSpawnOptions
+      ))
+      nodes.push(node)
+
+      const id = await node.api.id()
+      node.api.peerId = id
+
+      return node.api
+    }
+
+    const teardown = () => {
+      return Promise.all(nodes.map(n => n.stop()))
+    }
+    return {
+      setup,
+      teardown
+    }
+  }
+}
+module.exports = {
+  createAsync,
+  create: createFactory
+}