feat: add HTTP Gateway to the js-ipfs daemon

Author: harshjv

gulpfile.js

@@ -4,7 +4,7 @@ const gulp = require('gulp')
 const parallel = require('async/parallel')
 const series = require('async/series')
 const createTempRepo = require('./test/utils/create-repo-nodejs.js')
-const HTTPAPI = require('./src/http-api')
+const HTTPAPI = require('./src/http')
 const leftPad = require('left-pad')
 
 let nodes = []

package.json

@@ -27,6 +27,7 @@
     "test:unit:node": "gulp test:node",
     "test:unit:node:core": "TEST=core npm run test:unit:node",
     "test:unit:node:http": "TEST=http npm run test:unit:node",
+    "test:unit:node:gateway": "TEST=gateway npm run test:unit:node",
     "test:unit:node:cli": "TEST=cli npm run test:unit:node",
     "test:unit:browser": "gulp test:browser",
     "test:interop": "npm run test:interop:node",
@@ -92,8 +93,10 @@
     "async": "^2.5.0",
     "bl": "^1.2.1",
     "boom": "^5.2.0",
-    "cids": "~0.5.1",
     "debug": "^3.0.1",
+    "cids": "^0.5.1",
+    "file-type": "^6.1.0",
+    "filesize": "^3.5.10",
     "fsm-event": "^2.1.0",
     "glob": "^7.1.2",
     "hapi": "^16.5.2",
@@ -126,6 +129,7 @@
     "lodash.sortby": "^4.7.0",
     "lodash.values": "^4.3.0",
     "mafmt": "^3.0.0",
+    "mime-types": "^2.1.16",
     "mkdirp": "~0.5.1",
     "multiaddr": "^3.0.0",
     "multihashes": "~0.4.9",

src/cli/commands/daemon.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const HttpAPI = require('../../http-api')
+const HttpAPI = require('../../http')
 const utils = require('../utils')
 const print = utils.print
 

src/http-api/resources/bitswap.js src/http/api/resources/bitswap.js

@@ -0,0 +1,114 @@
+'use strict'
+
+const mh = require('multihashes')
+const promisify = require('promisify-es6')
+const eachOfSeries = require('async/eachOfSeries')
+const debug = require('debug')
+const log = debug('jsipfs:http-gateway:resolver')
+log.error = debug('jsipfs:http-gateway:resolver:error')
+
+const html = require('./utils/html')
+const PathUtil = require('./utils/path')
+
+const INDEX_HTML_FILES = [ 'index.html', 'index.htm', 'index.shtml' ]
+
+const resolveDirectory = promisify((ipfs, path, multihash, callback) => {
+  if (!callback) {
+    callback = noop
+  }
+
+  mh.validate(mh.fromB58String(multihash))
+
+  ipfs
+   .object
+   .get(multihash, { enc: 'base58' })
+   .then((DAGNode) => {
+     const links = DAGNode.links
+     const indexFiles = links.filter((link) => INDEX_HTML_FILES.indexOf(link.name) !== -1)
+
+     // found index file in links
+     if (indexFiles.length > 0) {
+       return callback(null, indexFiles)
+     }
+
+     return callback(null, html.build(path, links))
+   })
+})
+
+const noop = function () {}
+
+const resolveMultihash = promisify((ipfs, path, callback) => {
+  if (!callback) {
+    callback = noop
+  }
+
+  const parts = PathUtil.splitPath(path)
+  const partsLength = parts.length
+
+  let currentMultihash = parts[0]
+
+  eachOfSeries(parts, (multihash, currentIndex, next) => {
+    // throws error when invalid multihash is passed
+    mh.validate(mh.fromB58String(currentMultihash))
+    log('currentMultihash: ', currentMultihash)
+    log('currentIndex: ', currentIndex, '/', partsLength)
+
+    ipfs
+     .object
+     .get(currentMultihash, { enc: 'base58' })
+     .then((DAGNode) => {
+      //  log('DAGNode: ', DAGNode)
+       if (currentIndex === partsLength - 1) {
+          // leaf node
+         log('leaf node: ', currentMultihash)
+        //  log('DAGNode: ', DAGNode.links)
+
+         if (DAGNode.links &&
+             DAGNode.links.length > 0 &&
+             DAGNode.links[0].name.length > 0) {
+           //  this is a directory.
+           let isDirErr = new Error('This dag node is a directory')
+           // add currentMultihash as a fileName so it can be used by resolveDirectory
+           isDirErr.fileName = currentMultihash
+           return next(isDirErr)
+         }
+
+         next()
+       } else {
+          // find multihash of requested named-file
+          // in current DAGNode's links
+         let multihashOfNextFile
+         const nextFileName = parts[currentIndex + 1]
+         const links = DAGNode.links
+
+         for (let link of links) {
+           if (link.name === nextFileName) {
+              // found multihash of requested named-file
+             multihashOfNextFile = mh.toB58String(link.multihash)
+             log('found multihash: ', multihashOfNextFile)
+             break
+           }
+         }
+
+         if (!multihashOfNextFile) {
+           log.error(`no link named "${nextFileName}" under ${currentMultihash}`)
+           throw new Error(`no link named "${nextFileName}" under ${currentMultihash}`)
+         }
+
+         currentMultihash = multihashOfNextFile
+         next()
+       }
+     })
+  }, (err) => {
+    if (err) {
+      log.error(err)
+      return callback(err)
+    }
+    callback(null, {multihash: currentMultihash})
+  })
+})
+
+module.exports = {
+  resolveDirectory,
+  resolveMultihash
+}

src/http-api/resources/block.js src/http/api/resources/block.js

@@ -0,0 +1,140 @@
+'use strict'
+
+const debug = require('debug')
+const log = debug('jsipfs:http-gateway')
+log.error = debug('jsipfs:http-gateway:error')
+const pull = require('pull-stream')
+const toPull = require('stream-to-pull-stream')
+const fileType = require('file-type')
+const mime = require('mime-types')
+const GatewayResolver = require('../resolver')
+const PathUtils = require('../utils/path')
+const Stream = require('stream')
+
+module.exports = {
+  checkHash: (request, reply) => {
+    if (!request.params.hash) {
+      return reply({
+        Message: 'Path Resolve error: path must contain at least one component',
+        Code: 0
+      }).code(400).takeover()
+    }
+
+    return reply({
+      ref: `/ipfs/${request.params.hash}`
+    })
+  },
+  handler: (request, reply) => {
+    const ref = request.pre.args.ref
+    const ipfs = request.server.app.ipfs
+
+    return GatewayResolver
+           .resolveMultihash(ipfs, ref)
+           .then((data) => {
+             ipfs
+             .files
+             .cat(data.multihash)
+             .then((stream) => {
+               if (ref.endsWith('/')) {
+                  // remove trailing slash for files
+                 return reply
+                        .redirect(PathUtils.removeTrailingSlash(ref))
+                        .permanent(true)
+               } else {
+                 if (!stream._read) {
+                   stream._read = () => {}
+                   stream._readableState = {}
+                 }
+                 //  response.continue()
+                 let filetypeChecked = false
+                 let stream2 = new Stream.PassThrough({highWaterMark: 1})
+                 let response = reply(stream2).hold()
+
+                 pull(
+                   toPull.source(stream),
+                   pull.drain((chunk) => {
+                     // Check file type.  do this once.
+                     if (chunk.length > 0 && !filetypeChecked) {
+                       log('got first chunk')
+                       let fileSignature = fileType(chunk)
+                       log('file type: ', fileSignature)
+
+                       filetypeChecked = true
+                       const mimeType = mime.lookup((fileSignature) ? fileSignature.ext : null)
+                       log('ref ', ref)
+                       log('mime-type ', mimeType)
+
+                       if (mimeType) {
+                         log('writing mimeType')
+
+                         response
+                           .header('Content-Type', mime.contentType(mimeType))
+                           .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput')
+                           .header('Access-Control-Allow-Methods', 'GET')
+                           .header('Access-Control-Allow-Origin', '*')
+                           .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput')
+                           .send()
+                       } else {
+                         response
+                          .header('Access-Control-Allow-Headers', 'X-Stream-Output, X-Chunked-Ouput')
+                          .header('Access-Control-Allow-Methods', 'GET')
+                          .header('Access-Control-Allow-Origin', '*')
+                          .header('Access-Control-Expose-Headers', 'X-Stream-Output, X-Chunked-Ouput')
+                          .send()
+                       }
+                     }
+
+                     stream2.write(chunk)
+                   }, (err) => {
+                     if (err) throw err
+                     log('stream ended.')
+                     stream2.end()
+                   })
+                 )
+               }
+             })
+             .catch((err) => {
+               if (err) {
+                 log.error(err)
+                 return reply(err.toString()).code(500)
+               }
+             })
+           }).catch((err) => {
+             log('err: ', err.toString(), ' fileName: ', err.fileName)
+
+             const errorToString = err.toString()
+             if (errorToString === 'Error: This dag node is a directory') {
+               return GatewayResolver
+                      .resolveDirectory(ipfs, ref, err.fileName)
+                      .then((data) => {
+                        if (typeof data === 'string') {
+                          // no index file found
+                          if (!ref.endsWith('/')) {
+                            // for a directory, if URL doesn't end with a /
+                            // append / and redirect permanent to that URL
+                            return reply.redirect(`${ref}/`).permanent(true)
+                          } else {
+                            // send directory listing
+                            return reply(data)
+                          }
+                        } else {
+                          // found index file
+                          // redirect to URL/<found-index-file>
+                          return reply.redirect(PathUtils.joinURLParts(ref, data[0].name))
+                        }
+                      }).catch((err) => {
+                        log.error(err)
+                        return reply(err.toString()).code(500)
+                      })
+             } else if (errorToString.startsWith('Error: no link named')) {
+               return reply(errorToString).code(404)
+             } else if (errorToString.startsWith('Error: multihash length inconsistent') ||
+                       errorToString.startsWith('Error: Non-base58 character')) {
+               return reply({Message: errorToString, code: 0}).code(400)
+             } else {
+               log.error(err)
+               return reply({Message: errorToString, code: 0}).code(500)
+             }
+           })
+  }
+}

src/http-api/resources/bootstrap.js src/http/api/resources/bootstrap.js

@@ -0,0 +1,5 @@
+'use strict'
+
+module.exports = {
+  gateway: require('./gateway')
+}

src/http-api/resources/config.js src/http/api/resources/config.js

@@ -0,0 +1,18 @@
+'use strict'
+
+const resources = require('../resources')
+
+module.exports = (server) => {
+  const gateway = server.select('Gateway')
+
+  gateway.route({
+    method: '*',
+    path: '/ipfs/{hash*}',
+    config: {
+      pre: [
+        { method: resources.gateway.checkHash, assign: 'args' }
+      ],
+      handler: resources.gateway.handler
+    }
+  })
+}

src/http-api/resources/files.js src/http/api/resources/files.js

@@ -0,0 +1,5 @@
+'use strict'
+
+module.exports = (server) => {
+  require('./gateway')(server)
+}