feat: support specify hash algorithm in files.add

package.json

@@ -99,14 +99,14 @@
     "hapi": "^16.5.2",
     "hapi-set-header": "^1.0.2",
     "hoek": "^4.2.0",
-    "ipfs-api": "^14.3.3",
+    "ipfs-api": "^14.3.5",
     "ipfs-bitswap": "~0.17.2",
     "ipfs-block": "~0.6.0",
     "ipfs-block-service": "~0.12.0",
     "ipfs-multipart": "~0.1.0",
     "ipfs-repo": "~0.17.0",
     "ipfs-unixfs": "~0.1.13",
-    "ipfs-unixfs-engine": "~0.22.3",
+    "ipfs-unixfs-engine": "~0.22.5",
     "ipld-resolver": "~0.13.2",
     "is-ipfs": "^0.3.0",
     "is-stream": "^1.1.0",

src/cli/commands/files/add.js

@@ -8,6 +8,7 @@ const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 const zip = require('pull-zip')
 const toPull = require('stream-to-pull-stream')
+const mh = require('multihashes')
 const utils = require('../../utils')
 const print = require('../../utils').print
 
@@ -151,6 +152,11 @@ module.exports = {
       type: 'boolean',
       default: false,
       describe: 'Write no output'
+    },
+    hash: {
+      type: 'string',
+      choices: [undefined].concat(Object.keys(mh.names)),
+      describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)'
     }
   },
 
@@ -160,8 +166,9 @@ module.exports = {
     const options = {
       strategy: argv.trickle ? 'trickle' : 'balanced',
       shardSplitThreshold: argv.enableShardingExperiment ? argv.shardSplitThreshold : Infinity,
-      'cid-version': argv['cid-version'],
-      'raw-leaves': argv['raw-leaves']
+      cidVersion: argv.cidVersion,
+      rawLeaves: argv.rawLeaves,
+      hashAlg: argv.hash
     }
 
     // Temporary restriction on raw-leaves:
@@ -172,11 +179,23 @@ module.exports = {
     // cid-version > 0 unless explicitly set to false.
     //
     // This retains feature parity without having to implement raw-leaves.
-    if (argv['cid-version'] > 0 && argv['raw-leaves'] !== false) {
+    if (argv.cidVersion > 0 && argv.rawLeaves !== false) {
       throw new Error('Implied argument raw-leaves must be passed and set to false when cid-version is > 0')
     }
 
-    if (argv['raw-leaves']) {
+    // Temporary restriction on raw-leaves:
+    // When hash != undefined then raw-leaves MUST be present and false.
+    //
+    // This is because raw-leaves is not yet implemented in js-ipfs,
+    // and go-ipfs changes the value of raw-leaves to true when
+    // hash != undefined unless explicitly set to false.
+    //
+    // This retains feature parity without having to implement raw-leaves.
+    if (argv.hash && argv.rawLeaves !== false) {
+      throw new Error('Implied argument raw-leaves must be passed and set to false when hash argument is specified')
+    }
+
+    if (argv.rawLeaves) {
       throw new Error('Not implemented: raw-leaves')
     }
 

src/core/components/files.js

@@ -20,6 +20,10 @@ module.exports = function files (self) {
       shardSplitThreshold: self._options.EXPERIMENTAL.sharding ? 1000 : Infinity
     }, options)
 
+    if (opts.hashAlg && opts.cidVersion !== 1) {
+      opts.cidVersion = 1
+    }
+
     return pull(
       pull.map(normalizeContent),
       pull.flatten(),
@@ -65,6 +69,13 @@ module.exports = function files (self) {
         return callback(new Error('Invalid arguments, data must be an object, Buffer or readable stream'))
       }
 
+      options = options || {}
+
+      // CID v0 is for multihashes encoded with sha2-256
+      if (options.hashAlg && options.cidVersion !== 1) {
+        options.cidVersion = 1
+      }
+
       pull(
         pull.values(normalizeContent(data)),
         importer(self._ipldResolver, options),
@@ -117,15 +128,15 @@ module.exports = function files (self) {
 function prepareFile (self, opts, file, callback) {
   opts = opts || {}
 
-  waterfall([
-    (cb) => self.object.get(file.multihash, cb),
-    (node, cb) => {
-      let cid = new CID(node.multihash)
+  let cid = new CID(file.multihash)
 
-      if (opts['cid-version'] === 1) {
-        cid = cid.toV1()
-      }
+  if (opts.cidVersion === 1) {
+    cid = cid.toV1()
+  }
 
+  waterfall([
+    (cb) => self.object.get(cid, cb),
+    (node, cb) => {
       const b58Hash = cid.toBaseEncodedString()
 
       cb(null, {

src/core/components/object.js

@@ -10,20 +10,6 @@ const mh = require('multihashes')
 const Unixfs = require('ipfs-unixfs')
 const assert = require('assert')
 
-function normalizeMultihash (multihash, enc) {
-  if (typeof multihash === 'string') {
-    if (enc === 'base58' || !enc) {
-      return multihash
-    }
-
-    return new Buffer(multihash, enc)
-  } else if (Buffer.isBuffer(multihash)) {
-    return multihash
-  } else {
-    throw new Error('unsupported multihash')
-  }
-}
-
 function parseBuffer (buf, encoding, callback) {
   switch (encoding) {
     case 'json':
@@ -178,20 +164,17 @@ module.exports = function object (self) {
       }
     }),
 
-    get: promisify((multihash, options, callback) => {
+    get: promisify((cid, options, callback) => {
       if (typeof options === 'function') {
         callback = options
         options = {}
       }
 
-      let mh
-
       try {
-        mh = normalizeMultihash(multihash, options.enc)
+        cid = new CID(cid)
       } catch (err) {
         return callback(err)
       }
-      const cid = new CID(mh)
 
       self._ipldResolver.get(cid, (err, result) => {
         if (err) {
@@ -204,13 +187,19 @@ module.exports = function object (self) {
       })
     }),
 
-    data: promisify((multihash, options, callback) => {
+    data: promisify((cid, options, callback) => {
       if (typeof options === 'function') {
         callback = options
         options = {}
       }
 
-      self.object.get(multihash, options, (err, node) => {
+      try {
+        cid = new CID(cid)
+      } catch (err) {
+        return callback(err)
+      }
+
+      self.object.get(cid, options, (err, node) => {
         if (err) {
           return callback(err)
         }

src/http-api/resources/files.js

@@ -145,19 +145,19 @@ exports.add = {
     query: Joi.object()
       .keys({
         'cid-version': Joi.number().integer().min(0).max(1),
+        hash: Joi.string().valid(Object.keys(mh.names)),
         // Temporary restriction on raw-leaves:
-        // When cid-version=1 then raw-leaves MUST be present and false.
+        // When cid-version > 0 or hash != undefined then raw-leaves MUST be
+        // present and false.
         //
         // This is because raw-leaves is not yet implemented in js-ipfs,
         // and go-ipfs changes the value of raw-leaves to true when
-        // cid-version > 0 unless explicitly set to false.
+        // cid-version > 0 or hash != undefined unless explicitly set to false.
         //
         // This retains feature parity without having to implement raw-leaves.
-        'raw-leaves': Joi.any().when('cid-version', {
-          is: 1,
-          then: Joi.boolean().valid(false).required(),
-          otherwise: Joi.boolean().valid(false)
-        })
+        'raw-leaves': Joi.boolean().valid(false)
+          .when('cid-version', { is: 1, then: Joi.required() })
+          .when('hash', { is: Joi.string(), then: Joi.required() })
       })
       // TODO: Necessary until validate "recursive", "stream-channels" etc.
       .options({ allowUnknown: true })
@@ -208,8 +208,9 @@ exports.add = {
     })
 
     const options = {
-      'cid-version': request.query['cid-version'],
-      'raw-leaves': request.query['raw-leaves']
+      cidVersion: request.query['cid-version'],
+      rawLeaves: request.query['raw-leaves'],
+      hashAlg: request.query['hash']
     }
 
     pull(

src/http-api/resources/object.js

@@ -7,6 +7,7 @@ const DAGLink = dagPB.DAGLink
 const DAGNode = dagPB.DAGNode
 const waterfall = require('async/waterfall')
 const series = require('async/series')
+const CID = require('cids')
 const debug = require('debug')
 const log = debug('jsipfs:http-api:object')
 log.error = debug('jsipfs:http-api:object:error')
@@ -20,9 +21,7 @@ exports.parseKey = (request, reply) => {
   }
 
   try {
-    return reply({
-      key: mh.fromB58String(request.query.arg)
-    })
+    return reply({ key: new CID(request.query.arg) })
   } catch (err) {
     log.error(err)
     return reply({

test/cli/files.js

@@ -6,8 +6,22 @@ const fs = require('fs')
 const path = require('path')
 const compareDir = require('dir-compare').compareSync
 const rimraf = require('rimraf').sync
+const CID = require('cids')
+const mh = require('multihashes')
 const runOnAndOff = require('../utils/on-and-off')
 
+// TODO: Test against all algorithms Object.keys(mh.names)
+// This subset is known to work with both go-ipfs and js-ipfs as of 2017-09-05
+const HASH_ALGS = [
+  'sha1',
+  'sha2-256',
+  'sha2-512',
+  'keccak-224',
+  'keccak-256',
+  'keccak-384',
+  'keccak-512'
+]
+
 describe('files', () => runOnAndOff((thing) => {
   let ipfs
   const readme = fs.readFileSync(path.join(process.cwd(), '/src/init-files/init-docs/readme'))
@@ -222,6 +236,17 @@ describe('files', () => runOnAndOff((thing) => {
       })
   })
 
+  HASH_ALGS.forEach((name) => {
+    it(`add with hash=${name} and raw-leaves=false`, () => {
+      return ipfs(`add src/init-files/init-docs/readme --hash=${name} --raw-leaves=false`)
+        .then((out) => {
+          const hash = out.split(' ')[1]
+          const cid = new CID(hash)
+          expect(mh.decode(cid.multihash).name).to.equal(name)
+        })
+    })
+  })
+
   it('cat', () => {
     return ipfs('files cat QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB')
       .then((out) => {