feat: remove all esoteric ipld formats (#3360)

We bundle ipld formats for ethereum, zcash, bitcoin and git though they speak beyond the files part of the Interplanetary File System.

Since #3347 we can now configure extra IPLD formats in the http client, in-process and daemon nodes, we no longer have to bundle these formats for them to be supported, instead the user can choose to configure their node with the formats they require.

This makes the behaviour of core the same in node as it is in the browser and also means we don't waste time installing deps our users may not use.

If they do use them, they can configure the node as they see fit.
    
BREAKING CHANGE: only dag-pb, dag-cbor and raw formats are supported out of the box, any others will need to be configured during node startup.

Author: achingbrain

examples/explore-ethereum-blockchain/package.json

@@ -13,6 +13,7 @@
     "ipfs": "^0.51.0",
     "ipfs-http-client": "^48.0.0",
     "ipfsd-ctl": "^7.0.2",
+    "ipld-ethereum": "^5.0.1",
     "test-ipfs-example": "^2.0.3"
   }
 }

examples/explore-ethereum-blockchain/test.js

@@ -15,7 +15,14 @@ const df = createFactory({
 async function runTest () {
   const ipfsd = await df.spawn({
     type: 'proc',
-    test: true
+    test: true,
+    ipfsOptions: {
+      ipld: {
+        formats: [
+          ...Object.values(require('ipld-ethereum'))
+        ]
+      }
+    }
   })
 
   const cids = []

examples/traverse-ipld-graphs/README.md

@@ -16,6 +16,32 @@ $ npm install
 $ npm run build
 ```
 
+## IPLD Formats
+
+[IPLD](https://docs.ipld.io/) can read many datatypes, all of which are represented as blocks in the blockstore of your IPFS node.  In order to turn a block into a data structure it can use, IPLD uses different codecs to turn `Uint8Arrays` into JavaScript objects and back.
+
+By default IPFS is bundled with [dag-pb](https://www.npmjs.com/package/ipld-dag-pb), [dag-cbor](https://www.npmjs.com/package/ipld-dag-cbor) and [raw](https://www.npmjs.com/package/ipld-raw) codecs which allow reading UnixFS files and JavaScript objects from the blockstore.
+
+To configure other types, we must pass the `ipld.formats` option to the `IPFS.create()` function:
+
+```javascript
+const IPFS = require('ipfs')
+
+const node = await IPFS.create({
+  ipld: {
+    formats: [
+      require('ipld-git'),
+      require('ipld-zcash'),
+      require('ipld-bitcoin'),
+      ...Object.values(require('ipld-ethereum')) // this format exports multiple codecs so flatten into a list
+      // etc, etc
+    ]
+  }
+})
+```
+
+See [ipld/interface-ipld-format](https://github.com/ipld/interface-ipld-format) for a list of modules that implement the `ipld-format` interface.
+
 ## [create nodes to build a graph](./put.js)
 
 ## [retrieve a node from a graph](./get.js)

examples/traverse-ipld-graphs/create-node.js

@@ -15,7 +15,8 @@ function createNode (options) {
         API: '/ip4/127.0.0.1/tcp/0',
         Gateway: '/ip4/127.0.0.1/tcp/0'
       }
-    }
+    },
+    ipld: options.ipld
   })
 }
 

examples/traverse-ipld-graphs/eth.js

@@ -9,7 +9,13 @@ const fs = require('fs').promises
 const uint8ArrayToString = require('uint8arrays/to-string')
 
 async function main () {
-  const ipfs = await createNode()
+  const ipfs = await createNode({
+    ipld: {
+      formats: [
+        ...Object.values(require('ipld-ethereum'))
+      ]
+    }
+  })
 
   console.log('\nStart of the example:')
 

examples/traverse-ipld-graphs/git.js

@@ -9,7 +9,13 @@ const fs = require('fs').promises
 const uint8ArrayToString = require('uint8arrays/to-string')
 
 async function main () {
-  const ipfs = await createNode()
+  const ipfs = await createNode({
+    ipld: {
+      formats: [
+        require('ipld-git')
+      ]
+    }
+  })
 
   console.log('\nStart of the example:')
 

examples/traverse-ipld-graphs/package.json

@@ -17,6 +17,8 @@
     "ipfs": "^0.51.0",
     "ipld-block": "^0.10.1",
     "ipld-dag-pb": "^0.20.0",
+    "ipld-git": "^0.6.1",
+    "ipld-ethereum": "^5.0.1",
     "multihashing-async": "^2.0.1"
   }
 }

packages/ipfs-core/package.json

@@ -22,7 +22,6 @@
     "./src/runtime/libp2p-pubsub-routers-nodejs.js": "./src/runtime/libp2p-pubsub-routers-browser.js",
     "./src/runtime/preload-nodejs.js": "./src/runtime/preload-browser.js",
     "./src/runtime/repo-nodejs.js": "./src/runtime/repo-browser.js",
-    "./src/runtime/ipld-nodejs.js": "./src/runtime/ipld-browser.js",
     "./test/utils/create-repo-nodejs.js": "./test/utils/create-repo-browser.js",
     "ipfs-utils/src/files/glob-source": false
   },
@@ -78,14 +77,10 @@
     "ipfs-unixfs-importer": "^3.0.4",
     "ipfs-utils": "^4.0.0",
     "ipld": "^0.27.2",
-    "ipld-bitcoin": "^0.4.0",
     "ipld-block": "^0.10.1",
     "ipld-dag-cbor": "^0.17.0",
     "ipld-dag-pb": "^0.20.0",
-    "ipld-ethereum": "^5.0.1",
-    "ipld-git": "^0.6.1",
     "ipld-raw": "^6.0.0",
-    "ipld-zcash": "^0.5.0",
     "ipns": "^0.8.0",
     "is-domain-name": "^1.0.1",
     "is-ipfs": "^2.0.0",
@@ -129,6 +124,7 @@
     "delay": "^4.4.0",
     "interface-ipfs-core": "^0.141.0",
     "ipfsd-ctl": "^7.0.2",
+    "ipld-git": "^0.6.1",
     "iso-random-stream": "^1.1.1",
     "iso-url": "^0.4.7",
     "nanoid": "^3.1.12",

packages/ipfs-core/src/components/init.js

@@ -24,7 +24,7 @@ const BlockService = require('ipfs-block-service')
  * @typedef {import('.').IPLD} IPLD
  */
 const Ipld = require('ipld')
-const getDefaultIpldOptions = require('../runtime/ipld-nodejs')
+const getDefaultIpldOptions = require('../runtime/ipld')
 
 const createPreloader = require('../preload')
 const { ERR_REPO_NOT_INITIALIZED } = require('ipfs-repo').errors
@@ -486,8 +486,8 @@ function createApi ({
  * @property {import('.').IPLDConfig} [ipld] - Modify the default IPLD config. This object
  * will be *merged* with the default config; it will not replace it. Check IPLD
  * [docs](https://github.com/ipld/js-ipld#ipld-constructor) for more information
- * on the available options. (Default: [`ipld-nodejs.js`]
- * (https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-nodejs.js) in Node.js, [`ipld-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-browser.js)
+ * on the available options. (Default: [`ipld.js`]
+ * (https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld.js)
  * in browsers)
  * @property {object|Function} [libp2p] - The libp2p option allows you to build
  * your libp2p node by configuration, or via a bundle function. If you are

packages/ipfs-core/src/index.js

@@ -135,8 +135,8 @@ module.exports = {
  * @property {import('./components').IPLDConfig} [ipld] - Modify the default IPLD config. This object
  * will be *merged* with the default config; it will not replace it. Check IPLD
  * [docs](https://github.com/ipld/js-ipld#ipld-constructor) for more information
- * on the available options. (Default: [`ipld-nodejs.js`]
- * (https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-nodejs.js) in Node.js, [`ipld-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-browser.js)
+ * on the available options. (Default: [`ipld.js`]
+ * (https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld.js)
  * in browsers)
  * @property {object|Function} [libp2p] - The libp2p option allows you to build
  * your libp2p node by configuration, or via a bundle function. If you are

packages/ipfs-core/src/runtime/ipld-browser.js

@@ -12,36 +12,6 @@ const IpldFormats = {
   },
   get [multicodec.RAW] () {
     return require('ipld-raw')
-  },
-  get [multicodec.BITCOIN_BLOCK] () {
-    return require('ipld-bitcoin')
-  },
-  get [multicodec.ETH_ACCOUNT_SNAPSHOT] () {
-    return require('ipld-ethereum').ethAccountSnapshot
-  },
-  get [multicodec.ETH_BLOCK] () {
-    return require('ipld-ethereum').ethBlock
-  },
-  get [multicodec.ETH_BLOCK_LIST] () {
-    return require('ipld-ethereum').ethBlockList
-  },
-  get [multicodec.ETH_STATE_TRIE] () {
-    return require('ipld-ethereum').ethStateTrie
-  },
-  get [multicodec.ETH_STORAGE_TRIE] () {
-    return require('ipld-ethereum').ethStorageTrie
-  },
-  get [multicodec.ETH_TX] () {
-    return require('ipld-ethereum').ethTx
-  },
-  get [multicodec.ETH_TX_TRIE] () {
-    return require('ipld-ethereum').ethTxTrie
-  },
-  get [multicodec.GIT_RAW] () {
-    return require('ipld-git')
-  },
-  get [multicodec.ZCASH_BLOCK] () {
-    return require('ipld-zcash')
   }
 }
 

packages/ipfs-core/src/runtime/ipld-nodejs.js packages/ipfs-core/src/runtime/ipld.js

@@ -187,7 +187,7 @@ Modify the default IPFS node config. This object will be *merged* with the defau
 
  | Type | Default |
 |------|---------|
-| object |  [`ipld-nodejs.js`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-nodejs.js) in Node.js, [`ipld-browser.js`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld-browser.js) in browsers |
+| object |  [`ipld.js`](https://github.com/ipfs/js-ipfs/tree/master/packages/ipfs/src/core/runtime/ipld.js) |
 
  Modify the default IPLD config. This object will be *merged* with the default config; it will not replace it. Check IPLD [docs](https://github.com/ipld/js-ipld#ipld-constructor) for more information on the available options.