feat: make ipfs.get output tarballs (#3785)

`ipfs get` on the cli writes files/folders to disk, also tarballs
and gzipped tarballs. Also gzipped individual files.

In the API it's very similar to `ipfs.ls`.  Make it more like the
cli so there's a clear reason to use one or the other.

Also adds types to `interface-ipfs-core` to better catch broken
types in `ipfs-core-types`.

BREAKING CHANGE: the output type of `ipfs.get` has changed and the `recursive` option has been removed from `ipfs.ls` since it was not supported everywhere

Author: achingbrain

package.json

@@ -38,7 +38,7 @@
     "test:electron-renderer": "aegir test -t electron-renderer",
     "test:chrome": "aegir test -t browser -t webworker -- --browsers ChromeHeadless",
     "test:firefox": "aegir test -t browser -t webworker -- --browsers FirefoxHeadless",
-    "lint": "aegir lint",
+    "lint": "aegir ts -p check && aegir lint",
     "coverage": "npx nyc -r html npm run test:node -- --bail",
     "clean": "rimraf ./dist",
     "dep-check": "aegir dep-check -i ipfs-core -i rimraf -i ipfs-core-types -i abort-controller"
@@ -56,8 +56,6 @@
     "ipfs-utils": "^8.1.4",
     "it-first": "^1.0.6",
     "it-last": "^1.0.4",
-    "it-map": "^1.0.4",
-    "it-tar": "^3.0.0",
     "it-to-stream": "^1.0.0",
     "merge-options": "^3.0.4",
     "multiaddr": "^10.0.0",
@@ -72,9 +70,8 @@
     "aegir": "^34.0.2",
     "delay": "^5.0.0",
     "go-ipfs": "0.8.0",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "it-all": "^1.0.4",
-    "it-concat": "^2.0.0",
     "it-first": "^1.0.4",
     "nock": "^13.0.2",
     "p-defer": "^3.0.0",

src/config/replace.js

@@ -21,7 +21,6 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('config/replace', {
       timeout: options.timeout,
       signal,

src/dag/put.js

@@ -36,7 +36,6 @@ module.exports = (codecs, options) => {
       const controller = new AbortController()
       const signal = abortSignal(controller.signal, settings.signal)
 
-      // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
       const res = await api.post('dag/put', {
         timeout: settings.timeout,
         signal,

src/dht/put.js

@@ -7,6 +7,7 @@ const toUrlSearchParams = require('../lib/to-url-search-params')
 const multipartRequest = require('../lib/multipart-request')
 const abortSignal = require('../lib/abort-signal')
 const { AbortController } = require('native-abort-controller')
+const uint8ArrayToString = require('uint8arrays/to-string')
 
 /**
  * @typedef {import('../types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -22,12 +23,11 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('dht/put', {
       timeout: options.timeout,
       signal,
       searchParams: toUrlSearchParams({
-        arg: key,
+        arg: uint8ArrayToString(key),
         ...options
       }),
       ...(

src/files/write.js

@@ -22,7 +22,6 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('files/write', {
       timeout: options.timeout,
       signal,

src/get.js

@@ -1,11 +1,8 @@
 'use strict'
 
-// @ts-ignore no types
-const Tar = require('it-tar')
 const { CID } = require('multiformats/cid')
 const configure = require('./lib/configure')
 const toUrlSearchParams = require('./lib/to-url-search-params')
-const map = require('it-map')
 
 /**
  * @typedef {import('./types').HTTPClientExtraOptions} HTTPClientExtraOptions
@@ -17,36 +14,25 @@ module.exports = configure(api => {
    * @type {RootAPI["get"]}
    */
   async function * get (path, options = {}) {
+    /** @type {Record<string, any>} */
+    const opts = {
+      arg: `${path instanceof Uint8Array ? CID.decode(path) : path}`,
+      ...options
+    }
+
+    if (opts.compressionLevel) {
+      opts['compression-level'] = opts.compressionLevel
+      delete opts.compressionLevel
+    }
+
     const res = await api.post('get', {
       timeout: options.timeout,
       signal: options.signal,
-      searchParams: toUrlSearchParams({
-        arg: `${path instanceof Uint8Array ? CID.decode(path) : path}`,
-        ...options
-      }),
+      searchParams: toUrlSearchParams(opts),
       headers: options.headers
     })
 
-    const extractor = Tar.extract()
-
-    for await (const { header, body } of extractor(res.iterator())) {
-      if (header.type === 'directory') {
-        // @ts-ignore - Missing the following properties from type 'Directory':
-        // cid, name, size, depthts
-        yield {
-          type: 'dir',
-          path: header.name
-        }
-      } else {
-        // @ts-ignore - Missing the following properties from type 'File':
-        // cid, name, size, depthts
-        yield {
-          type: 'file',
-          path: header.name,
-          content: map(body, (chunk) => chunk.slice()) // convert bl to Buffer/Uint8Array
-        }
-      }
-    }
+    yield * res.iterator()
   }
 
   return get

src/index.js

@@ -10,16 +10,15 @@ const Multihashes = require('ipfs-core-utils/src/multihashes')
 const Multibases = require('ipfs-core-utils/src/multibases')
 const dagPb = require('@ipld/dag-pb')
 const dagCbor = require('@ipld/dag-cbor')
-const raw = require('multiformats/codecs/raw')
-const json = require('multiformats/codecs/json')
-const { sha256, sha512 } = require('multiformats/hashes/sha2')
 const { identity } = require('multiformats/hashes/identity')
-const { base58btc } = require('multiformats/bases/base58')
+const { bases, hashes, codecs } = require('multiformats/basics')
 
 /**
  * @typedef {import('./types').EndpointConfig} EndpointConfig
  * @typedef {import('./types').Options} Options
  * @typedef {import('multiformats/codecs/interface').BlockCodec<any, any>} BlockCodec
+ * @typedef {import('multiformats/hashes/interface').MultihashHasher} MultihashHasher
+ * @typedef {import('multiformats/bases/interface').MultibaseCodec<any>} MultibaseCodec
  * @typedef {import('./types').IPFSHTTPClient} IPFSHTTPClient
  */
 
@@ -37,16 +36,33 @@ function create (options = {}) {
     decode: (id) => id
   }
 
-  const bases = new Multibases({
-    bases: [base58btc].concat(options.ipld && options.ipld.bases ? options.ipld.bases : []),
+  /** @type {MultibaseCodec[]} */
+  const multibaseCodecs = Object.values(bases);
+
+  (options.ipld && options.ipld.bases ? options.ipld.bases : []).forEach(base => multibaseCodecs.push(base))
+
+  const multibases = new Multibases({
+    bases: multibaseCodecs,
     loadBase: options.ipld && options.ipld.loadBase
   })
-  const codecs = new Multicodecs({
-    codecs: [dagPb, dagCbor, raw, json, id].concat(options.ipld?.codecs || []),
+
+  /** @type {BlockCodec[]} */
+  const blockCodecs = Object.values(codecs);
+
+  [dagPb, dagCbor, id].concat((options.ipld && options.ipld.codecs) || []).forEach(codec => blockCodecs.push(codec))
+
+  const multicodecs = new Multicodecs({
+    codecs: blockCodecs,
     loadCodec: options.ipld && options.ipld.loadCodec
   })
-  const hashers = new Multihashes({
-    hashers: [sha256, sha512, identity].concat(options.ipld && options.ipld.hashers ? options.ipld.hashers : []),
+
+  /** @type {MultihashHasher[]} */
+  const multihashHashers = Object.values(hashes);
+
+  (options.ipld && options.ipld.hashers ? options.ipld.hashers : []).forEach(hasher => multihashHashers.push(hasher))
+
+  const multihashes = new Multihashes({
+    hashers: multihashHashers,
     loadHasher: options.ipld && options.ipld.loadHasher
   })
 
@@ -60,7 +76,7 @@ function create (options = {}) {
     cat: require('./cat')(options),
     commands: require('./commands')(options),
     config: require('./config')(options),
-    dag: require('./dag')(codecs, options),
+    dag: require('./dag')(multicodecs, options),
     dht: require('./dht')(options),
     diag: require('./diag')(options),
     dns: require('./dns')(options),
@@ -74,7 +90,7 @@ function create (options = {}) {
     ls: require('./ls')(options),
     mount: require('./mount')(options),
     name: require('./name')(options),
-    object: require('./object')(codecs, options),
+    object: require('./object')(multicodecs, options),
     pin: require('./pin')(options),
     ping: require('./ping')(options),
     pubsub: require('./pubsub')(options),
@@ -86,9 +102,9 @@ function create (options = {}) {
     stop: require('./stop')(options),
     swarm: require('./swarm')(options),
     version: require('./version')(options),
-    bases,
-    codecs,
-    hashers
+    bases: multibases,
+    codecs: multicodecs,
+    hashers: multihashes
   }
 
   return client

src/lib/multipart-request.browser.js

@@ -2,7 +2,7 @@
 
 // Import browser version otherwise electron-renderer will end up with node
 // version and fail.
-const normaliseInput = require('ipfs-core-utils/src/files/normalise-input/index.browser')
+const { normaliseInput } = require('ipfs-core-utils/src/files/normalise-input/index.browser')
 const modeToString = require('./mode-to-string')
 
 /**

src/lib/multipart-request.node.js

@@ -1,6 +1,6 @@
 'use strict'
 
-const normaliseInput = require('ipfs-core-utils/src/files/normalise-input')
+const { normaliseInput } = require('ipfs-core-utils/src/files/normalise-input')
 const { nanoid } = require('nanoid')
 const modeToString = require('./mode-to-string')
 const merge = require('merge-options').bind({ ignoreUndefined: true })

src/ls.js

@@ -39,8 +39,7 @@ module.exports = configure((api, opts) => {
         path: pathStr + (link.Name ? `/${link.Name}` : ''),
         size: link.Size,
         cid: hash,
-        type: typeOf(link),
-        depth: link.Depth || 1
+        type: typeOf(link)
       }
 
       if (link.Mode) {

src/name/publish.js

@@ -18,7 +18,7 @@ module.exports = configure(api => {
       timeout: options.timeout,
       signal: options.signal,
       searchParams: toUrlSearchParams({
-        arg: path,
+        arg: `${path}`,
         ...options
       }),
       headers: options.headers

src/object/patch/append-data.js

@@ -21,7 +21,6 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('object/patch/append-data', {
       timeout: options.timeout,
       signal,

src/object/patch/set-data.js

@@ -21,7 +21,6 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('object/patch/set-data', {
       timeout: options.timeout,
       signal,

src/pin/add-all.js

@@ -2,7 +2,7 @@
 
 const { CID } = require('multiformats/cid')
 const configure = require('../lib/configure')
-const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input')
+const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input')
 const toUrlSearchParams = require('../lib/to-url-search-params')
 
 /**

src/pin/rm-all.js

@@ -2,7 +2,7 @@
 
 const { CID } = require('multiformats/cid')
 const configure = require('../lib/configure')
-const normaliseInput = require('ipfs-core-utils/src/pins/normalise-input')
+const { normaliseInput } = require('ipfs-core-utils/src/pins/normalise-input')
 const toUrlSearchParams = require('../lib/to-url-search-params')
 
 /**

src/pubsub/publish.js

@@ -25,7 +25,6 @@ module.exports = configure(api => {
     const controller = new AbortController()
     const signal = abortSignal(controller.signal, options.signal)
 
-    // @ts-ignore https://github.com/ipfs/js-ipfs-utils/issues/90
     const res = await api.post('pubsub/pub', {
       timeout: options.timeout,
       signal,

src/pubsub/subscribe.js

@@ -41,39 +41,37 @@ module.exports = (options, subsTracker) => {
       const ffWorkaround = setTimeout(() => done(), 1000)
 
       // Do this async to not block Firefox
-      setTimeout(() => {
-        api.post('pubsub/sub', {
-          timeout: options.timeout,
-          signal: options.signal,
-          searchParams: toUrlSearchParams({
-            arg: topic,
-            ...options
-          }),
-          headers: options.headers
-        })
-          .catch((err) => {
-            // Initial subscribe fail, ensure we clean up
-            subsTracker.unsubscribe(topic, handler)
+      api.post('pubsub/sub', {
+        timeout: options.timeout,
+        signal: options.signal,
+        searchParams: toUrlSearchParams({
+          arg: topic,
+          ...options
+        }),
+        headers: options.headers
+      })
+        .catch((err) => {
+          // Initial subscribe fail, ensure we clean up
+          subsTracker.unsubscribe(topic, handler)
 
-            fail(err)
+          fail(err)
+        })
+        .then((response) => {
+          clearTimeout(ffWorkaround)
+
+          if (!response) {
+            // if there was no response, the subscribe failed
+            return
+          }
+
+          readMessages(response, {
+            onMessage: handler,
+            onEnd: () => subsTracker.unsubscribe(topic, handler),
+            onError: options.onError
           })
-          .then((response) => {
-            clearTimeout(ffWorkaround)
-
-            if (!response) {
-              // if there was no response, the subscribe failed
-              return
-            }
 
-            readMessages(response, {
-              onMessage: handler,
-              onEnd: () => subsTracker.unsubscribe(topic, handler),
-              onError: options.onError
-            })
-
-            done()
-          })
-      }, 0)
+          done()
+        })
 
       return result
     }

src/types.d.ts

@@ -2,6 +2,8 @@ import { Agent as HttpAgent } from 'http'
 import { Agent as HttpsAgent } from 'https'
 import { Multiaddr } from 'multiaddr'
 import type { BlockCodec } from 'multiformats/codecs/interface'
+import type { MultihashHasher } from 'multiformats/hashes/interface'
+import type { MultibaseCodec } from 'multiformats/bases/interface'
 import type { IPFS } from 'ipfs-core-types'
 
 export interface Options {
@@ -12,7 +14,7 @@ export interface Options {
   timeout?: number | string
   apiPath?: string
   url?: URL|string|Multiaddr
-  ipld?: IPLDOptions
+  ipld?: Partial<IPLDOptions>
   agent?: HttpAgent | HttpsAgent
 }