feat: make ipfs.get output tarballs (#3785)

`ipfs get` on the cli writes files/folders to disk, also tarballs
and gzipped tarballs. Also gzipped individual files.

In the API it's very similar to `ipfs.ls`.  Make it more like the
cli so there's a clear reason to use one or the other.

Also adds types to `interface-ipfs-core` to better catch broken
types in `ipfs-core-types`.

BREAKING CHANGE: the output type of `ipfs.get` has changed and the `recursive` option has been removed from `ipfs.ls` since it was not supported everywhere

Author: achingbrain

.travis.yml

@@ -84,6 +84,7 @@ jobs:
     - stage: test
       name: lint
       script:
+        - npm run build
         - npm run lint -- $RUN_SINCE --concurrency 1
 
     - stage: test

docs/core-api/FILES.md

@@ -447,53 +447,39 @@ An optional object which may have the following keys:
 
 | Name | Type | Default | Description |
 | ---- | ---- | ------- | ----------- |
+| archive | `boolean` | `undefined` | Return the file/directory in a tarball |
+| compress | `boolean` | `false` | Gzip the returned stream |
+| compressionLevel | `Number` | `undefined` | How much compression to apply (1-9) |
 | timeout | `Number` | `undefined` | A timeout in ms |
 | signal | [AbortSignal][] | `undefined` |  Can be used to cancel any long running requests started as a result of this call |
 
 #### Returns
 
 | Type | Description |
 | -------- | -------- |
-| `AsyncIterable<Object>` | An async iterable that yields objects representing the files |
+| `AsyncIterable<Uint8Array>` | An async iterable that yields bytes |
 
-Each yielded object is of the form:
+What is streamed as a response depends on the options passed and what the `ipfsPath` resolves to.
 
-```js
-{
-  type: string, // 'file' or 'dir'
-  path: string, // a deeply nested path within the directory structure
-  content?: <AsyncIterable<Uint8Array>>, // only present if `type` is 'file'
-  mode: Number, // implicit if not provided - 0644 for files, 0755 for directories
-  mtime?: { secs: Number, nsecs: Number }
-}
-```
-
-Here, each `path` corresponds to the name of a file, and `content` is an async iterable with the file contents.
+1. If `ipfsPath` resolves to a file:
+   * By default you will get a tarball containing the file
+   * Pass `compress: true` (and an optional `compressionLevel`) to instead get the gzipped file contents
+   * Pass `compress: true` (and an optional `compressionLevel`) AND `archive: true` to get a gzipped tarball containing the file
+2. If `ipfsPath` resolves to a directory:
+   * By default you will get a tarball containing the contents of the directory
+   * Passing `compress: true` will cause an error
+   * Pass `compress: true` (and an optional `compressionLevel`) AND `archive: true` to get a gzipped tarball containing the contents of the directory
 
 #### Example
 
 ```JavaScript
 const cid = 'QmQ2r6iMNpky5f1m4cnm3Yqw8VSvjuKpTcK1X7dBR1LkJF'
 
-for await (const file of ipfs.get(cid)) {
-  console.log(file.type, file.path)
-
-  if (!file.content) continue;
-
-  const content = []
-
-  for await (const chunk of file.content) {
-    content.push(chunk)
-  }
-
-  console.log(content)
+for await (const buf of ipfs.get(cid)) {
+  // do something with buf
 }
 ```
 
-When invoking this method via the HTTP API client, the response arrives as a stream containing either the entire contents of the file (if the passed [CID][] resolves to a file) or recursive directory tree and all files contained therein (if the passed [CID][] resolves to a directory).
-
-If you are iterating over a directory, in order to proceed to the next entry in the stream, you must consume the `content` field of the current entry if it is present.
-
 A great source of [examples](https://github.com/ipfs/js-ipfs/blob/master/packages/interface-ipfs-core/src/get.js) can be found in the tests for this API.
 
 ### `ipfs.ls(ipfsPath)`

examples/browser-ipns-publish/package.json

@@ -28,7 +28,7 @@
   "devDependencies": {
     "delay": "^5.0.0",
     "execa": "^5.0.0",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "go-ipfs": "0.8.0",
     "parcel": "2.0.0-beta.2",
     "path": "^0.12.7",

examples/http-client-browser-pubsub/package.json

@@ -22,7 +22,7 @@
     "execa": "^5.0.0",
     "go-ipfs": "0.8.0",
     "ipfs": "^0.56.1",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "parcel": "2.0.0-beta.2",
     "test-ipfs-example": "^3.0.0"
   }

examples/http-client-bundle-webpack/package.json

@@ -25,7 +25,7 @@
     "copy-webpack-plugin": "^8.1.0",
     "execa": "^5.0.0",
     "ipfs": "^0.56.1",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "react-hot-loader": "^4.12.21",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^3.0.0",

examples/http-client-name-api/package.json

@@ -18,7 +18,7 @@
   "devDependencies": {
     "execa": "^5.0.0",
     "go-ipfs": "0.8.0",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "parcel": "2.0.0-beta.2",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^3.0.0"

examples/ipfs-client-add-files/package.json

@@ -16,7 +16,7 @@
   "devDependencies": {
     "execa": "^5.0.0",
     "ipfs": "^0.56.1",
-    "ipfsd-ctl": "^9.0.0",
+    "ipfsd-ctl": "^10.0.3",
     "parcel": "2.0.0-beta.2",
     "rimraf": "^3.0.2",
     "test-ipfs-example": "^3.0.0"

packages/interface-ipfs-core/package.json

@@ -13,9 +13,10 @@
     "ipfs-utils/src/files/glob-source": false
   },
   "scripts": {
+    "build": "aegir build",
     "lint": "aegir lint",
     "test": "echo 'No tests here'",
-    "dep-check": "aegir dep-check -i abort-controller"
+    "dep-check": "aegir dep-check -i abort-controller -i ipfs-core-types"
   },
   "files": [
     "src/",
@@ -35,17 +36,21 @@
       "test/fixtures/*"
     ]
   },
+  "types": "dist/src/index.d.ts",
   "dependencies": {
     "@ipld/car": "^3.1.6",
     "@ipld/dag-cbor": "^6.0.5",
     "@ipld/dag-pb": "^2.1.3",
+    "@types/readable-stream": "^2.3.11",
+    "@types/pako": "^1.0.2",
     "abort-controller": "^3.0.0",
     "aegir": "^34.0.2",
     "delay": "^5.0.0",
     "err-code": "^3.0.1",
     "interface-blockstore": "^1.0.0",
+    "ipfs-core-types": "^0.6.0",
     "ipfs-unixfs": "^5.0.0",
-    "ipfs-unixfs-importer": "^8.0.0",
+    "ipfs-unixfs-importer": "^8.0.2",
     "ipfs-utils": "^8.1.4",
     "ipns": "^0.13.2",
     "is-ipfs": "^6.0.1",
@@ -57,7 +62,9 @@
     "it-first": "^1.0.4",
     "it-last": "^1.0.4",
     "it-map": "^1.0.4",
+    "it-pipe": "^1.1.0",
     "it-pushable": "^1.4.2",
+    "it-tar": "^4.0.0",
     "it-to-buffer": "^2.0.0",
     "libp2p-crypto": "^0.19.6",
     "libp2p-websockets": "^0.16.1",
@@ -67,6 +74,7 @@
     "native-abort-controller": "^1.0.3",
     "p-map": "^4.0.0",
     "p-retry": "^4.5.0",
+    "pako": "^1.0.2",
     "peer-id": "^0.15.1",
     "readable-stream": "^3.4.0",
     "uint8arrays": "^2.1.6"