feat: use it-tar (#2758)

`it-tar` now supports packing \o/

package.json

@@ -123,6 +123,7 @@
  "it-glob": "0.0.7",
  "it-last": "^1.0.1",
  "it-pipe": "^1.1.0",
+ "it-tar": "^1.2.1",
  "it-to-stream": "^0.1.1",
  "iterable-ndjson": "^1.1.0",
  "jsondiffpatch": "~0.3.11",
@@ -165,7 +166,6 @@
  "semver": "^7.1.2",
  "stream-to-it": "^0.2.0",
  "streaming-iterables": "^4.1.1",
- "tar-stream": "^2.0.0",
  "temp": "~0.9.0",
  "timeout-abort-controller": "^1.1.0",
  "update-notifier": "^4.0.0",

src/http/api/resources/files-regular.js

@@ -2,7 +2,7 @@
 
 const multipart = require('ipfs-multipart')
 const debug = require('debug')
-const tar = require('tar-stream')
+const tar = require('it-tar')
 const log = debug('ipfs:http-api:files')
 log.error = debug('ipfs:http-api:files:error')
 const toIterable = require('stream-to-it')
@@ -11,16 +11,20 @@ const Boom = require('@hapi/boom')
 const { PassThrough } = require('stream')
 const multibase = require('multibase')
 const isIpfs = require('is-ipfs')
-const { promisify } = require('util')
 const { cidToString } = require('../../../utils/cid')
 const { Format } = require('../../../core/components/refs')
 const pipe = require('it-pipe')
 const all = require('it-all')
-const concat = require('it-concat')
 const ndjson = require('iterable-ndjson')
 const { map } = require('streaming-iterables')
 const streamResponse = require('../../utils/stream-response')
 
+const toBuffer = async function * (source) {
+ for await (const chunk of source) {
+ yield chunk.slice()
+ }
+}
+
 function numberFromQuery (query, key) {
  if (query && query[key] !== undefined) {
  const value = parseInt(query[key], 10)
@@ -86,32 +90,24 @@ exports.get = {
  const { ipfs } = request.server.app
  const { key } = request.pre.args
 
- const pack = tar.pack()
- pack.entry = promisify(pack.entry.bind(pack))
+ return streamResponse(request, h, () => pipe(
+ ipfs.get(key),
+ async function * (source) {
+ for await (const file of source) {
+ const header = {
+ name: file.path
+ }
 
- const streamFiles = async () => {
- try {
- for await (const file of ipfs.get(key)) {
  if (file.content) {
- const content = await concat(file.content)
- pack.entry({ name: file.path, size: file.size }, content.slice())
+ yield { header: { ...header, size: file.size }, body: toBuffer(file.content) }
  } else {
- pack.entry({ name: file.path, type: 'directory' })
+ yield { header: { ...header, type: 'directory' } }
  }
  }
- pack.finalize()
- } catch (err) {
- log.error(err)
- pack.emit('error', err)
- pack.destroy()
- }
- }
-
- streamFiles()
-
- // reply must be called right away so that tar-stream offloads its content
- // otherwise it will block in large files
- return h.response(pack).header('X-Stream-Output', '1')
+ },
+ tar.pack(),
+ toBuffer
+ ))
  }
 }