From bd89bb081be62d0b9de2e92857e43c794ac12475 Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Fri, 7 Feb 2020 23:00:52 +0000 Subject: [PATCH 1/2] feat: use it-tar --- package.json | 2 +- src/http/api/resources/files-regular.js | 44 +++++++++++-------------- 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/package.json b/package.json index 45034674e5..c054124eec 100644 --- a/package.json +++ b/package.json @@ -123,6 +123,7 @@ "it-glob": "0.0.7", "it-last": "^1.0.1", "it-pipe": "^1.1.0", + "it-tar": "^1.2.0", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "jsondiffpatch": "~0.3.11", @@ -165,7 +166,6 @@ "semver": "^7.1.2", "stream-to-it": "^0.2.0", "streaming-iterables": "^4.1.1", - "tar-stream": "^2.0.0", "temp": "~0.9.0", "timeout-abort-controller": "^1.1.0", "update-notifier": "^4.0.0", diff --git a/src/http/api/resources/files-regular.js b/src/http/api/resources/files-regular.js index ff8377fae3..5d4e728e5f 100644 --- a/src/http/api/resources/files-regular.js +++ b/src/http/api/resources/files-regular.js @@ -2,7 +2,7 @@ const multipart = require('ipfs-multipart') const debug = require('debug') -const tar = require('tar-stream') +const tar = require('it-tar') const log = debug('ipfs:http-api:files') log.error = debug('ipfs:http-api:files:error') const toIterable = require('stream-to-it') @@ -11,16 +11,20 @@ const Boom = require('@hapi/boom') const { PassThrough } = require('stream') const multibase = require('multibase') const isIpfs = require('is-ipfs') -const { promisify } = require('util') const { cidToString } = require('../../../utils/cid') const { Format } = require('../../../core/components/refs') const pipe = require('it-pipe') const all = require('it-all') -const concat = require('it-concat') const ndjson = require('iterable-ndjson') const { map } = require('streaming-iterables') const streamResponse = require('../../utils/stream-response') +const toBuffer = async function * (source) { + for await (const chunk of source) { + yield chunk.slice() + } +} + function numberFromQuery (query, key) { if (query && query[key] !== undefined) { const value = parseInt(query[key], 10) @@ -86,32 +90,24 @@ exports.get = { const { ipfs } = request.server.app const { key } = request.pre.args - const pack = tar.pack() - pack.entry = promisify(pack.entry.bind(pack)) + return streamResponse(request, h, () => pipe( + ipfs.get(key), + async function * (source) { + for await (const file of source) { + const header = { + name: file.path + } - const streamFiles = async () => { - try { - for await (const file of ipfs.get(key)) { if (file.content) { - const content = await concat(file.content) - pack.entry({ name: file.path, size: file.size }, content.slice()) + yield { header: { ...header, size: file.size }, body: toBuffer(file.content) } } else { - pack.entry({ name: file.path, type: 'directory' }) + yield { header: { ...header, type: 'directory' } } } } - pack.finalize() - } catch (err) { - log.error(err) - pack.emit('error', err) - pack.destroy() - } - } - - streamFiles() - - // reply must be called right away so that tar-stream offloads its content - // otherwise it will block in large files - return h.response(pack).header('X-Stream-Output', '1') + }, + tar.pack(), + toBuffer + )) } } From bd3ad7174cf2dcff7fbf38ed83fa868fa6ae136f Mon Sep 17 00:00:00 2001 From: Alan Shaw Date: Sat, 8 Feb 2020 23:31:31 +0000 Subject: [PATCH 2/2] chore: update it-tar dep --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c054124eec..dc94d3f58b 100644 --- a/package.json +++ b/package.json @@ -123,7 +123,7 @@ "it-glob": "0.0.7", "it-last": "^1.0.1", "it-pipe": "^1.1.0", - "it-tar": "^1.2.0", + "it-tar": "^1.2.1", "it-to-stream": "^0.1.1", "iterable-ndjson": "^1.1.0", "jsondiffpatch": "~0.3.11",