Skip to content

Commit 70ddd00

Browse files
feat: download tars from @helia/verified-fetch (#442)
Adds support for downloading tar archives of UnixFS directories --------- Co-authored-by: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com>
1 parent 703980c commit 70ddd00

6 files changed

+239
-6
lines changed

package.json

+8-1
Original file line numberDiff line numberDiff line change
@@ -141,8 +141,8 @@
141141
"release": "aegir release"
142142
},
143143
"dependencies": {
144-
"@helia/car": "^3.0.0",
145144
"@helia/block-brokers": "^2.0.1",
145+
"@helia/car": "^3.0.0",
146146
"@helia/http": "^1.0.1",
147147
"@helia/interface": "^4.0.0",
148148
"@helia/ipns": "^6.0.0",
@@ -155,7 +155,11 @@
155155
"@libp2p/peer-id": "^4.0.5",
156156
"cborg": "^4.0.9",
157157
"hashlru": "^2.3.0",
158+
"interface-blockstore": "^5.2.10",
158159
"ipfs-unixfs-exporter": "^13.5.0",
160+
"it-map": "^3.0.5",
161+
"it-pipe": "^3.0.1",
162+
"it-tar": "^6.0.4",
159163
"it-to-browser-readablestream": "^2.0.6",
160164
"multiformats": "^13.1.0",
161165
"progress-events": "^1.0.0"
@@ -173,9 +177,12 @@
173177
"@types/sinon": "^17.0.3",
174178
"aegir": "^42.2.2",
175179
"blockstore-core": "^4.4.0",
180+
"browser-readablestream-to-it": "^2.0.5",
176181
"datastore-core": "^9.2.8",
177182
"helia": "^4.0.1",
183+
"ipfs-unixfs-importer": "^15.2.4",
178184
"ipns": "^9.0.0",
185+
"it-all": "^3.0.4",
179186
"it-last": "^3.0.4",
180187
"it-to-buffer": "^4.0.5",
181188
"magic-bytes.js": "^1.8.0",

src/utils/get-tar-stream.ts

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
import { CodeError } from '@libp2p/interface'
2+
import { exporter, recursive, type UnixFSEntry } from 'ipfs-unixfs-exporter'
3+
import map from 'it-map'
4+
import { pipe } from 'it-pipe'
5+
import { pack, type TarEntryHeader, type TarImportCandidate } from 'it-tar'
6+
import type { AbortOptions } from '@libp2p/interface'
7+
import type { Blockstore } from 'interface-blockstore'
8+
9+
const EXPORTABLE = ['file', 'raw', 'directory']
10+
11+
function toHeader (file: UnixFSEntry): Partial<TarEntryHeader> & { name: string } {
12+
let mode: number | undefined
13+
let mtime: Date | undefined
14+
15+
if (file.type === 'file' || file.type === 'directory') {
16+
mode = file.unixfs.mode
17+
mtime = file.unixfs.mtime != null ? new Date(Number(file.unixfs.mtime.secs * 1000n)) : undefined
18+
}
19+
20+
return {
21+
name: file.path,
22+
mode,
23+
mtime,
24+
size: Number(file.size),
25+
type: file.type === 'directory' ? 'directory' : 'file'
26+
}
27+
}
28+
29+
function toTarImportCandidate (entry: UnixFSEntry): TarImportCandidate {
30+
if (!EXPORTABLE.includes(entry.type)) {
31+
throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
32+
}
33+
34+
const candidate: TarImportCandidate = {
35+
header: toHeader(entry)
36+
}
37+
38+
if (entry.type === 'file' || entry.type === 'raw') {
39+
candidate.body = entry.content()
40+
}
41+
42+
return candidate
43+
}
44+
45+
export async function * tarStream (ipfsPath: string, blockstore: Blockstore, options?: AbortOptions): AsyncGenerator<Uint8Array> {
46+
const file = await exporter(ipfsPath, blockstore, options)
47+
48+
if (file.type === 'file' || file.type === 'raw') {
49+
yield * pipe(
50+
[toTarImportCandidate(file)],
51+
pack()
52+
)
53+
54+
return
55+
}
56+
57+
if (file.type === 'directory') {
58+
yield * pipe(
59+
recursive(ipfsPath, blockstore, options),
60+
(source) => map(source, (entry) => toTarImportCandidate(entry)),
61+
pack()
62+
)
63+
64+
return
65+
}
66+
67+
throw new CodeError('Not a UnixFS node', 'ERR_NOT_UNIXFS')
68+
}

src/utils/select-output-type.ts

+2-1
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ const CID_TYPE_MAP: Record<number, string[]> = {
5555
'application/octet-stream',
5656
'application/vnd.ipld.raw',
5757
'application/vnd.ipfs.ipns-record',
58-
'application/vnd.ipld.car'
58+
'application/vnd.ipld.car',
59+
'application/x-tar'
5960
]
6061
}
6162

src/verified-fetch.ts

+11-3
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import { dagCborToSafeJSON } from './utils/dag-cbor-to-safe-json.js'
1414
import { getContentDispositionFilename } from './utils/get-content-disposition-filename.js'
1515
import { getETag } from './utils/get-e-tag.js'
1616
import { getStreamFromAsyncIterable } from './utils/get-stream-from-async-iterable.js'
17+
import { tarStream } from './utils/get-tar-stream.js'
1718
import { parseResource } from './utils/parse-resource.js'
1819
import { notAcceptableResponse, notSupportedResponse, okResponse } from './utils/responses.js'
1920
import { selectOutputType, queryFormatToAcceptHeader } from './utils/select-output-type.js'
@@ -151,11 +152,16 @@ export class VerifiedFetch {
151152
* directory structure referenced by the `CID`.
152153
*/
153154
private async handleTar ({ cid, path, options }: FetchHandlerFunctionArg): Promise<Response> {
154-
if (cid.code !== dagPbCode) {
155-
return notAcceptableResponse('only dag-pb CIDs can be returned in TAR files')
155+
if (cid.code !== dagPbCode && cid.code !== rawCode) {
156+
return notAcceptableResponse('only UnixFS data can be returned in a TAR file')
156157
}
157158

158-
return notSupportedResponse('application/tar support is not implemented')
159+
const stream = toBrowserReadableStream<Uint8Array>(tarStream(`/ipfs/${cid}/${path}`, this.helia.blockstore, options))
160+
161+
const response = okResponse(stream)
162+
response.headers.set('content-type', 'application/x-tar')
163+
164+
return response
159165
}
160166

161167
private async handleJson ({ cid, path, accept, options }: FetchHandlerFunctionArg): Promise<Response> {
@@ -397,6 +403,8 @@ export class VerifiedFetch {
397403
} else if (accept === 'application/x-tar') {
398404
// the user requested a TAR file
399405
reqFormat = 'tar'
406+
query.download = true
407+
query.filename = query.filename ?? `${cid.toString()}.tar`
400408
response = await this.handleTar({ cid, path, options })
401409
} else {
402410
// derive the handler from the CID type

test/tar.spec.ts

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
import { unixfs } from '@helia/unixfs'
2+
import { stop } from '@libp2p/interface'
3+
import { expect } from 'aegir/chai'
4+
import browserReadableStreamToIt from 'browser-readablestream-to-it'
5+
import all from 'it-all'
6+
import last from 'it-last'
7+
import { pipe } from 'it-pipe'
8+
import { extract } from 'it-tar'
9+
import toBuffer from 'it-to-buffer'
10+
import { VerifiedFetch } from '../src/verified-fetch.js'
11+
import { createHelia } from './fixtures/create-offline-helia.js'
12+
import type { Helia } from '@helia/interface'
13+
import type { FileCandidate } from 'ipfs-unixfs-importer'
14+
15+
describe('tar files', () => {
16+
let helia: Helia
17+
let verifiedFetch: VerifiedFetch
18+
19+
beforeEach(async () => {
20+
helia = await createHelia()
21+
verifiedFetch = new VerifiedFetch({
22+
helia
23+
})
24+
})
25+
26+
afterEach(async () => {
27+
await stop(helia, verifiedFetch)
28+
})
29+
30+
it('should support fetching a TAR file', async () => {
31+
const file = Uint8Array.from([0, 1, 2, 3, 4])
32+
const fs = unixfs(helia)
33+
const cid = await fs.addBytes(file)
34+
35+
const resp = await verifiedFetch.fetch(cid, {
36+
headers: {
37+
accept: 'application/x-tar'
38+
}
39+
})
40+
expect(resp.status).to.equal(200)
41+
expect(resp.headers.get('content-type')).to.equal('application/x-tar')
42+
expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`)
43+
44+
if (resp.body == null) {
45+
throw new Error('Download failed')
46+
}
47+
48+
const entries = await pipe(
49+
browserReadableStreamToIt(resp.body),
50+
extract(),
51+
async source => all(source)
52+
)
53+
54+
expect(entries).to.have.lengthOf(1)
55+
await expect(toBuffer(entries[0].body)).to.eventually.deep.equal(file)
56+
})
57+
58+
it('should support fetching a TAR file containing a directory', async () => {
59+
const directory: FileCandidate[] = [{
60+
path: 'foo.txt',
61+
content: Uint8Array.from([0, 1, 2, 3, 4])
62+
}, {
63+
path: 'bar.txt',
64+
content: Uint8Array.from([5, 6, 7, 8, 9])
65+
}, {
66+
path: 'baz/qux.txt',
67+
content: Uint8Array.from([1, 2, 3, 4, 5])
68+
}]
69+
70+
const fs = unixfs(helia)
71+
const importResult = await last(fs.addAll(directory, {
72+
wrapWithDirectory: true
73+
}))
74+
75+
if (importResult == null) {
76+
throw new Error('Import failed')
77+
}
78+
79+
const resp = await verifiedFetch.fetch(importResult.cid, {
80+
headers: {
81+
accept: 'application/x-tar'
82+
}
83+
})
84+
expect(resp.status).to.equal(200)
85+
expect(resp.headers.get('content-type')).to.equal('application/x-tar')
86+
expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${importResult.cid.toString()}.tar"`)
87+
88+
if (resp.body == null) {
89+
throw new Error('Download failed')
90+
}
91+
92+
const entries = await pipe(
93+
browserReadableStreamToIt(resp.body),
94+
extract(),
95+
async source => all(source)
96+
)
97+
98+
expect(entries).to.have.lengthOf(5)
99+
expect(entries[0]).to.have.nested.property('header.name', importResult.cid.toString())
100+
101+
expect(entries[1]).to.have.nested.property('header.name', `${importResult.cid}/${directory[1].path}`)
102+
await expect(toBuffer(entries[1].body)).to.eventually.deep.equal(directory[1].content)
103+
104+
expect(entries[2]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path?.split('/')[0]}`)
105+
106+
expect(entries[3]).to.have.nested.property('header.name', `${importResult.cid}/${directory[2].path}`)
107+
await expect(toBuffer(entries[3].body)).to.eventually.deep.equal(directory[2].content)
108+
109+
expect(entries[4]).to.have.nested.property('header.name', `${importResult.cid}/${directory[0].path}`)
110+
await expect(toBuffer(entries[4].body)).to.eventually.deep.equal(directory[0].content)
111+
})
112+
113+
it('should support fetching a TAR file by format', async () => {
114+
const file = Uint8Array.from([0, 1, 2, 3, 4])
115+
const fs = unixfs(helia)
116+
const cid = await fs.addBytes(file)
117+
118+
const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar`)
119+
expect(resp.status).to.equal(200)
120+
expect(resp.headers.get('content-type')).to.equal('application/x-tar')
121+
expect(resp.headers.get('content-disposition')).to.equal(`attachment; filename="${cid.toString()}.tar"`)
122+
})
123+
124+
it('should support specifying a filename for a TAR file', async () => {
125+
const file = Uint8Array.from([0, 1, 2, 3, 4])
126+
const fs = unixfs(helia)
127+
const cid = await fs.addBytes(file)
128+
129+
const resp = await verifiedFetch.fetch(`ipfs://${cid}?filename=foo.bar`, {
130+
headers: {
131+
accept: 'application/x-tar'
132+
}
133+
})
134+
expect(resp.status).to.equal(200)
135+
expect(resp.headers.get('content-type')).to.equal('application/x-tar')
136+
expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"')
137+
})
138+
139+
it('should support fetching a TAR file by format with a filename', async () => {
140+
const file = Uint8Array.from([0, 1, 2, 3, 4])
141+
const fs = unixfs(helia)
142+
const cid = await fs.addBytes(file)
143+
144+
const resp = await verifiedFetch.fetch(`ipfs://${cid}?format=tar&filename=foo.bar`)
145+
expect(resp.status).to.equal(200)
146+
expect(resp.headers.get('content-type')).to.equal('application/x-tar')
147+
expect(resp.headers.get('content-disposition')).to.equal('attachment; filename="foo.bar"')
148+
})
149+
})

test/verified-fetch.spec.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ describe('@helia/verifed-fetch', () => {
8080
})
8181

8282
const formatsAndAcceptHeaders = [
83-
['tar', 'application/x-tar']
83+
['ipns-record', 'application/vnd.ipfs.ipns-record']
8484
]
8585

8686
for (const [format, acceptHeader] of formatsAndAcceptHeaders) {

0 commit comments

Comments
 (0)