Skip to content

Commit e51ff72

Browse files
achingbrainachingbrain
committed
feat: consolidate ipfs.add input normalisation
Allows input normalisation function to be shared between ipfs and the http client.
1 parent a864dda commit e51ff72

File tree

3 files changed

+374
-1
lines changed

3 files changed

+374
-1
lines changed

‎package.json

+3-1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
"license": "MIT",
2828
"dependencies": {
2929
"buffer": "^5.2.1",
30+
"err-code": "^2.0.0",
3031
"is-buffer": "^2.0.3",
3132
"is-electron": "^2.2.0",
3233
"is-pull-stream": "0.0.0",
@@ -36,9 +37,10 @@
3637
},
3738
"devDependencies": {
3839
"aegir": "^20.0.0",
40+
"async-iterator-all": "^1.0.0",
3941
"chai": "^4.2.0",
4042
"dirty-chai": "^2.0.1",
41-
"electron": "^5.0.7",
43+
"electron": "^6.0.6",
4244
"electron-mocha": "^8.0.3",
4345
"pull-stream": "^3.6.13"
4446
},

‎src/files/normalise-input.js

+233
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,233 @@
1+
'use strict'
2+
3+
const errCode = require('err-code')
4+
const { Buffer } = require('buffer')
5+
6+
/*
7+
* Transform one of:
8+
*
9+
* ```
10+
* Buffer|ArrayBuffer|TypedArray
11+
* Blob|File
12+
* { path, content: Blob }
13+
* { path, content: String }
14+
* { path, content: Iterable<Number> }
15+
* { path, content: Iterable<Buffer> }
16+
* { path, content: Iterable<Iterable<Number>> }
17+
* { path, content: AsyncIterable<Iterable<Number>> }
18+
* String
19+
* Iterable<Number>
20+
* Iterable<Buffer>
21+
* Iterable<Blob>
22+
* Iterable<{ path, content: Buffer }>
23+
* Iterable<{ path, content: Blob }>
24+
* Iterable<{ path, content: Iterable<Number> }>
25+
* Iterable<{ path, content: AsyncIterable<Buffer> }>
26+
* AsyncIterable<Buffer>
27+
* AsyncIterable<{ path, content: Buffer }>
28+
* AsyncIterable<{ path, content: Blob }>
29+
* AsyncIterable<{ path, content: Iterable<Buffer> }>
30+
* AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
31+
* ```
32+
* Into:
33+
*
34+
* ```
35+
* AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
36+
* ```
37+
*
38+
* @param input Object
39+
* @return AsyncInterable<{ path, content: AsyncIterable<Buffer> }>
40+
*/
41+
module.exports = function normaliseInput (input) {
42+
// must give us something
43+
if (input === null || input === undefined) {
44+
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
45+
}
46+
47+
// { path, content: ? }
48+
if (isFileObject(input)) {
49+
return (async function * () { // eslint-disable-line require-await
50+
yield toFileObject(input)
51+
})()
52+
}
53+
54+
// String
55+
if (typeof input === 'string' || input instanceof String) {
56+
return (async function * () { // eslint-disable-line require-await
57+
yield toFileObject(input)
58+
})()
59+
}
60+
61+
// Buffer|ArrayBuffer|TypedArray
62+
// Blob|File
63+
if (isBytes(input) || isBloby(input)) {
64+
return (async function * () { // eslint-disable-line require-await
65+
yield toFileObject(input)
66+
})()
67+
}
68+
69+
// Iterable<?>
70+
if (input[Symbol.iterator]) {
71+
// Iterable<Number>
72+
if (!isNaN(input[0])) {
73+
return (async function * () { // eslint-disable-line require-await
74+
yield toFileObject([input])
75+
})()
76+
}
77+
78+
// Iterable<Buffer>
79+
// Iterable<Blob>
80+
return (async function * () { // eslint-disable-line require-await
81+
for (const chunk of input) {
82+
yield toFileObject(chunk)
83+
}
84+
})()
85+
}
86+
87+
// AsyncIterable<?>
88+
if (input[Symbol.asyncIterator]) {
89+
return (async function * () { // eslint-disable-line require-await
90+
for await (const chunk of input) {
91+
yield toFileObject(chunk)
92+
}
93+
})()
94+
}
95+
96+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
97+
}
98+
99+
function toFileObject (input) {
100+
return {
101+
path: input.path || '',
102+
content: toAsyncIterable(input.content || input)
103+
}
104+
}
105+
106+
function toAsyncIterable (input) {
107+
// Buffer|ArrayBuffer|TypedArray|array of bytes
108+
if (isBytes(input)) {
109+
return (async function * () { // eslint-disable-line require-await
110+
yield toBuffer(input)
111+
})()
112+
}
113+
114+
if (typeof input === 'string' || input instanceof String) {
115+
return (async function * () { // eslint-disable-line require-await
116+
yield toBuffer(input)
117+
})()
118+
}
119+
120+
// Blob|File
121+
if (isBloby(input)) {
122+
return blobToAsyncGenerator(input)
123+
}
124+
125+
// Iterator<?>
126+
if (input[Symbol.iterator]) {
127+
if (!isNaN(input[0])) {
128+
return (async function * () { // eslint-disable-line require-await
129+
yield toBuffer(input)
130+
})()
131+
}
132+
133+
return (async function * () { // eslint-disable-line require-await
134+
for (const chunk of input) {
135+
yield toBuffer(chunk)
136+
}
137+
})()
138+
}
139+
140+
// AsyncIterable<?>
141+
if (input[Symbol.asyncIterator]) {
142+
return (async function * () {
143+
for await (const chunk of input) {
144+
yield toBuffer(chunk)
145+
}
146+
})()
147+
}
148+
149+
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
150+
}
151+
152+
function toBuffer (chunk) {
153+
if (isBytes(chunk)) {
154+
return chunk
155+
}
156+
157+
if (typeof chunk === 'string' || chunk instanceof String) {
158+
return Buffer.from(chunk)
159+
}
160+
161+
if (Array.isArray(chunk)) {
162+
return Buffer.from(chunk)
163+
}
164+
165+
throw new Error('Unexpected input: ' + typeof chunk)
166+
}
167+
168+
function isBytes (obj) {
169+
return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer
170+
}
171+
172+
function isBloby (obj) {
173+
return typeof Blob !== 'undefined' && obj instanceof global.Blob
174+
}
175+
176+
// An object with a path or content property
177+
function isFileObject (obj) {
178+
return typeof obj === 'object' && (obj.path || obj.content)
179+
}
180+
181+
function blobToAsyncGenerator (blob) {
182+
if (typeof blob.stream === 'function') {
183+
// firefox < 69 does not support blob.stream()
184+
return streamBlob(blob)
185+
}
186+
187+
return readBlob(blob)
188+
}
189+
190+
async function * streamBlob (blob) {
191+
const reader = blob.stream().getReader()
192+
193+
while (true) {
194+
const result = await reader.read()
195+
196+
if (result.done) {
197+
return
198+
}
199+
200+
yield result.value
201+
}
202+
}
203+
204+
async function * readBlob (blob, options) {
205+
options = options || {}
206+
207+
const reader = new global.FileReader()
208+
const chunkSize = options.chunkSize || 1024 * 1024
209+
let offset = options.offset || 0
210+
211+
const getNextChunk = () => new Promise((resolve, reject) => {
212+
reader.onloadend = e => {
213+
const data = e.target.result
214+
resolve(data.byteLength === 0 ? null : data)
215+
}
216+
reader.onerror = reject
217+
218+
const end = offset + chunkSize
219+
const slice = blob.slice(offset, end)
220+
reader.readAsArrayBuffer(slice)
221+
offset = end
222+
})
223+
224+
while (true) {
225+
const data = await getNextChunk()
226+
227+
if (data == null) {
228+
return
229+
}
230+
231+
yield Buffer.from(data)
232+
}
233+
}

‎test/files/normalise-input.spec.js

+138
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,138 @@
1+
'use strict'
2+
3+
/* eslint-env mocha */
4+
const chai = require('chai')
5+
const dirtyChai = require('dirty-chai')
6+
const normalise = require('../../src/files/normalise-input')
7+
const { supportsFileReader } = require('../../src/supports')
8+
const { Buffer } = require('buffer')
9+
const all = require('async-iterator-all')
10+
11+
chai.use(dirtyChai)
12+
const expect = chai.expect
13+
14+
const STRING = 'hello world'
15+
const BUFFER = Buffer.from(STRING)
16+
const ARRAY = Array.from(BUFFER)
17+
let BLOB
18+
19+
if (supportsFileReader) {
20+
BLOB = new global.Blob([
21+
STRING
22+
])
23+
}
24+
25+
async function verifyNormalisation (input) {
26+
expect(input.length).to.equal(1)
27+
28+
if (!input[0].content[Symbol.asyncIterator] && !input[0].content[Symbol.iterator]) {
29+
chai.assert.fail(`Content should have been an iterable or an async iterable`)
30+
}
31+
32+
expect(await all(input[0].content)).to.deep.equal([BUFFER])
33+
expect(input[0].path).to.equal('')
34+
}
35+
36+
async function testContent (input) {
37+
const result = await all(normalise(input))
38+
39+
await verifyNormalisation(result)
40+
}
41+
42+
function iterableOf (thing) {
43+
return [thing]
44+
}
45+
46+
function asyncIterableOf (thing) {
47+
return (async function * () { // eslint-disable-line require-await
48+
yield thing
49+
}())
50+
}
51+
52+
describe('normalise-input', function () {
53+
function testInputType (content, name) {
54+
it(name, async function () {
55+
await testContent(content)
56+
})
57+
58+
it(`Iterable<${name}>`, async function () {
59+
await testContent(iterableOf(content))
60+
})
61+
62+
it(`AsyncIterable<${name}>`, async function () {
63+
await testContent(asyncIterableOf(content))
64+
})
65+
66+
if (name !== 'Blob') {
67+
it(`AsyncIterable<Iterable<${name}>>`, async function () {
68+
await testContent(asyncIterableOf(iterableOf(content)))
69+
})
70+
71+
it(`AsyncIterable<AsyncIterable<${name}>>`, async function () {
72+
await testContent(asyncIterableOf(asyncIterableOf(content)))
73+
})
74+
}
75+
76+
it(`{ path: '', content: ${name} }`, async function () {
77+
await testContent({ path: '', content })
78+
})
79+
80+
if (name !== 'Blob') {
81+
it(`{ path: '', content: Iterable<${name}> }`, async function () {
82+
await testContent({ path: '', content: iterableOf(content) })
83+
})
84+
85+
it(`{ path: '', content: AsyncIterable<${name}> }`, async function () {
86+
await testContent({ path: '', content: asyncIterableOf(content) })
87+
})
88+
}
89+
90+
it(`Iterable<{ path: '', content: ${name} }`, async function () {
91+
await testContent(iterableOf({ path: '', content }))
92+
})
93+
94+
if (name !== 'Blob') {
95+
it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () {
96+
await testContent(iterableOf({ path: '', content: iterableOf(content) }))
97+
})
98+
99+
it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
100+
await testContent(iterableOf({ path: '', content: asyncIterableOf(content) }))
101+
})
102+
}
103+
104+
it(`AsyncIterable<{ path: '', content: ${name} }`, async function () {
105+
await testContent(asyncIterableOf({ path: '', content }))
106+
})
107+
108+
if (name !== 'Blob') {
109+
it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () {
110+
await testContent(asyncIterableOf({ path: '', content: iterableOf(content) }))
111+
})
112+
113+
it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () {
114+
await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content) }))
115+
})
116+
}
117+
}
118+
119+
describe('String', () => {
120+
testInputType(STRING, 'String')
121+
})
122+
123+
describe('Buffer', () => {
124+
testInputType(BUFFER, 'Buffer')
125+
})
126+
127+
describe('Blob', () => {
128+
if (!supportsFileReader) {
129+
return
130+
}
131+
132+
testInputType(BLOB, 'Blob')
133+
})
134+
135+
describe('Iterable<Number>', () => {
136+
testInputType(ARRAY, 'Iterable<Number>')
137+
})
138+
})

0 commit comments

Comments
 (0)