diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index fe338e47cb..8756a2902f 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -7,6 +7,9 @@ _Explore the Mutable File System through interactive coding challenges in our [P - [The Regular API](#the-regular-api) - [`ipfs.add(data, [options])`](#ipfsadddata-options) - [Parameters](#parameters) + - [FileStream](#filestream) + - [FileObject](#fileobject) + - [FileContent](#filecontent) - [Options](#options) - [Returns](#returns) - [`ipfs.addAll(source, [options])`](#ipfsaddallsource-options) @@ -108,12 +111,19 @@ The regular, top-level API for add, cat, get and ls Files on IPFS `data` may be: -* `Blob` -* `String` -* `Uint8Array` +* `FileContent` (see below for definition) * `FileObject` (see below for definition) -* `Iterable` -* `AsyncIterable` +* `FileStream` (see below for definition) + +##### FileStream + +`FileStream` is a stream of `FileContent` or `FileObject` entries of the type: + +```js +Iterable | AsyncIterable | ReadableStream +``` + +##### FileObject `FileObject` is a plain JS object of the following form: @@ -136,10 +146,12 @@ If no `content` is passed, then the item is treated as an empty directory. One of `path` or `content` _must_ be passed. +##### FileContent + `FileContent` is one of the following types: ```js -Uint8Array | Blob | String | Iterable | AsyncIterable +Uint8Array | Blob | String | Iterable | AsyncIterable | ReadableStream ``` `UnixTime` is one of the following types: diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 3c791413bd..b08de20343 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -28,17 +28,21 @@ }, "license": "MIT", "dependencies": { + "blob-to-it": "0.0.1", + "browser-readablestream-to-it": "0.0.1", "buffer": "^5.6.0", "cids": "^0.8.3", "err-code": "^2.0.0", - "ipfs-utils": "^2.2.2" + "ipfs-utils": "^2.2.2", + "it-all": "^1.0.1", + "it-map": "^1.0.0", + "it-peekable": "0.0.1" }, "devDependencies": { "aegir": "^23.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.3.0", - "dirty-chai": "^2.0.1", - "it-all": "^1.0.1" + "dirty-chai": "^2.0.1" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input.js deleted file mode 100644 index c3f73168c5..0000000000 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ /dev/null @@ -1,298 +0,0 @@ -'use strict' - -const errCode = require('err-code') -const { Buffer } = require('buffer') -const globalThis = require('ipfs-utils/src/globalthis') - -/* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * Into: - * - * ``` - * AsyncIterable<{ path, content: AsyncIterable }> - * ``` - * - * @param input Object - * @return AsyncInterable<{ path, content: AsyncIterable }> - */ -module.exports = function normaliseInput (input) { - // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') - } - - // String - if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })()) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj) - } - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = toAsyncIterable(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = toAsyncIterable(input) - } - - return obj -} - -function toAsyncIterable (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() - } - - // Bloby - if (isBloby(input)) { - return blobToAsyncGenerator(input) - } - - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToIt(input) - } - - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return - } - - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() - } - - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) -} - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} - -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new globalThis.FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js new file mode 100644 index 0000000000..c96260b528 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js @@ -0,0 +1,18 @@ +'use strict' + +const normaliseContent = require('./normalise-content.browser') +const normaliseInput = require('./normalise-input') + +/* + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: Blob }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param input Object + * @return AsyncInterable<{ path, mode, mtime, content: Blob }> + */ +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.js b/packages/ipfs-core-utils/src/files/normalise-input/index.js new file mode 100644 index 0000000000..00dd946bfa --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.js @@ -0,0 +1,18 @@ +'use strict' + +const normaliseContent = require('./normalise-content') +const normaliseInput = require('./normalise-input') + +/* + * Transforms any of the `ipfs.add` input types into + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> + * ``` + * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * + * @param input Object + * @return AsyncInterable<{ path, mode, mtime, content: AsyncIterable }> + */ +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js new file mode 100644 index 0000000000..fe55cde87c --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -0,0 +1,65 @@ +'use strict' + +const errCode = require('err-code') +const { Blob } = require('ipfs-utils/src/globalthis') +const itPeekable = require('it-peekable') +const browserStreamToIt = require('browser-readablestream-to-it') + +const { + isBytes, + isBlob +} = require('./utils') + +async function toBlob (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string' || input instanceof String) { + return new Blob([input]) + } + + // Blob | File + if (isBlob(input)) { + return input + } + + // Browser stream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + return itToBlob(peekable) + } + + peekable.push(value) + + // (Async)Iterable + if (Number.isInteger(value)) { + return itToBlob(peekable) + } + + // (Async)Iterable + if (isBytes(value) || typeof value === 'string' || value instanceof String) { + return itToBlob(peekable) + } + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +async function itToBlob (stream) { + const parts = [] + + for await (const chunk of stream) { + parts.push(chunk) + } + + return new Blob(parts) +} + +module.exports = toBlob diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js new file mode 100644 index 0000000000..c55f938e0f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -0,0 +1,66 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const browserStreamToIt = require('browser-readablestream-to-it') +const blobToIt = require('blob-to-it') +const itPeekable = require('it-peekable') +const all = require('it-all') +const map = require('it-map') +const { + isBytes, + isBlob +} = require('./utils') + +async function * toAsyncIterable (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string' || input instanceof String) { + yield toBuffer(input) + return + } + + // Blob + if (isBlob(input)) { + yield * blobToIt(input) + return + } + + // Browser stream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } + + peekable.push(value) + + // (Async)Iterable + if (Number.isInteger(value)) { + yield toBuffer(await all(peekable)) + return + } + + // (Async)Iterable + if (isBytes(value) || typeof value === 'string' || value instanceof String) { + yield * map(peekable, chunk => toBuffer(chunk)) + return + } + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +function toBuffer (chunk) { + return isBytes(chunk) ? chunk : Buffer.from(chunk) +} + +module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js new file mode 100644 index 0000000000..90bb00015f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -0,0 +1,100 @@ +'use strict' + +const errCode = require('err-code') +const browserStreamToIt = require('browser-readablestream-to-it') +const itPeekable = require('it-peekable') +const map = require('it-map') +const { + isBytes, + isBlob, + isFileObject +} = require('./utils') + +module.exports = async function * normaliseInput (input, normaliseContent) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') + } + + // String + if (typeof input === 'string' || input instanceof String) { + yield toFileObject(input, normaliseContent) + return + } + + // Buffer|ArrayBuffer|TypedArray + // Blob|File + if (isBytes(input) || isBlob(input)) { + yield toFileObject(input, normaliseContent) + return + } + + // Browser ReadableStream + if (typeof input.getReader === 'function') { + input = browserStreamToIt(input) + } + + // Iterable + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value)) { + yield toFileObject(peekable, normaliseContent) + return + } + + // (Async)Iterable + // (Async)Iterable + // (Async)Iterable<{ path, content }> + if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } + + // (Async)Iterable<(Async)Iterable> + // (Async)Iterable> + // ReadableStream<(Async)Iterable> + // ReadableStream> + if (value[Symbol.iterator] || value[Symbol.asyncIterator] || typeof value.getReader === 'function') { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } + } + + // { path, content: ? } + // Note: Detected _after_ (Async)Iterable because Node.js streams have a + // `path` property that passes this check. + if (isFileObject(input)) { + yield toFileObject(input, normaliseContent) + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +async function toFileObject (input, normaliseContent) { + const obj = { + path: input.path || '', + mode: input.mode, + mtime: input.mtime + } + + if (input.content) { + obj.content = await normaliseContent(input.content) + } else if (!input.path) { // Not already a file object with path or content prop + obj.content = await normaliseContent(input) + } + + return obj +} diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js new file mode 100644 index 0000000000..c09f0c8241 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -0,0 +1,22 @@ +'use strict' + +const { Blob } = require('ipfs-utils/src/globalthis') + +function isBytes (obj) { + return ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBlob (obj) { + return typeof Blob !== 'undefined' && obj instanceof Blob +} + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} + +module.exports = { + isBytes, + isBlob, + isFileObject +} diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index e8491bd768..7e206c2bd3 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -2,37 +2,44 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') -const normalise = require('../../src/files/normalise-input') -const { supportsFileReader } = require('ipfs-utils/src/supports') +const blobToIt = require('blob-to-it') const { Buffer } = require('buffer') const all = require('it-all') -const globalThis = require('ipfs-utils/src/globalthis') +const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') +const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') + +let normalise = require('../../src/files/normalise-input') + +if (isBrowser || isWebWorker) { + normalise = require('../../src/files/normalise-input/index.browser') +} const STRING = () => 'hello world' +const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers const BUFFER = () => Buffer.from(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) let BLOB -let WINDOW_READABLE_STREAM -if (supportsFileReader) { - BLOB = () => new globalThis.Blob([ +if (Blob) { + BLOB = () => new Blob([ STRING() ]) - - WINDOW_READABLE_STREAM = () => new globalThis.ReadableStream({ - start (controller) { - controller.enqueue(BUFFER()) - controller.close() - } - }) } async function verifyNormalisation (input) { expect(input.length).to.equal(1) - expect(input[0].content[Symbol.asyncIterator] || input[0].content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') - expect(await all(input[0].content)).to.deep.equal([BUFFER()]) expect(input[0].path).to.equal('') + + let content = input[0].content + + if (isBrowser || isWebWorker) { + expect(content).to.be.an.instanceOf(Blob) + content = blobToIt(input[0].content) + } + + expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) } async function testContent (input) { @@ -51,6 +58,15 @@ function asyncIterableOf (thing) { }()) } +function browserReadableStreamOf (thing) { + return new ReadableStream({ + start (controller) { + controller.enqueue(thing) + controller.close() + } + }) +} + describe('normalise-input', function () { function testInputType (content, name, isBytes) { it(name, async function () { @@ -58,6 +74,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<${name}>`, async function () { await testContent(iterableOf(content())) }) @@ -72,6 +94,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`{ path: '', content: ReadableStream<${name}> }`, async function () { + await testContent({ path: '', content: browserReadableStreamOf(content()) }) + }) + } + it(`{ path: '', content: Iterable<${name}> }`, async function () { await testContent({ path: '', content: iterableOf(content()) }) }) @@ -81,6 +109,12 @@ describe('normalise-input', function () { }) } + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<{ path: '', content: ${name} }`, async function () { await testContent(iterableOf({ path: '', content: content() })) }) @@ -90,6 +124,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(iterableOf({ path: '', content: iterableOf(content()) })) }) @@ -98,6 +138,12 @@ describe('normalise-input', function () { await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) }) + if (ReadableStream) { + it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) }) @@ -109,7 +155,8 @@ describe('normalise-input', function () { } describe('String', () => { - testInputType(STRING, 'String', false) + testInputType(STRING, 'String', true) + testInputType(NEWSTRING, 'new String()', true) }) describe('Buffer', () => { @@ -117,21 +164,13 @@ describe('normalise-input', function () { }) describe('Blob', () => { - if (!supportsFileReader) { + if (!Blob) { return } testInputType(BLOB, 'Blob', false) }) - describe('window.ReadableStream', () => { - if (!supportsFileReader) { - return - } - - testInputType(WINDOW_READABLE_STREAM, 'window.ReadableStream', false) - }) - describe('Iterable', () => { testInputType(ARRAY, 'Iterable', false) }) diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 6b768851bc..eb6d4ed75b 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -15,9 +15,10 @@ ], "main": "src/index.js", "browser": { - "./src/lib/to-stream.js": "./src/lib/to-stream.browser.js", + "./src/lib/multipart-request.js": "./src/lib/multipart-request.browser.js", "ipfs-utils/src/files/glob-source": false, - "go-ipfs": false + "go-ipfs": false, + "ipfs-core-utils/src/files/normalise-input": "ipfs-core-utils/src/files/normalise-input/index.browser.js" }, "repository": { "type": "git", diff --git a/packages/ipfs-http-client/src/lib/multipart-request.browser.js b/packages/ipfs-http-client/src/lib/multipart-request.browser.js new file mode 100644 index 0000000000..550c85ba30 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/multipart-request.browser.js @@ -0,0 +1,58 @@ +'use strict' + +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') +const modeToString = require('./mode-to-string') +const mtimeToObject = require('./mtime-to-object') +const { File, FormData } = require('ipfs-utils/src/globalthis') + +async function multipartRequest (source = '', abortController, headers = {}) { + const formData = new FormData() + let index = 0 + + for await (const { content, path, mode, mtime } of normaliseInput(source)) { + let fileSuffix = '' + const type = content ? 'file' : 'dir' + + if (index > 0) { + fileSuffix = `-${index}` + } + + let fieldName = type + fileSuffix + const qs = [] + + if (mode !== null && mode !== undefined) { + qs.push(`mode=${modeToString(mode)}`) + } + + if (mtime != null) { + const { + secs, nsecs + } = mtimeToObject(mtime) + + qs.push(`mtime=${secs}`) + + if (nsecs != null) { + qs.push(`mtime-nsecs=${nsecs}`) + } + } + + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + if (content) { + formData.set(fieldName, content, encodeURIComponent(path)) + } else { + formData.set(fieldName, new File([''], encodeURIComponent(path), { type: 'application/x-directory' })) + } + + index++ + } + + return { + headers, + body: formData + } +} + +module.exports = multipartRequest diff --git a/packages/ipfs-http-client/src/lib/multipart-request.js b/packages/ipfs-http-client/src/lib/multipart-request.js index eee4e26b1b..ba712e02ef 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.js @@ -1,11 +1,11 @@ 'use strict' const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') -const toStream = require('./to-stream') const { nanoid } = require('nanoid') const modeToString = require('../lib/mode-to-string') const mtimeToObject = require('../lib/mtime-to-object') const merge = require('merge-options').bind({ ignoreUndefined: true }) +const toStream = require('it-to-stream') async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { async function * streamFiles (source) { @@ -22,12 +22,11 @@ async function multipartRequest (source = '', abortController, headers = {}, bou fileSuffix = `-${index}` } - yield `--${boundary}\r\n` - yield `Content-Disposition: form-data; name="${type}${fileSuffix}"; filename="${encodeURIComponent(path)}"\r\n` - yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` + let fieldName = type + fileSuffix + const qs = [] if (mode !== null && mode !== undefined) { - yield `mode: ${modeToString(mode)}\r\n` + qs.push(`mode=${modeToString(mode)}`) } if (mtime != null) { @@ -35,13 +34,20 @@ async function multipartRequest (source = '', abortController, headers = {}, bou secs, nsecs } = mtimeToObject(mtime) - yield `mtime: ${secs}\r\n` + qs.push(`mtime=${secs}`) if (nsecs != null) { - yield `mtime-nsecs: ${nsecs}\r\n` + qs.push(`mtime-nsecs=${nsecs}`) } } + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + yield `--${boundary}\r\n` + yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` + yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` yield '\r\n' if (content) { diff --git a/packages/ipfs-http-client/src/lib/to-stream.browser.js b/packages/ipfs-http-client/src/lib/to-stream.browser.js deleted file mode 100644 index 9f5784fedb..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.browser.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -// browsers can't stream. When the 'Send ReadableStream in request body' row -// is green here: https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Browser_compatibility -// we'll be able to wrap the passed iterator in the it-to-browser-readablestream module -// in the meantime we have to convert the whole thing to a BufferSource of some sort -const toBuffer = require('it-to-buffer') -const { Buffer } = require('buffer') - -module.exports = (it) => { - async function * bufferise (source) { - for await (const chunk of source) { - if (Buffer.isBuffer(chunk)) { - yield chunk - } else { - yield Buffer.from(chunk) - } - } - } - - return toBuffer(bufferise(it)) -} diff --git a/packages/ipfs-http-client/src/lib/to-stream.js b/packages/ipfs-http-client/src/lib/to-stream.js deleted file mode 100644 index f0f59ffc50..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const toStream = require('it-to-stream') - -module.exports = (it) => { - return toStream.readable(it) -} diff --git a/packages/ipfs-http-client/test/files.spec.js b/packages/ipfs-http-client/test/files.spec.js new file mode 100644 index 0000000000..9ad1be1ff7 --- /dev/null +++ b/packages/ipfs-http-client/test/files.spec.js @@ -0,0 +1,38 @@ +/* eslint-env mocha */ + +'use strict' + +const { Buffer } = require('buffer') +const { expect } = require('interface-ipfs-core/src/utils/mocha') +const f = require('./utils/factory')() + +describe('.add', function () { + this.timeout(20 * 1000) + + let ipfs + + before(async function () { + ipfs = (await f.spawn()).api + }) + + after(() => f.clean()) + + it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => { + const data = Buffer.from('some data') + const result = await ipfs.add(data, { + mode: 0o600, + mtime: { + secs: 1000, + nsecs: 0 + } + }) + + expect(result).to.not.have.property('mode') + expect(result).to.not.have.property('mtime') + expect(result).to.have.property('cid') + + const { cid } = result + expect(cid).to.have.property('codec', 'dag-pb') + expect(cid.toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + }) +}) diff --git a/packages/ipfs/src/http/utils/multipart-request-parser.js b/packages/ipfs/src/http/utils/multipart-request-parser.js index 6ee29d52bc..84ea04be69 100644 --- a/packages/ipfs/src/http/utils/multipart-request-parser.js +++ b/packages/ipfs/src/http/utils/multipart-request-parser.js @@ -3,6 +3,7 @@ const Content = require('@hapi/content') const multipart = require('it-multipart') const { Buffer } = require('buffer') +const qs = require('querystring') const multipartFormdataType = 'multipart/form-data' const applicationDirectory = 'application/x-directory' @@ -69,20 +70,6 @@ async function * parseEntry (stream, options) { const entry = {} - if (part.headers.mtime) { - entry.mtime = { - secs: parseInt(part.headers.mtime, 10) - } - - if (part.headers['mtime-nsecs']) { - entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10) - } - } - - if (part.headers.mode) { - entry.mode = parseInt(part.headers.mode, 8) - } - if (isDirectory(type.mime)) { entry.type = 'directory' } else if (type.mime === applicationSymlink) { @@ -92,6 +79,21 @@ async function * parseEntry (stream, options) { } const disposition = parseDisposition(part.headers['content-disposition']) + const query = qs.parse(disposition.name.split('?').pop()) + + if (query.mode) { + entry.mode = parseInt(query.mode, 8) + } + + if (query.mtime) { + entry.mtime = { + secs: parseInt(query.mtime, 10) + } + + if (query['mtime-nsecs']) { + entry.mtime.nsecs = parseInt(query['mtime-nsecs'], 10) + } + } entry.name = decodeURIComponent(disposition.filename) entry.body = part.body diff --git a/packages/ipfs/test/http-api/inject/mfs/write.js b/packages/ipfs/test/http-api/inject/mfs/write.js index 5a8eca4d38..4d466a5187 100644 --- a/packages/ipfs/test/http-api/inject/mfs/write.js +++ b/packages/ipfs/test/http-api/inject/mfs/write.js @@ -29,11 +29,28 @@ const defaultOptions = { signal: sinon.match.instanceOf(AbortSignal) } -async function send (text, headers = {}) { +async function send (text, options = {}) { + let fieldName = 'file-0' + const query = [] + + if (options.mode) { + query.push(`mode=${options.mode}`) + } + + if (options.mtime) { + query.push(`mtime=${options.mtime}`) + } + + if (options.mtimeNsecs) { + query.push(`mtime-nsecs=${options.mtimeNsecs}`) + } + + if (query.length) { + fieldName = `${fieldName}?${query.join('&')}` + } + const form = new FormData() - form.append('file-0', Buffer.from(text), { - header: headers - }) + form.append(fieldName, Buffer.from(text)) return { headers: form.getHeaders(),