From e4a1684006ae95a153044b7cdad43cd39ea08b3c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 16:27:29 +0100 Subject: [PATCH 01/15] fix: normalise add input to blob in the browser To support streaming of native types with no buffering, normalise add input to async iterators in node and Blobs in the browser. That is, if the user passes a blob in the browser leave it alone as enumerating blob contents cause the files to be read. Browser FormData objects do not allow you to specify headers for each multipart part which means we can't pass unixfs metadata via the headers so we turn the metadata into a querystring and append it to the field name for each multipart part as a workaround. BREAKING CHANGES: - Removes the `mode`, `mtime` and `mtime-nsec` headers from multipart requests - Passes `mode`, `mtime` and `mtime-nsec` as querystring parameters appended to the field name of multipart requests --- packages/ipfs-http-client/package.json | 2 +- .../src/lib/multipart-request.browser.js | 58 +++++ .../src/lib/multipart-request.js | 20 +- .../src/lib/normalise-input.browser.js | 233 ++++++++++++++++++ .../src/lib/to-stream.browser.js | 22 -- .../ipfs-http-client/src/lib/to-stream.js | 7 - .../http/utils/multipart-request-parser.js | 30 +-- 7 files changed, 321 insertions(+), 51 deletions(-) create mode 100644 packages/ipfs-http-client/src/lib/multipart-request.browser.js create mode 100644 packages/ipfs-http-client/src/lib/normalise-input.browser.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.browser.js delete mode 100644 packages/ipfs-http-client/src/lib/to-stream.js diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 6b768851bc..3ac8b9fe0a 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -15,7 +15,7 @@ ], "main": "src/index.js", "browser": { - "./src/lib/to-stream.js": "./src/lib/to-stream.browser.js", + "./src/lib/multipart-request.js": "./src/lib/multipart-request.browser.js", "ipfs-utils/src/files/glob-source": false, "go-ipfs": false }, diff --git a/packages/ipfs-http-client/src/lib/multipart-request.browser.js b/packages/ipfs-http-client/src/lib/multipart-request.browser.js new file mode 100644 index 0000000000..8bf9e35af6 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/multipart-request.browser.js @@ -0,0 +1,58 @@ +'use strict' + +const normaliseInput = require('./normalise-input.browser') +const modeToString = require('./mode-to-string') +const mtimeToObject = require('./mtime-to-object') +const { File, FormData } = require('ipfs-utils/src/globalthis') + +async function multipartRequest (source = '', abortController, headers = {}) { + const formData = new FormData() + let index = 0 + + for await (const { content, path, mode, mtime } of normaliseInput(source)) { + let fileSuffix = '' + const type = content ? 'file' : 'dir' + + if (index > 0) { + fileSuffix = `-${index}` + } + + let fieldName = type + fileSuffix + const qs = [] + + if (mode !== null && mode !== undefined) { + qs.push(`mode=${modeToString(mode)}`) + } + + if (mtime != null) { + const { + secs, nsecs + } = mtimeToObject(mtime) + + qs.push(`mtime=${secs}`) + + if (nsecs != null) { + qs.push(`mtime-nsecs=${nsecs}`) + } + } + + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + if (content) { + formData.set(fieldName, content, encodeURIComponent(path)) + } else { + formData.set(fieldName, new File([''], encodeURIComponent(path), { type: 'application/x-directory' })) + } + + index++ + } + + return { + headers, + body: formData + } +} + +module.exports = multipartRequest diff --git a/packages/ipfs-http-client/src/lib/multipart-request.js b/packages/ipfs-http-client/src/lib/multipart-request.js index eee4e26b1b..ba712e02ef 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.js @@ -1,11 +1,11 @@ 'use strict' const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') -const toStream = require('./to-stream') const { nanoid } = require('nanoid') const modeToString = require('../lib/mode-to-string') const mtimeToObject = require('../lib/mtime-to-object') const merge = require('merge-options').bind({ ignoreUndefined: true }) +const toStream = require('it-to-stream') async function multipartRequest (source = '', abortController, headers = {}, boundary = `-----------------------------${nanoid()}`) { async function * streamFiles (source) { @@ -22,12 +22,11 @@ async function multipartRequest (source = '', abortController, headers = {}, bou fileSuffix = `-${index}` } - yield `--${boundary}\r\n` - yield `Content-Disposition: form-data; name="${type}${fileSuffix}"; filename="${encodeURIComponent(path)}"\r\n` - yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` + let fieldName = type + fileSuffix + const qs = [] if (mode !== null && mode !== undefined) { - yield `mode: ${modeToString(mode)}\r\n` + qs.push(`mode=${modeToString(mode)}`) } if (mtime != null) { @@ -35,13 +34,20 @@ async function multipartRequest (source = '', abortController, headers = {}, bou secs, nsecs } = mtimeToObject(mtime) - yield `mtime: ${secs}\r\n` + qs.push(`mtime=${secs}`) if (nsecs != null) { - yield `mtime-nsecs: ${nsecs}\r\n` + qs.push(`mtime-nsecs=${nsecs}`) } } + if (qs.length) { + fieldName = `${fieldName}?${qs.join('&')}` + } + + yield `--${boundary}\r\n` + yield `Content-Disposition: form-data; name="${fieldName}"; filename="${encodeURIComponent(path)}"\r\n` + yield `Content-Type: ${content ? 'application/octet-stream' : 'application/x-directory'}\r\n` yield '\r\n' if (content) { diff --git a/packages/ipfs-http-client/src/lib/normalise-input.browser.js b/packages/ipfs-http-client/src/lib/normalise-input.browser.js new file mode 100644 index 0000000000..d7a9eb8cb4 --- /dev/null +++ b/packages/ipfs-http-client/src/lib/normalise-input.browser.js @@ -0,0 +1,233 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const { Blob } = require('ipfs-utils/src/globalthis') + +/* + * Transform one of: + * + * ``` + * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] + * Bloby (Blob|File) [single file] + * String [single file] + * { path, content: Bytes } [single file] + * { path, content: Bloby } [single file] + * { path, content: String } [single file] + * { path, content: Iterable } [single file] + * { path, content: Iterable } [single file] + * { path, content: AsyncIterable } [single file] + * Iterable [single file] + * Iterable [single file] + * Iterable [multiple files] + * Iterable [multiple files] + * Iterable<{ path, content: Bytes }> [multiple files] + * Iterable<{ path, content: Bloby }> [multiple files] + * Iterable<{ path, content: String }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: AsyncIterable }> [multiple files] + * AsyncIterable [single file] + * AsyncIterable [multiple files] + * AsyncIterable [multiple files] + * AsyncIterable<{ path, content: Bytes }> [multiple files] + * AsyncIterable<{ path, content: Bloby }> [multiple files] + * AsyncIterable<{ path, content: String }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] + * ``` + * Into: + * + * ``` + * AsyncIterable<{ path, content: Blob, mode, mtime }> + * ``` + * + * @param input Object + * @return AsyncInterable<{ path, content: Blob, mode, mtime }> + */ +module.exports = function normaliseInput (input) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') + } + + // String + // Buffer|ArrayBuffer|TypedArray + // Blob|File + if (typeof input === 'string' || input instanceof String || isBytes(input) || isBloby(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + // Iterable + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + + if (first.done) { + yield * iterator + return + } + + // Iterable + // Iterable + if (Number.isInteger(first.value) || isBytes(first.value)) { + return toFileObject((function * () { + yield first.value + yield * iterator + })()) + } + + // Iterable + // Iterable + // Iterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value) + + for (const obj of iterator) { + yield toFileObject(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // window.ReadableStream + if (typeof input.getReader === 'function') { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + const iterator = input[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) return iterator + + // AsyncIterable + if (isBytes(first.value)) { + yield toFileObject((async function * () { // eslint-disable-line require-await + yield first.value + yield * iterator + })()) + return + } + + // AsyncIterable + // AsyncIterable + // AsyncIterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value) + for await (const obj of iterator) { + yield toFileObject(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // { path, content: ? } + // Note: Detected _after_ AsyncIterable because Node.js streams have a + // `path` property that passes this check. + if (isFileObject(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +async function toFileObject (input) { + const obj = { + path: input.path || '', + mode: input.mode, + mtime: input.mtime + } + + if (input.content) { + obj.content = await toBlob(input.content) + } else if (!input.path) { // Not already a file object with path or content prop + obj.content = await toBlob(input) + } + + return obj +} + +function toBlob (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string') { + return new Blob([input]) + } + + // Bloby + if (isBloby(input)) { + return input + } + + // Browser stream + if (typeof input.getReader === 'function') { + return browserStreamToBlob(input) + } + + // Iterator + if (input[Symbol.iterator]) { + return itToBlob(input[Symbol.iterator]()) + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return itToBlob(input[Symbol.asyncIterator]()) + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBloby (obj) { + return obj instanceof Blob +} + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} + +async function itToBlob (stream) { + const parts = [] + + for await (const chunk of stream) { + parts.push(chunk) + } + + return new Blob(parts) +} + +async function browserStreamToBlob (stream) { + const parts = [] + const reader = stream.getReader() + + while (true) { + const result = await reader.read() + + if (result.done) { + break + } + + parts.push(result.value) + } + + return new Blob(parts) +} diff --git a/packages/ipfs-http-client/src/lib/to-stream.browser.js b/packages/ipfs-http-client/src/lib/to-stream.browser.js deleted file mode 100644 index 9f5784fedb..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.browser.js +++ /dev/null @@ -1,22 +0,0 @@ -'use strict' - -// browsers can't stream. When the 'Send ReadableStream in request body' row -// is green here: https://developer.mozilla.org/en-US/docs/Web/API/Request/Request#Browser_compatibility -// we'll be able to wrap the passed iterator in the it-to-browser-readablestream module -// in the meantime we have to convert the whole thing to a BufferSource of some sort -const toBuffer = require('it-to-buffer') -const { Buffer } = require('buffer') - -module.exports = (it) => { - async function * bufferise (source) { - for await (const chunk of source) { - if (Buffer.isBuffer(chunk)) { - yield chunk - } else { - yield Buffer.from(chunk) - } - } - } - - return toBuffer(bufferise(it)) -} diff --git a/packages/ipfs-http-client/src/lib/to-stream.js b/packages/ipfs-http-client/src/lib/to-stream.js deleted file mode 100644 index f0f59ffc50..0000000000 --- a/packages/ipfs-http-client/src/lib/to-stream.js +++ /dev/null @@ -1,7 +0,0 @@ -'use strict' - -const toStream = require('it-to-stream') - -module.exports = (it) => { - return toStream.readable(it) -} diff --git a/packages/ipfs/src/http/utils/multipart-request-parser.js b/packages/ipfs/src/http/utils/multipart-request-parser.js index 6ee29d52bc..84ea04be69 100644 --- a/packages/ipfs/src/http/utils/multipart-request-parser.js +++ b/packages/ipfs/src/http/utils/multipart-request-parser.js @@ -3,6 +3,7 @@ const Content = require('@hapi/content') const multipart = require('it-multipart') const { Buffer } = require('buffer') +const qs = require('querystring') const multipartFormdataType = 'multipart/form-data' const applicationDirectory = 'application/x-directory' @@ -69,20 +70,6 @@ async function * parseEntry (stream, options) { const entry = {} - if (part.headers.mtime) { - entry.mtime = { - secs: parseInt(part.headers.mtime, 10) - } - - if (part.headers['mtime-nsecs']) { - entry.mtime.nsecs = parseInt(part.headers['mtime-nsecs'], 10) - } - } - - if (part.headers.mode) { - entry.mode = parseInt(part.headers.mode, 8) - } - if (isDirectory(type.mime)) { entry.type = 'directory' } else if (type.mime === applicationSymlink) { @@ -92,6 +79,21 @@ async function * parseEntry (stream, options) { } const disposition = parseDisposition(part.headers['content-disposition']) + const query = qs.parse(disposition.name.split('?').pop()) + + if (query.mode) { + entry.mode = parseInt(query.mode, 8) + } + + if (query.mtime) { + entry.mtime = { + secs: parseInt(query.mtime, 10) + } + + if (query['mtime-nsecs']) { + entry.mtime.nsecs = parseInt(query['mtime-nsecs'], 10) + } + } entry.name = decodeURIComponent(disposition.filename) entry.body = part.body From efc443b892a6d177cf8ee109c50f7cf6ac6bc620 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 17:04:17 +0100 Subject: [PATCH 02/15] chore: add missing dep --- packages/ipfs-http-client/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 3ac8b9fe0a..3878876e0a 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -45,6 +45,7 @@ "buffer": "^5.6.0", "cids": "^0.8.3", "debug": "^4.1.0", + "err-code": "^2.0.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.0", "ipfs-utils": "^2.2.2", From 541b6c2bd8ba4306bdedda523f0d0e3a96258fa1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 17:19:54 +0100 Subject: [PATCH 03/15] chore: split content normalisation out --- packages/ipfs-core-utils/package.json | 3 + .../src/files/normalise-content.browser.js | 71 ++++++ .../src/files/normalise-content.js | 132 ++++++++++ .../src/files/normalise-input.js | 121 +-------- packages/ipfs-http-client/package.json | 1 - .../src/lib/multipart-request.browser.js | 2 +- .../src/lib/normalise-input.browser.js | 233 ------------------ 7 files changed, 220 insertions(+), 343 deletions(-) create mode 100644 packages/ipfs-core-utils/src/files/normalise-content.browser.js create mode 100644 packages/ipfs-core-utils/src/files/normalise-content.js delete mode 100644 packages/ipfs-http-client/src/lib/normalise-input.browser.js diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 3c791413bd..3d650f8cd8 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -40,5 +40,8 @@ "delay": "^4.3.0", "dirty-chai": "^2.0.1", "it-all": "^1.0.1" + }, + "browser": { + "./src/files/normalise-content": "./src/files/normalise-content.browser.js" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-content.browser.js new file mode 100644 index 0000000000..168595546e --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-content.browser.js @@ -0,0 +1,71 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const { Blob } = require('ipfs-utils/src/globalthis') + +function toBlob (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string') { + return new Blob([input]) + } + + // Bloby + if (isBloby(input)) { + return input + } + + // Browser stream + if (typeof input.getReader === 'function') { + return browserStreamToBlob(input) + } + + // Iterator + if (input[Symbol.iterator]) { + return itToBlob(input[Symbol.iterator]()) + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return itToBlob(input[Symbol.asyncIterator]()) + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBloby (obj) { + return obj instanceof Blob +} + +async function itToBlob (stream) { + const parts = [] + + for await (const chunk of stream) { + parts.push(chunk) + } + + return new Blob(parts) +} + +async function browserStreamToBlob (stream) { + const parts = [] + const reader = stream.getReader() + + while (true) { + const result = await reader.read() + + if (result.done) { + break + } + + parts.push(result.value) + } + + return new Blob(parts) +} + +module.exports = toBlob diff --git a/packages/ipfs-core-utils/src/files/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-content.js new file mode 100644 index 0000000000..1d0410a0a3 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-content.js @@ -0,0 +1,132 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const { Blob, FileReader } = require('ipfs-utils/src/globalthis') + +function toAsyncIterable (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string') { + return (async function * () { // eslint-disable-line require-await + yield toBuffer(input) + })() + } + + // Bloby + if (isBloby(input)) { + return blobToAsyncGenerator(input) + } + + // Browser stream + if (typeof input.getReader === 'function') { + return browserStreamToIt(input) + } + + // Iterator + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + if (first.done) return iterator + + // Iterable + if (Number.isInteger(first.value)) { + yield toBuffer(Array.from((function * () { + yield first.value + yield * iterator + })())) + return + } + + // Iterable + if (isBytes(first.value)) { + yield toBuffer(first.value) + for (const chunk of iterator) { + yield toBuffer(chunk) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + for await (const chunk of input) { + yield toBuffer(chunk) + } + })() + } + + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') +} + +function toBuffer (chunk) { + return isBytes(chunk) ? chunk : Buffer.from(chunk) +} + +function blobToAsyncGenerator (blob) { + if (typeof blob.stream === 'function') { + // firefox < 69 does not support blob.stream() + return browserStreamToIt(blob.stream()) + } + + return readBlob(blob) +} + +async function * readBlob (blob, options) { + options = options || {} + + const reader = new FileReader() + const chunkSize = options.chunkSize || 1024 * 1024 + let offset = options.offset || 0 + + const getNextChunk = () => new Promise((resolve, reject) => { + reader.onloadend = e => { + const data = e.target.result + resolve(data.byteLength === 0 ? null : data) + } + reader.onerror = reject + + const end = offset + chunkSize + const slice = blob.slice(offset, end) + reader.readAsArrayBuffer(slice) + offset = end + }) + + while (true) { + const data = await getNextChunk() + + if (data == null) { + return + } + + yield Buffer.from(data) + } +} + +function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBloby (obj) { + return obj instanceof Blob +} + +async function * browserStreamToIt (stream) { + const reader = stream.getReader() + + while (true) { + const result = await reader.read() + + if (result.done) { + return + } + + yield result.value + } +} + +module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input.js index c3f73168c5..076da4c5e4 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input.js @@ -3,6 +3,7 @@ const errCode = require('err-code') const { Buffer } = require('buffer') const globalThis = require('ipfs-utils/src/globalthis') +const normaliseContent = require('./normalise-content') /* * Transform one of: @@ -37,10 +38,17 @@ const globalThis = require('ipfs-utils/src/globalthis') * AsyncIterable<{ path, content: Iterable }> [multiple files] * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] * ``` - * Into: + * + * In node: + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> + * ``` + * + * And in the browser: * * ``` - * AsyncIterable<{ path, content: AsyncIterable }> + * AsyncIterable<{ path, mode, mtime, content: Blob }> * ``` * * @param input Object @@ -151,7 +159,7 @@ module.exports = function normaliseInput (input) { throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') } -function toFileObject (input) { +async function toFileObject (input) { const obj = { path: input.path || '', mode: input.mode, @@ -159,77 +167,14 @@ function toFileObject (input) { } if (input.content) { - obj.content = toAsyncIterable(input.content) + obj.content = await normaliseContent(input.content) } else if (!input.path) { // Not already a file object with path or content prop - obj.content = toAsyncIterable(input) + obj.content = await normaliseContent(input) } return obj } -function toAsyncIterable (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() - } - - // Bloby - if (isBloby(input)) { - return blobToAsyncGenerator(input) - } - - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToIt(input) - } - - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return - } - - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() - } - - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function toBuffer (chunk) { - return isBytes(chunk) ? chunk : Buffer.from(chunk) -} - function isBytes (obj) { return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer } @@ -243,15 +188,6 @@ function isFileObject (obj) { return typeof obj === 'object' && (obj.path || obj.content) } -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - async function * browserStreamToIt (stream) { const reader = stream.getReader() @@ -265,34 +201,3 @@ async function * browserStreamToIt (stream) { yield result.value } } - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new globalThis.FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 3878876e0a..3ac8b9fe0a 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -45,7 +45,6 @@ "buffer": "^5.6.0", "cids": "^0.8.3", "debug": "^4.1.0", - "err-code": "^2.0.0", "form-data": "^3.0.0", "ipfs-core-utils": "^0.3.0", "ipfs-utils": "^2.2.2", diff --git a/packages/ipfs-http-client/src/lib/multipart-request.browser.js b/packages/ipfs-http-client/src/lib/multipart-request.browser.js index 8bf9e35af6..550c85ba30 100644 --- a/packages/ipfs-http-client/src/lib/multipart-request.browser.js +++ b/packages/ipfs-http-client/src/lib/multipart-request.browser.js @@ -1,6 +1,6 @@ 'use strict' -const normaliseInput = require('./normalise-input.browser') +const normaliseInput = require('ipfs-core-utils/src/files/normalise-input') const modeToString = require('./mode-to-string') const mtimeToObject = require('./mtime-to-object') const { File, FormData } = require('ipfs-utils/src/globalthis') diff --git a/packages/ipfs-http-client/src/lib/normalise-input.browser.js b/packages/ipfs-http-client/src/lib/normalise-input.browser.js deleted file mode 100644 index d7a9eb8cb4..0000000000 --- a/packages/ipfs-http-client/src/lib/normalise-input.browser.js +++ /dev/null @@ -1,233 +0,0 @@ -'use strict' - -const errCode = require('err-code') -const { Buffer } = require('buffer') -const { Blob } = require('ipfs-utils/src/globalthis') - -/* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * Into: - * - * ``` - * AsyncIterable<{ path, content: Blob, mode, mtime }> - * ``` - * - * @param input Object - * @return AsyncInterable<{ path, content: Blob, mode, mtime }> - */ -module.exports = function normaliseInput (input) { - // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') - } - - // String - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (typeof input === 'string' || input instanceof String || isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - - if (first.done) { - yield * iterator - return - } - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - return toFileObject((function * () { - yield first.value - yield * iterator - })()) - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -async function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = await toBlob(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = await toBlob(input) - } - - return obj -} - -function toBlob (input) { - // Bytes | String - if (isBytes(input) || typeof input === 'string') { - return new Blob([input]) - } - - // Bloby - if (isBloby(input)) { - return input - } - - // Browser stream - if (typeof input.getReader === 'function') { - return browserStreamToBlob(input) - } - - // Iterator - if (input[Symbol.iterator]) { - return itToBlob(input[Symbol.iterator]()) - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return itToBlob(input[Symbol.asyncIterator]()) - } - - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') -} - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return obj instanceof Blob -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} - -async function itToBlob (stream) { - const parts = [] - - for await (const chunk of stream) { - parts.push(chunk) - } - - return new Blob(parts) -} - -async function browserStreamToBlob (stream) { - const parts = [] - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - break - } - - parts.push(result.value) - } - - return new Blob(parts) -} From 0d6fdb5f5b1b01eff9146a824c5fe241cbf44088 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 18:27:01 +0100 Subject: [PATCH 04/15] chore: fix up tests --- packages/ipfs-core-utils/package.json | 2 +- .../content.browser.js} | 14 ++-- .../content.js} | 71 ++--------------- .../index.js} | 37 ++------- .../src/files/normalise-input/utils.js | 79 +++++++++++++++++++ .../test/files/normalise-input.spec.js | 20 +++-- 6 files changed, 114 insertions(+), 109 deletions(-) rename packages/ipfs-core-utils/src/files/{normalise-content.browser.js => normalise-input/content.browser.js} (84%) rename packages/ipfs-core-utils/src/files/{normalise-content.js => normalise-input/content.js} (53%) rename packages/ipfs-core-utils/src/files/{normalise-input.js => normalise-input/index.js} (87%) create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/utils.js diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 3d650f8cd8..23afbeb7b2 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -42,6 +42,6 @@ "it-all": "^1.0.1" }, "browser": { - "./src/files/normalise-content": "./src/files/normalise-content.browser.js" + "./src/files/normalise-input/content": "./src/files/normalise-input/content.browser.js" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/content.browser.js similarity index 84% rename from packages/ipfs-core-utils/src/files/normalise-content.browser.js rename to packages/ipfs-core-utils/src/files/normalise-input/content.browser.js index 168595546e..32a12c2315 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/content.browser.js @@ -1,9 +1,13 @@ 'use strict' const errCode = require('err-code') -const { Buffer } = require('buffer') const { Blob } = require('ipfs-utils/src/globalthis') +const { + isBytes, + isBloby +} = require('./utils') + function toBlob (input) { // Bytes | String if (isBytes(input) || typeof input === 'string') { @@ -33,14 +37,6 @@ function toBlob (input) { throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') } -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return obj instanceof Blob -} - async function itToBlob (stream) { const parts = [] diff --git a/packages/ipfs-core-utils/src/files/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/content.js similarity index 53% rename from packages/ipfs-core-utils/src/files/normalise-content.js rename to packages/ipfs-core-utils/src/files/normalise-input/content.js index 1d0410a0a3..e062f1a178 100644 --- a/packages/ipfs-core-utils/src/files/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/content.js @@ -2,7 +2,12 @@ const errCode = require('err-code') const { Buffer } = require('buffer') -const { Blob, FileReader } = require('ipfs-utils/src/globalthis') +const { + isBytes, + isBloby, + browserStreamToIt, + blobToIt +} = require('./utils') function toAsyncIterable (input) { // Bytes | String @@ -14,7 +19,7 @@ function toAsyncIterable (input) { // Bloby if (isBloby(input)) { - return blobToAsyncGenerator(input) + return blobToIt(input) } // Browser stream @@ -67,66 +72,4 @@ function toBuffer (chunk) { return isBytes(chunk) ? chunk : Buffer.from(chunk) } -function blobToAsyncGenerator (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return obj instanceof Blob -} - -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} - module.exports = toAsyncIterable diff --git a/packages/ipfs-core-utils/src/files/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/index.js similarity index 87% rename from packages/ipfs-core-utils/src/files/normalise-input.js rename to packages/ipfs-core-utils/src/files/normalise-input/index.js index 076da4c5e4..5b5df5b868 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.js @@ -1,9 +1,13 @@ 'use strict' const errCode = require('err-code') -const { Buffer } = require('buffer') -const globalThis = require('ipfs-utils/src/globalthis') -const normaliseContent = require('./normalise-content') +const normaliseContent = require('./content') +const { + isBytes, + isBloby, + isFileObject, + browserStreamToIt +} = require('./utils') /* * Transform one of: @@ -174,30 +178,3 @@ async function toFileObject (input) { return obj } - -function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer -} - -function isBloby (obj) { - return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob -} - -// An object with a path or content property -function isFileObject (obj) { - return typeof obj === 'object' && (obj.path || obj.content) -} - -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js new file mode 100644 index 0000000000..7c8e79a562 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -0,0 +1,79 @@ +'use strict' + +const { Buffer } = require('buffer') +const { Blob, FileReader } = require('ipfs-utils/src/globalthis') + +function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBloby (obj) { + return typeof Blob !== 'undefined' && obj instanceof Blob +} + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} + +async function * browserStreamToIt (stream) { + const reader = stream.getReader() + + while (true) { + const result = await reader.read() + + if (result.done) { + return + } + + yield result.value + } +} + +function blobToIt (blob) { + if (typeof blob.stream === 'function') { + // firefox < 69 does not support blob.stream() + return browserStreamToIt(blob.stream()) + } + + return readBlob(blob) +} + +async function * readBlob (blob, options) { + options = options || {} + + const reader = new FileReader() + const chunkSize = options.chunkSize || 1024 * 1024 + let offset = options.offset || 0 + + const getNextChunk = () => new Promise((resolve, reject) => { + reader.onloadend = e => { + const data = e.target.result + resolve(data.byteLength === 0 ? null : data) + } + reader.onerror = reject + + const end = offset + chunkSize + const slice = blob.slice(offset, end) + reader.readAsArrayBuffer(slice) + offset = end + }) + + while (true) { + const data = await getNextChunk() + + if (data == null) { + return + } + + yield Buffer.from(data) + } +} + +module.exports = { + isBytes, + isBloby, + isFileObject, + browserStreamToIt, + blobToIt +} diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index e8491bd768..df1500bb13 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -6,7 +6,8 @@ const normalise = require('../../src/files/normalise-input') const { supportsFileReader } = require('ipfs-utils/src/supports') const { Buffer } = require('buffer') const all = require('it-all') -const globalThis = require('ipfs-utils/src/globalthis') +const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') +const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') const STRING = () => 'hello world' const BUFFER = () => Buffer.from(STRING()) @@ -16,11 +17,11 @@ let BLOB let WINDOW_READABLE_STREAM if (supportsFileReader) { - BLOB = () => new globalThis.Blob([ + BLOB = () => new Blob([ STRING() ]) - WINDOW_READABLE_STREAM = () => new globalThis.ReadableStream({ + WINDOW_READABLE_STREAM = () => new ReadableStream({ start (controller) { controller.enqueue(BUFFER()) controller.close() @@ -30,8 +31,17 @@ if (supportsFileReader) { async function verifyNormalisation (input) { expect(input.length).to.equal(1) - expect(input[0].content[Symbol.asyncIterator] || input[0].content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') - expect(await all(input[0].content)).to.deep.equal([BUFFER()]) + + const content = input[0].content + + if (isBrowser || isWebWorker) { + expect(content).to.be.an.instanceOf(Blob) + await expect(content.arrayBuffer()).to.eventually.deep.equal([BUFFER()]) + } else { + expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') + expect(await all(content)).to.deep.equal([BUFFER()]) + } + expect(input[0].path).to.equal('') } From 95b6b8b00794af0867673446f979eed54a50399a Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 18:42:24 +0100 Subject: [PATCH 05/15] fix: turn blob to iterator in test --- .../test/files/normalise-input.spec.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index df1500bb13..35bbce5d5f 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -3,6 +3,7 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') const normalise = require('../../src/files/normalise-input') +const { blobToIt } = require('../../src/files/normalise-input/utils') const { supportsFileReader } = require('ipfs-utils/src/supports') const { Buffer } = require('buffer') const all = require('it-all') @@ -31,18 +32,17 @@ if (supportsFileReader) { async function verifyNormalisation (input) { expect(input.length).to.equal(1) + expect(input[0].path).to.equal('') - const content = input[0].content + let content = input[0].content if (isBrowser || isWebWorker) { expect(content).to.be.an.instanceOf(Blob) - await expect(content.arrayBuffer()).to.eventually.deep.equal([BUFFER()]) - } else { - expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') - expect(await all(content)).to.deep.equal([BUFFER()]) + content = blobToIt(input[0].content) } - expect(input[0].path).to.equal('') + expect(content[Symbol.asyncIterator] || content[Symbol.iterator]).to.be.ok('Content should have been an iterable or an async iterable') + await expect(all(content)).to.eventually.deep.equal([BUFFER()]) } async function testContent (input) { From 5f91a2e321e941dcf7949682e8b6ba9c35702747 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 20 Jul 2020 19:23:24 +0100 Subject: [PATCH 06/15] chore: fix http tests --- .../ipfs/test/http-api/inject/mfs/write.js | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/packages/ipfs/test/http-api/inject/mfs/write.js b/packages/ipfs/test/http-api/inject/mfs/write.js index 5a8eca4d38..4d466a5187 100644 --- a/packages/ipfs/test/http-api/inject/mfs/write.js +++ b/packages/ipfs/test/http-api/inject/mfs/write.js @@ -29,11 +29,28 @@ const defaultOptions = { signal: sinon.match.instanceOf(AbortSignal) } -async function send (text, headers = {}) { +async function send (text, options = {}) { + let fieldName = 'file-0' + const query = [] + + if (options.mode) { + query.push(`mode=${options.mode}`) + } + + if (options.mtime) { + query.push(`mtime=${options.mtime}`) + } + + if (options.mtimeNsecs) { + query.push(`mtime-nsecs=${options.mtimeNsecs}`) + } + + if (query.length) { + fieldName = `${fieldName}?${query.join('&')}` + } + const form = new FormData() - form.append('file-0', Buffer.from(text), { - header: headers - }) + form.append(fieldName, Buffer.from(text)) return { headers: form.getHeaders(), From 634400b3436379fe1b993e9383482f9bb667017c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 09:55:38 +0100 Subject: [PATCH 07/15] chore: only normalise to blob in the http client --- packages/ipfs-core-utils/package.json | 3 - .../files/normalise-input/index.browser.js | 49 ++++++ .../src/files/normalise-input/index.js | 139 +----------------- ...rowser.js => normalise-content.browser.js} | 0 .../{content.js => normalise-content.js} | 0 .../files/normalise-input/normalise-input.js | 130 ++++++++++++++++ packages/ipfs-http-client/package.json | 3 +- 7 files changed, 185 insertions(+), 139 deletions(-) create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/index.browser.js rename packages/ipfs-core-utils/src/files/normalise-input/{content.browser.js => normalise-content.browser.js} (100%) rename packages/ipfs-core-utils/src/files/normalise-input/{content.js => normalise-content.js} (100%) create mode 100644 packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 23afbeb7b2..3c791413bd 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -40,8 +40,5 @@ "delay": "^4.3.0", "dirty-chai": "^2.0.1", "it-all": "^1.0.1" - }, - "browser": { - "./src/files/normalise-input/content": "./src/files/normalise-input/content.browser.js" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js new file mode 100644 index 0000000000..215eef3aa2 --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js @@ -0,0 +1,49 @@ +'use strict' + +const normaliseContent = require('./normalise-content.browser') +const normaliseInput = require('./normalise-input') + +/* + * Transform one of: + * + * ``` + * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] + * Bloby (Blob|File) [single file] + * String [single file] + * { path, content: Bytes } [single file] + * { path, content: Bloby } [single file] + * { path, content: String } [single file] + * { path, content: Iterable } [single file] + * { path, content: Iterable } [single file] + * { path, content: AsyncIterable } [single file] + * Iterable [single file] + * Iterable [single file] + * Iterable [multiple files] + * Iterable [multiple files] + * Iterable<{ path, content: Bytes }> [multiple files] + * Iterable<{ path, content: Bloby }> [multiple files] + * Iterable<{ path, content: String }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: AsyncIterable }> [multiple files] + * AsyncIterable [single file] + * AsyncIterable [multiple files] + * AsyncIterable [multiple files] + * AsyncIterable<{ path, content: Bytes }> [multiple files] + * AsyncIterable<{ path, content: Bloby }> [multiple files] + * AsyncIterable<{ path, content: String }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] + * ``` + * + * Into: + * + * ``` + * AsyncIterable<{ path, mode, mtime, content: Blob }> + * ``` + * + * @param input Object + * @return AsyncInterable<{ path, content: Blob }> + */ +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.js b/packages/ipfs-core-utils/src/files/normalise-input/index.js index 5b5df5b868..de60a6e18b 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/index.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.js @@ -1,13 +1,7 @@ 'use strict' -const errCode = require('err-code') -const normaliseContent = require('./content') -const { - isBytes, - isBloby, - isFileObject, - browserStreamToIt -} = require('./utils') +const normaliseContent = require('./normalise-content') +const normaliseInput = require('./normalise-input') /* * Transform one of: @@ -43,138 +37,13 @@ const { * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] * ``` * - * In node: + * Into: * * ``` * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> * ``` * - * And in the browser: - * - * ``` - * AsyncIterable<{ path, mode, mtime, content: Blob }> - * ``` - * * @param input Object * @return AsyncInterable<{ path, content: AsyncIterable }> */ -module.exports = function normaliseInput (input) { - // must give us something - if (input === null || input === undefined) { - throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') - } - - // String - if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Buffer|ArrayBuffer|TypedArray - // Blob|File - if (isBytes(input) || isBloby(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })()) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream - if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj) - } - })() - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })()) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value) - for await (const obj of iterator) { - yield toFileObject(obj) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a - // `path` property that passes this check. - if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input) - })() - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') -} - -async function toFileObject (input) { - const obj = { - path: input.path || '', - mode: input.mode, - mtime: input.mtime - } - - if (input.content) { - obj.content = await normaliseContent(input.content) - } else if (!input.path) { // Not already a file object with path or content prop - obj.content = await normaliseContent(input) - } - - return obj -} +module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js similarity index 100% rename from packages/ipfs-core-utils/src/files/normalise-input/content.browser.js rename to packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js diff --git a/packages/ipfs-core-utils/src/files/normalise-input/content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js similarity index 100% rename from packages/ipfs-core-utils/src/files/normalise-input/content.js rename to packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js new file mode 100644 index 0000000000..64f924940f --- /dev/null +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -0,0 +1,130 @@ +'use strict' + +const errCode = require('err-code') +const { + isBytes, + isBloby, + isFileObject, + browserStreamToIt +} = require('./utils') + +module.exports = function normaliseInput (input, normaliseContent) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') + } + + // String + if (typeof input === 'string' || input instanceof String) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input, normaliseContent) + })() + } + + // Buffer|ArrayBuffer|TypedArray + // Blob|File + if (isBytes(input) || isBloby(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input, normaliseContent) + })() + } + + // Iterable + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + if (first.done) return iterator + + // Iterable + // Iterable + if (Number.isInteger(first.value) || isBytes(first.value)) { + yield toFileObject((function * () { + yield first.value + yield * iterator + })(), normaliseContent) + return + } + + // Iterable + // Iterable + // Iterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value, normaliseContent) + for (const obj of iterator) { + yield toFileObject(obj, normaliseContent) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // window.ReadableStream + if (typeof input.getReader === 'function') { + return (async function * () { + for await (const obj of browserStreamToIt(input)) { + yield toFileObject(obj, normaliseContent) + } + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + const iterator = input[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) return iterator + + // AsyncIterable + if (isBytes(first.value)) { + yield toFileObject((async function * () { // eslint-disable-line require-await + yield first.value + yield * iterator + })(), normaliseContent) + return + } + + // AsyncIterable + // AsyncIterable + // AsyncIterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value, normaliseContent) + for await (const obj of iterator) { + yield toFileObject(obj, normaliseContent) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // { path, content: ? } + // Note: Detected _after_ AsyncIterable because Node.js streams have a + // `path` property that passes this check. + if (isFileObject(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input, normaliseContent) + })() + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +async function toFileObject (input, normaliseContent) { + const obj = { + path: input.path || '', + mode: input.mode, + mtime: input.mtime + } + + if (input.content) { + obj.content = await normaliseContent(input.content) + } else if (!input.path) { // Not already a file object with path or content prop + obj.content = await normaliseContent(input) + } + + return obj +} diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 3ac8b9fe0a..eb6d4ed75b 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -17,7 +17,8 @@ "browser": { "./src/lib/multipart-request.js": "./src/lib/multipart-request.browser.js", "ipfs-utils/src/files/glob-source": false, - "go-ipfs": false + "go-ipfs": false, + "ipfs-core-utils/src/files/normalise-input": "ipfs-core-utils/src/files/normalise-input/index.browser.js" }, "repository": { "type": "git", From 9ae9e77205bcb2d1d16aa928c435abd2877e4805 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 10:14:52 +0100 Subject: [PATCH 08/15] chore: fix up tests --- .../ipfs-core-utils/test/files/normalise-input.spec.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 35bbce5d5f..cc649b8d11 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -2,7 +2,6 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') -const normalise = require('../../src/files/normalise-input') const { blobToIt } = require('../../src/files/normalise-input/utils') const { supportsFileReader } = require('ipfs-utils/src/supports') const { Buffer } = require('buffer') @@ -10,6 +9,12 @@ const all = require('it-all') const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') const { isBrowser, isWebWorker } = require('ipfs-utils/src/env') +let normalise = require('../../src/files/normalise-input') + +if (isBrowser || isWebWorker) { + normalise = require('../../src/files/normalise-input/index.browser') +} + const STRING = () => 'hello world' const BUFFER = () => Buffer.from(STRING()) const ARRAY = () => Array.from(BUFFER()) From 83523461b6a307fbd16236c4c9a0e81ec2dcefa5 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 10:53:30 +0100 Subject: [PATCH 09/15] chore: add test for passing metadata in field name to go-ipfs --- packages/ipfs-core-utils/package.json | 2 + .../normalise-content.browser.js | 2 +- .../normalise-input/normalise-content.js | 6 +- .../files/normalise-input/normalise-input.js | 4 +- .../src/files/normalise-input/utils.js | 60 +------------------ packages/ipfs-http-client/test/files.spec.js | 38 ++++++++++++ 6 files changed, 48 insertions(+), 64 deletions(-) create mode 100644 packages/ipfs-http-client/test/files.spec.js diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index 3c791413bd..fa475194af 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -28,6 +28,8 @@ }, "license": "MIT", "dependencies": { + "blob-to-it": "0.0.1", + "browser-readablestream-to-it": "0.0.1", "buffer": "^5.6.0", "cids": "^0.8.3", "err-code": "^2.0.0", diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js index 32a12c2315..e1e0263d8f 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -14,7 +14,7 @@ function toBlob (input) { return new Blob([input]) } - // Bloby + // Blob | File if (isBloby(input)) { return input } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index e062f1a178..1e554817bd 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -2,11 +2,11 @@ const errCode = require('err-code') const { Buffer } = require('buffer') +const browserStreamToIt = require('browser-readablestream-to-it') +const blobToIt = require('blob-to-it') const { isBytes, - isBloby, - browserStreamToIt, - blobToIt + isBloby } = require('./utils') function toAsyncIterable (input) { diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js index 64f924940f..cdc72388c6 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -1,11 +1,11 @@ 'use strict' const errCode = require('err-code') +const browserStreamToIt = require('browser-readablestream-to-it') const { isBytes, isBloby, - isFileObject, - browserStreamToIt + isFileObject } = require('./utils') module.exports = function normaliseInput (input, normaliseContent) { diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js index 7c8e79a562..a3a09f353e 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/utils.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -1,7 +1,7 @@ 'use strict' const { Buffer } = require('buffer') -const { Blob, FileReader } = require('ipfs-utils/src/globalthis') +const { Blob } = require('ipfs-utils/src/globalthis') function isBytes (obj) { return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer @@ -16,64 +16,8 @@ function isFileObject (obj) { return typeof obj === 'object' && (obj.path || obj.content) } -async function * browserStreamToIt (stream) { - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - return - } - - yield result.value - } -} - -function blobToIt (blob) { - if (typeof blob.stream === 'function') { - // firefox < 69 does not support blob.stream() - return browserStreamToIt(blob.stream()) - } - - return readBlob(blob) -} - -async function * readBlob (blob, options) { - options = options || {} - - const reader = new FileReader() - const chunkSize = options.chunkSize || 1024 * 1024 - let offset = options.offset || 0 - - const getNextChunk = () => new Promise((resolve, reject) => { - reader.onloadend = e => { - const data = e.target.result - resolve(data.byteLength === 0 ? null : data) - } - reader.onerror = reject - - const end = offset + chunkSize - const slice = blob.slice(offset, end) - reader.readAsArrayBuffer(slice) - offset = end - }) - - while (true) { - const data = await getNextChunk() - - if (data == null) { - return - } - - yield Buffer.from(data) - } -} - module.exports = { isBytes, isBloby, - isFileObject, - browserStreamToIt, - blobToIt + isFileObject } diff --git a/packages/ipfs-http-client/test/files.spec.js b/packages/ipfs-http-client/test/files.spec.js new file mode 100644 index 0000000000..9ad1be1ff7 --- /dev/null +++ b/packages/ipfs-http-client/test/files.spec.js @@ -0,0 +1,38 @@ +/* eslint-env mocha */ + +'use strict' + +const { Buffer } = require('buffer') +const { expect } = require('interface-ipfs-core/src/utils/mocha') +const f = require('./utils/factory')() + +describe('.add', function () { + this.timeout(20 * 1000) + + let ipfs + + before(async function () { + ipfs = (await f.spawn()).api + }) + + after(() => f.clean()) + + it('should ignore metadata until https://github.com/ipfs/go-ipfs/issues/6920 is implemented', async () => { + const data = Buffer.from('some data') + const result = await ipfs.add(data, { + mode: 0o600, + mtime: { + secs: 1000, + nsecs: 0 + } + }) + + expect(result).to.not.have.property('mode') + expect(result).to.not.have.property('mtime') + expect(result).to.have.property('cid') + + const { cid } = result + expect(cid).to.have.property('codec', 'dag-pb') + expect(cid.toString()).to.equal('QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS') + }) +}) From 4407e45d011d4a0d30f386a18a0b4d251f7adcc0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 11:33:48 +0100 Subject: [PATCH 10/15] chore: remove redundant buffer check --- packages/ipfs-core-utils/src/files/normalise-input/utils.js | 3 +-- packages/ipfs-core-utils/test/files/normalise-input.spec.js | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js index a3a09f353e..0ce9fe2c45 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/utils.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -1,10 +1,9 @@ 'use strict' -const { Buffer } = require('buffer') const { Blob } = require('ipfs-utils/src/globalthis') function isBytes (obj) { - return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer + return ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer } function isBloby (obj) { diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index cc649b8d11..a5ff8d1224 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -2,7 +2,7 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') -const { blobToIt } = require('../../src/files/normalise-input/utils') +const blobToIt = require('blob-to-it') const { supportsFileReader } = require('ipfs-utils/src/supports') const { Buffer } = require('buffer') const all = require('it-all') From 044e2751c6682dc0a001a277dd6f8ab765c86a51 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 12:12:39 +0100 Subject: [PATCH 11/15] chore: rename isbloby to isblob --- .../files/normalise-input/normalise-content.browser.js | 4 ++-- .../src/files/normalise-input/normalise-input.js | 8 ++++---- .../ipfs-core-utils/src/files/normalise-input/utils.js | 4 ++-- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js index e1e0263d8f..d657e3817e 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -5,7 +5,7 @@ const { Blob } = require('ipfs-utils/src/globalthis') const { isBytes, - isBloby + isBlob } = require('./utils') function toBlob (input) { @@ -15,7 +15,7 @@ function toBlob (input) { } // Blob | File - if (isBloby(input)) { + if (isBlob(input)) { return input } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js index cdc72388c6..0e516c3cd9 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -4,7 +4,7 @@ const errCode = require('err-code') const browserStreamToIt = require('browser-readablestream-to-it') const { isBytes, - isBloby, + isBlob, isFileObject } = require('./utils') @@ -23,7 +23,7 @@ module.exports = function normaliseInput (input, normaliseContent) { // Buffer|ArrayBuffer|TypedArray // Blob|File - if (isBytes(input) || isBloby(input)) { + if (isBytes(input) || isBlob(input)) { return (async function * () { // eslint-disable-line require-await yield toFileObject(input, normaliseContent) })() @@ -49,7 +49,7 @@ module.exports = function normaliseInput (input, normaliseContent) { // Iterable // Iterable // Iterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + if (isFileObject(first.value) || isBlob(first.value) || typeof first.value === 'string') { yield toFileObject(first.value, normaliseContent) for (const obj of iterator) { yield toFileObject(obj, normaliseContent) @@ -89,7 +89,7 @@ module.exports = function normaliseInput (input, normaliseContent) { // AsyncIterable // AsyncIterable // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + if (isFileObject(first.value) || isBlob(first.value) || typeof first.value === 'string') { yield toFileObject(first.value, normaliseContent) for await (const obj of iterator) { yield toFileObject(obj, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/utils.js b/packages/ipfs-core-utils/src/files/normalise-input/utils.js index 0ce9fe2c45..c09f0c8241 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/utils.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/utils.js @@ -6,7 +6,7 @@ function isBytes (obj) { return ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer } -function isBloby (obj) { +function isBlob (obj) { return typeof Blob !== 'undefined' && obj instanceof Blob } @@ -17,6 +17,6 @@ function isFileObject (obj) { module.exports = { isBytes, - isBloby, + isBlob, isFileObject } From 9a6d98be3477e969bd51fc818f6da79fed813fcf Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 12:14:36 +0100 Subject: [PATCH 12/15] chore: rename isbloby to isblob --- .../src/files/normalise-input/normalise-content.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index 1e554817bd..9b93c3e896 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -6,7 +6,7 @@ const browserStreamToIt = require('browser-readablestream-to-it') const blobToIt = require('blob-to-it') const { isBytes, - isBloby + isBlob } = require('./utils') function toAsyncIterable (input) { @@ -17,8 +17,8 @@ function toAsyncIterable (input) { })() } - // Bloby - if (isBloby(input)) { + // Blob + if (isBlob(input)) { return blobToIt(input) } From d0e04db7742a16ed35ee5b137ed518524ad33ca7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 13:16:39 +0100 Subject: [PATCH 13/15] chore: convert normalise input to just yield from the function --- packages/ipfs-core-utils/package.json | 8 +- .../normalise-input/normalise-content.js | 68 +++++----- .../files/normalise-input/normalise-input.js | 117 ++++++------------ 3 files changed, 74 insertions(+), 119 deletions(-) diff --git a/packages/ipfs-core-utils/package.json b/packages/ipfs-core-utils/package.json index fa475194af..b08de20343 100644 --- a/packages/ipfs-core-utils/package.json +++ b/packages/ipfs-core-utils/package.json @@ -33,14 +33,16 @@ "buffer": "^5.6.0", "cids": "^0.8.3", "err-code": "^2.0.0", - "ipfs-utils": "^2.2.2" + "ipfs-utils": "^2.2.2", + "it-all": "^1.0.1", + "it-map": "^1.0.0", + "it-peekable": "0.0.1" }, "devDependencies": { "aegir": "^23.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.3.0", - "dirty-chai": "^2.0.1", - "it-all": "^1.0.1" + "dirty-chai": "^2.0.1" } } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index 9b93c3e896..df880803ca 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -4,65 +4,57 @@ const errCode = require('err-code') const { Buffer } = require('buffer') const browserStreamToIt = require('browser-readablestream-to-it') const blobToIt = require('blob-to-it') +const itPeekable = require('it-peekable') +const all = require('it-all') +const map = require('it-map') const { isBytes, isBlob } = require('./utils') -function toAsyncIterable (input) { +async function * toAsyncIterable (input) { // Bytes | String if (isBytes(input) || typeof input === 'string') { - return (async function * () { // eslint-disable-line require-await - yield toBuffer(input) - })() + yield toBuffer(input) + return } // Blob if (isBlob(input)) { - return blobToIt(input) + yield * blobToIt(input) + return } // Browser stream if (typeof input.getReader === 'function') { - return browserStreamToIt(input) + yield * browserStreamToIt(input) + return } - // Iterator - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() - // Iterable - if (Number.isInteger(first.value)) { - yield toBuffer(Array.from((function * () { - yield first.value - yield * iterator - })())) - return - } + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } - // Iterable - if (isBytes(first.value)) { - yield toBuffer(first.value) - for (const chunk of iterator) { - yield toBuffer(chunk) - } - return - } + peekable.push(value) - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } + // (Async)Iterable + if (Number.isInteger(value)) { + yield toBuffer(await all(peekable)) + return + } - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield toBuffer(chunk) - } - })() + // (Async)Iterable + if (isBytes(value)) { + yield * map(peekable, chunk => toBuffer(chunk)) + return + } } throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js index 0e516c3cd9..fbfc7d7885 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -2,13 +2,15 @@ const errCode = require('err-code') const browserStreamToIt = require('browser-readablestream-to-it') +const itPeekable = require('it-peekable') +const map = require('it-map') const { isBytes, isBlob, isFileObject } = require('./utils') -module.exports = function normaliseInput (input, normaliseContent) { +module.exports = async function * normaliseInput (input, normaliseContent) { // must give us something if (input === null || input === undefined) { throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') @@ -16,98 +18,57 @@ module.exports = function normaliseInput (input, normaliseContent) { // String if (typeof input === 'string' || input instanceof String) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input, normaliseContent) - })() + yield toFileObject(input, normaliseContent) + return } // Buffer|ArrayBuffer|TypedArray // Blob|File if (isBytes(input) || isBlob(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input, normaliseContent) - })() + yield toFileObject(input, normaliseContent) + return } - // Iterable - if (input[Symbol.iterator]) { - return (async function * () { // eslint-disable-line require-await - const iterator = input[Symbol.iterator]() - const first = iterator.next() - if (first.done) return iterator - - // Iterable - // Iterable - if (Number.isInteger(first.value) || isBytes(first.value)) { - yield toFileObject((function * () { - yield first.value - yield * iterator - })(), normaliseContent) - return - } - - // Iterable - // Iterable - // Iterable<{ path, content }> - if (isFileObject(first.value) || isBlob(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value, normaliseContent) - for (const obj of iterator) { - yield toFileObject(obj, normaliseContent) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() - } - - // window.ReadableStream + // Browser ReadableStream if (typeof input.getReader === 'function') { - return (async function * () { - for await (const obj of browserStreamToIt(input)) { - yield toFileObject(obj, normaliseContent) - } - })() + input = browserStreamToIt(input) } - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - const iterator = input[Symbol.asyncIterator]() - const first = await iterator.next() - if (first.done) return iterator - - // AsyncIterable - if (isBytes(first.value)) { - yield toFileObject((async function * () { // eslint-disable-line require-await - yield first.value - yield * iterator - })(), normaliseContent) - return - } - - // AsyncIterable - // AsyncIterable - // AsyncIterable<{ path, content }> - if (isFileObject(first.value) || isBlob(first.value) || typeof first.value === 'string') { - yield toFileObject(first.value, normaliseContent) - for await (const obj of iterator) { - yield toFileObject(obj, normaliseContent) - } - return - } - - throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') - })() + // Iterable + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + yield * peekable + return + } + + peekable.push(value) + + // (Async)Iterable + // (Async)Iterable + if (Number.isInteger(value) || isBytes(value)) { + yield toFileObject(peekable, normaliseContent) + return + } + + // (Async)Iterable + // (Async)Iterable + // (Async)Iterable<{ path, content }> + if (isFileObject(value) || isBlob(value) || typeof value === 'string') { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } } // { path, content: ? } - // Note: Detected _after_ AsyncIterable because Node.js streams have a + // Note: Detected _after_ (Async)Iterable because Node.js streams have a // `path` property that passes this check. if (isFileObject(input)) { - return (async function * () { // eslint-disable-line require-await - yield toFileObject(input, normaliseContent) - })() + yield toFileObject(input, normaliseContent) + return } throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') From 078bd4ad8ddcb14ae61c097a711aefdf4e5e86cc Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 14:37:01 +0100 Subject: [PATCH 14/15] chore: add tests around browser readable streams of things --- .../normalise-content.browser.js | 52 +++++++++---------- .../normalise-input/normalise-content.js | 7 ++- .../files/normalise-input/normalise-input.js | 2 +- .../test/files/normalise-input.spec.js | 38 +++++++------- 4 files changed, 48 insertions(+), 51 deletions(-) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js index d657e3817e..bc42be8ca8 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -2,15 +2,17 @@ const errCode = require('err-code') const { Blob } = require('ipfs-utils/src/globalthis') +const itPeekable = require('it-peekable') +const browserStreamToIt = require('browser-readablestream-to-it') const { isBytes, isBlob } = require('./utils') -function toBlob (input) { +async function toBlob (input) { // Bytes | String - if (isBytes(input) || typeof input === 'string') { + if (isBytes(input) || typeof input === 'string' || input instanceof String) { return new Blob([input]) } @@ -21,17 +23,30 @@ function toBlob (input) { // Browser stream if (typeof input.getReader === 'function') { - return browserStreamToBlob(input) + input = browserStreamToIt(input) } - // Iterator - if (input[Symbol.iterator]) { - return itToBlob(input[Symbol.iterator]()) - } + // (Async)Iterator + if (input[Symbol.iterator] || input[Symbol.asyncIterator]) { + const peekable = itPeekable(input) + const { value, done } = await peekable.peek() + + if (done) { + // make sure empty iterators result in empty files + return itToBlob(peekable) + } - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return itToBlob(input[Symbol.asyncIterator]()) + peekable.push(value) + + // (Async)Iterable + if (Number.isInteger(value)) { + return itToBlob(peekable) + } + + // (Async)Iterable + if (isBytes(value) || typeof value === 'string' || value instanceof String) { + return itToBlob(peekable) + } } throw errCode(new Error(`Unexpected input: ${input}`), 'ERR_UNEXPECTED_INPUT') @@ -47,21 +62,4 @@ async function itToBlob (stream) { return new Blob(parts) } -async function browserStreamToBlob (stream) { - const parts = [] - const reader = stream.getReader() - - while (true) { - const result = await reader.read() - - if (result.done) { - break - } - - parts.push(result.value) - } - - return new Blob(parts) -} - module.exports = toBlob diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index df880803ca..d220d4eb7d 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -14,7 +14,7 @@ const { async function * toAsyncIterable (input) { // Bytes | String - if (isBytes(input) || typeof input === 'string') { + if (isBytes(input) || typeof input === 'string' || input instanceof String) { yield toBuffer(input) return } @@ -27,8 +27,7 @@ async function * toAsyncIterable (input) { // Browser stream if (typeof input.getReader === 'function') { - yield * browserStreamToIt(input) - return + input = browserStreamToIt(input) } // (Async)Iterator @@ -51,7 +50,7 @@ async function * toAsyncIterable (input) { } // (Async)Iterable - if (isBytes(value)) { + if (isBytes(value) || typeof value === 'string' || value instanceof String) { yield * map(peekable, chunk => toBuffer(chunk)) return } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js index fbfc7d7885..f3dcfee09a 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -57,7 +57,7 @@ module.exports = async function * normaliseInput (input, normaliseContent) { // (Async)Iterable // (Async)Iterable // (Async)Iterable<{ path, content }> - if (isFileObject(value) || isBlob(value) || typeof value === 'string') { + if (isFileObject(value) || isBlob(value) || typeof value === 'string' || value instanceof String) { yield * map(peekable, (value) => toFileObject(value, normaliseContent)) return } diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index a5ff8d1224..5b74c9d823 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -3,7 +3,6 @@ /* eslint-env mocha */ const { expect } = require('../utils/chai') const blobToIt = require('blob-to-it') -const { supportsFileReader } = require('ipfs-utils/src/supports') const { Buffer } = require('buffer') const all = require('it-all') const { Blob, ReadableStream } = require('ipfs-utils/src/globalthis') @@ -16,23 +15,16 @@ if (isBrowser || isWebWorker) { } const STRING = () => 'hello world' +const NEWSTRING = () => new String('hello world') // eslint-disable-line no-new-wrappers const BUFFER = () => Buffer.from(STRING()) const ARRAY = () => Array.from(BUFFER()) const TYPEDARRAY = () => Uint8Array.from(ARRAY()) let BLOB -let WINDOW_READABLE_STREAM -if (supportsFileReader) { +if (Blob) { BLOB = () => new Blob([ STRING() ]) - - WINDOW_READABLE_STREAM = () => new ReadableStream({ - start (controller) { - controller.enqueue(BUFFER()) - controller.close() - } - }) } async function verifyNormalisation (input) { @@ -66,12 +58,27 @@ function asyncIterableOf (thing) { }()) } +function browserReadableStreamOf (thing) { + return new ReadableStream({ + start (controller) { + controller.enqueue(thing) + controller.close() + } + }) +} + describe('normalise-input', function () { function testInputType (content, name, isBytes) { it(name, async function () { await testContent(content()) }) + if (ReadableStream && isBytes) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + if (isBytes) { it(`Iterable<${name}>`, async function () { await testContent(iterableOf(content())) @@ -125,6 +132,7 @@ describe('normalise-input', function () { describe('String', () => { testInputType(STRING, 'String', false) + testInputType(NEWSTRING, 'new String()', false) }) describe('Buffer', () => { @@ -132,21 +140,13 @@ describe('normalise-input', function () { }) describe('Blob', () => { - if (!supportsFileReader) { + if (!Blob) { return } testInputType(BLOB, 'Blob', false) }) - describe('window.ReadableStream', () => { - if (!supportsFileReader) { - return - } - - testInputType(WINDOW_READABLE_STREAM, 'window.ReadableStream', false) - }) - describe('Iterable', () => { testInputType(ARRAY, 'Iterable', false) }) From b8a869cb7f7b884c59c35ced290fb2cdba382540 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 22 Jul 2020 17:28:01 +0100 Subject: [PATCH 15/15] chore: adds tests for more input types --- docs/core-api/FILES.md | 24 ++++++++--- .../files/normalise-input/index.browser.js | 39 ++---------------- .../src/files/normalise-input/index.js | 39 ++---------------- .../normalise-content.browser.js | 2 +- .../normalise-input/normalise-content.js | 2 +- .../files/normalise-input/normalise-input.js | 9 +++++ .../test/files/normalise-input.spec.js | 40 +++++++++++++++---- 7 files changed, 69 insertions(+), 86 deletions(-) diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index fe338e47cb..8756a2902f 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -7,6 +7,9 @@ _Explore the Mutable File System through interactive coding challenges in our [P - [The Regular API](#the-regular-api) - [`ipfs.add(data, [options])`](#ipfsadddata-options) - [Parameters](#parameters) + - [FileStream](#filestream) + - [FileObject](#fileobject) + - [FileContent](#filecontent) - [Options](#options) - [Returns](#returns) - [`ipfs.addAll(source, [options])`](#ipfsaddallsource-options) @@ -108,12 +111,19 @@ The regular, top-level API for add, cat, get and ls Files on IPFS `data` may be: -* `Blob` -* `String` -* `Uint8Array` +* `FileContent` (see below for definition) * `FileObject` (see below for definition) -* `Iterable` -* `AsyncIterable` +* `FileStream` (see below for definition) + +##### FileStream + +`FileStream` is a stream of `FileContent` or `FileObject` entries of the type: + +```js +Iterable | AsyncIterable | ReadableStream +``` + +##### FileObject `FileObject` is a plain JS object of the following form: @@ -136,10 +146,12 @@ If no `content` is passed, then the item is treated as an empty directory. One of `path` or `content` _must_ be passed. +##### FileContent + `FileContent` is one of the following types: ```js -Uint8Array | Blob | String | Iterable | AsyncIterable +Uint8Array | Blob | String | Iterable | AsyncIterable | ReadableStream ``` `UnixTime` is one of the following types: diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js index 215eef3aa2..c96260b528 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.browser.js @@ -4,46 +4,15 @@ const normaliseContent = require('./normalise-content.browser') const normaliseInput = require('./normalise-input') /* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * - * Into: + * Transforms any of the `ipfs.add` input types into * * ``` * AsyncIterable<{ path, mode, mtime, content: Blob }> * ``` * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * * @param input Object - * @return AsyncInterable<{ path, content: Blob }> + * @return AsyncInterable<{ path, mode, mtime, content: Blob }> */ module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/index.js b/packages/ipfs-core-utils/src/files/normalise-input/index.js index de60a6e18b..00dd946bfa 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/index.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/index.js @@ -4,46 +4,15 @@ const normaliseContent = require('./normalise-content') const normaliseInput = require('./normalise-input') /* - * Transform one of: - * - * ``` - * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] - * Bloby (Blob|File) [single file] - * String [single file] - * { path, content: Bytes } [single file] - * { path, content: Bloby } [single file] - * { path, content: String } [single file] - * { path, content: Iterable } [single file] - * { path, content: Iterable } [single file] - * { path, content: AsyncIterable } [single file] - * Iterable [single file] - * Iterable [single file] - * Iterable [multiple files] - * Iterable [multiple files] - * Iterable<{ path, content: Bytes }> [multiple files] - * Iterable<{ path, content: Bloby }> [multiple files] - * Iterable<{ path, content: String }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: Iterable }> [multiple files] - * Iterable<{ path, content: AsyncIterable }> [multiple files] - * AsyncIterable [single file] - * AsyncIterable [multiple files] - * AsyncIterable [multiple files] - * AsyncIterable<{ path, content: Bytes }> [multiple files] - * AsyncIterable<{ path, content: Bloby }> [multiple files] - * AsyncIterable<{ path, content: String }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: Iterable }> [multiple files] - * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] - * ``` - * - * Into: + * Transforms any of the `ipfs.add` input types into * * ``` * AsyncIterable<{ path, mode, mtime, content: AsyncIterable }> * ``` * + * See https://github.com/ipfs/js-ipfs/blob/master/docs/core-api/FILES.md#ipfsadddata-options + * * @param input Object - * @return AsyncInterable<{ path, content: AsyncIterable }> + * @return AsyncInterable<{ path, mode, mtime, content: AsyncIterable }> */ module.exports = (input) => normaliseInput(input, normaliseContent) diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js index bc42be8ca8..fe55cde87c 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.browser.js @@ -43,7 +43,7 @@ async function toBlob (input) { return itToBlob(peekable) } - // (Async)Iterable + // (Async)Iterable if (isBytes(value) || typeof value === 'string' || value instanceof String) { return itToBlob(peekable) } diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js index d220d4eb7d..c55f938e0f 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-content.js @@ -49,7 +49,7 @@ async function * toAsyncIterable (input) { return } - // (Async)Iterable + // (Async)Iterable if (isBytes(value) || typeof value === 'string' || value instanceof String) { yield * map(peekable, chunk => toBuffer(chunk)) return diff --git a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js index f3dcfee09a..90bb00015f 100644 --- a/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js +++ b/packages/ipfs-core-utils/src/files/normalise-input/normalise-input.js @@ -61,6 +61,15 @@ module.exports = async function * normaliseInput (input, normaliseContent) { yield * map(peekable, (value) => toFileObject(value, normaliseContent)) return } + + // (Async)Iterable<(Async)Iterable> + // (Async)Iterable> + // ReadableStream<(Async)Iterable> + // ReadableStream> + if (value[Symbol.iterator] || value[Symbol.asyncIterator] || typeof value.getReader === 'function') { + yield * map(peekable, (value) => toFileObject(value, normaliseContent)) + return + } } // { path, content: ? } diff --git a/packages/ipfs-core-utils/test/files/normalise-input.spec.js b/packages/ipfs-core-utils/test/files/normalise-input.spec.js index 5b74c9d823..7e206c2bd3 100644 --- a/packages/ipfs-core-utils/test/files/normalise-input.spec.js +++ b/packages/ipfs-core-utils/test/files/normalise-input.spec.js @@ -73,13 +73,13 @@ describe('normalise-input', function () { await testContent(content()) }) - if (ReadableStream && isBytes) { - it(`ReadableStream<${name}>`, async function () { - await testContent(browserReadableStreamOf(content())) - }) - } - if (isBytes) { + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<${name}>`, async function () { await testContent(iterableOf(content())) }) @@ -94,6 +94,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`{ path: '', content: ReadableStream<${name}> }`, async function () { + await testContent({ path: '', content: browserReadableStreamOf(content()) }) + }) + } + it(`{ path: '', content: Iterable<${name}> }`, async function () { await testContent({ path: '', content: iterableOf(content()) }) }) @@ -103,6 +109,12 @@ describe('normalise-input', function () { }) } + if (ReadableStream) { + it(`ReadableStream<${name}>`, async function () { + await testContent(browserReadableStreamOf(content())) + }) + } + it(`Iterable<{ path: '', content: ${name} }`, async function () { await testContent(iterableOf({ path: '', content: content() })) }) @@ -112,6 +124,12 @@ describe('normalise-input', function () { }) if (isBytes) { + if (ReadableStream) { + it(`Iterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(iterableOf({ path: '', content: iterableOf(content()) })) }) @@ -120,6 +138,12 @@ describe('normalise-input', function () { await testContent(iterableOf({ path: '', content: asyncIterableOf(content()) })) }) + if (ReadableStream) { + it(`AsyncIterable<{ path: '', content: ReadableStream<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: browserReadableStreamOf(content()) })) + }) + } + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { await testContent(asyncIterableOf({ path: '', content: iterableOf(content()) })) }) @@ -131,8 +155,8 @@ describe('normalise-input', function () { } describe('String', () => { - testInputType(STRING, 'String', false) - testInputType(NEWSTRING, 'new String()', false) + testInputType(STRING, 'String', true) + testInputType(NEWSTRING, 'new String()', true) }) describe('Buffer', () => {