From 67f2b4efc141a0871214bf0d3dacb89c403a8a7e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 13 Jul 2020 15:49:20 +0100 Subject: [PATCH 1/7] refactor: ipfs.add only works on single items, ipfs.addAll works on multiple `ipfs.add` returns single items only. Where a source would result in multiple items returned, only the last item is returned. `ipfs.addAll` retains the previous behaviour of `ipfs.add`. Examples: ```javascript const { cid, path, mode, mtime } = await ipfs.add('Hello world') ``` ```javascript const { cid } = await ipfs.add(Uint16Array.from([0, 1, 2])) ``` ```javascript const { cid } = await ipfs.add(fs.createReadStream('/path/to/file.txt')) ``` ```javascript const { cid } = await ipfs.add({ path: '/foo/bar/baz.txt', content: 'File content' }) // Creates a DAG with multiple levels of directories. // The returned cid is the CID for the root directory /foo // You can retrieve the file content with an IPFS path for await (const buf of ipfs.cat(`/ipfs/${cid}/bar/baz.txt`)) { ... } // Or get the CID of the nested file with ipfs.files.stat // (stat or something like it should really be a root level command) const { cid: fileCid } = await ipfs.files.stat(`/ipfs/${cid}/bar/baz.txt`) ``` BREAKING CHANGES: - `ipfs.add` only works on single items - a Uint8Array, a String, an AsyncIterable etc - `ipfs.addAll` works on multiple items --- docs/core-api/FILES.md | 205 ++++++++-- examples/browser-add-readable-stream/index.js | 12 +- examples/browser-browserify/src/index.js | 10 +- examples/browser-parceljs/public/index.js | 17 +- examples/browser-readablestream/utils.js | 2 +- examples/browser-script-tag/index.html | 5 +- .../browser-webpack/src/components/app.js | 17 +- examples/custom-ipfs-repo/index.js | 17 +- .../exchange-files-in-browser/public/app.js | 5 +- examples/exchange-files-in-browser/test.js | 6 +- examples/ipfs-101/1.js | 12 +- examples/ipfs-101/README.md | 10 +- packages/interface-ipfs-core/src/add-all.js | 381 ++++++++++++++++++ packages/interface-ipfs-core/src/add.js | 355 +++------------- packages/interface-ipfs-core/src/dht/get.js | 5 +- .../interface-ipfs-core/src/dht/provide.js | 13 +- packages/interface-ipfs-core/src/dht/put.js | 5 +- packages/interface-ipfs-core/src/files/cp.js | 6 +- .../interface-ipfs-core/src/files/read.js | 3 +- .../interface-ipfs-core/src/files/stat.js | 3 +- packages/interface-ipfs-core/src/index.js | 1 + packages/interface-ipfs-core/src/ls.js | 18 +- .../src/miscellaneous/resolve.js | 8 +- .../interface-ipfs-core/src/name/publish.js | 5 +- .../interface-ipfs-core/src/name/resolve.js | 17 +- .../interface-ipfs-core/src/object/get.js | 7 +- .../interface-ipfs-core/src/object/links.js | 9 +- packages/interface-ipfs-core/src/pin/add.js | 3 +- packages/interface-ipfs-core/src/pin/ls.js | 6 +- packages/interface-ipfs-core/src/pin/rm.js | 4 +- packages/interface-ipfs-core/src/repo/gc.js | 22 +- .../src/utils/create-sharded-directory.js | 2 +- .../src/utils/create-two-shards.js | 4 +- packages/ipfs-http-client/README.md | 12 +- .../examples/bundle-webpack/src/App.js | 20 +- .../examples/name-api/index.js | 9 +- .../upload-file-via-browser/src/App.js | 26 +- packages/ipfs-http-client/src/add-all.js | 63 +++ packages/ipfs-http-client/src/add.js | 64 +-- packages/ipfs-http-client/src/index.js | 1 + packages/ipfs-http-client/test/get.spec.js | 4 +- packages/ipfs-http-client/test/log.spec.js | 3 +- packages/ipfs/src/cli/commands/add.js | 2 +- .../core/components/{add => add-all}/index.js | 2 +- .../core/components/{add => add-all}/utils.js | 0 packages/ipfs/src/core/components/add.js | 9 + packages/ipfs/src/core/components/index.js | 1 + packages/ipfs/src/core/components/init.js | 7 +- packages/ipfs/src/core/components/start.js | 5 +- packages/ipfs/src/core/components/stop.js | 5 +- .../src/core/runtime/init-assets-nodejs.js | 4 +- .../src/http/api/resources/files-regular.js | 2 +- packages/ipfs/test/cli/add.js | 50 +-- .../core/{add.spec.js => add-all.spec.js} | 4 +- packages/ipfs/test/core/circuit-relay.spec.js | 5 +- .../ipfs/test/core/files-sharding.spec.js | 4 +- packages/ipfs/test/core/files.spec.js | 11 +- packages/ipfs/test/core/gc.spec.js | 5 +- packages/ipfs/test/core/kad-dht.node.js | 9 +- packages/ipfs/test/core/name-pubsub.js | 2 +- packages/ipfs/test/core/pin.js | 2 +- packages/ipfs/test/core/preload.spec.js | 45 +-- packages/ipfs/test/core/utils.js | 3 +- packages/ipfs/test/gateway/index.js | 12 +- packages/ipfs/test/http-api/inject/files.js | 8 +- 65 files changed, 927 insertions(+), 667 deletions(-) create mode 100644 packages/interface-ipfs-core/src/add-all.js create mode 100644 packages/ipfs-http-client/src/add-all.js rename packages/ipfs/src/core/components/{add => add-all}/index.js (97%) rename packages/ipfs/src/core/components/{add => add-all}/utils.js (100%) create mode 100644 packages/ipfs/src/core/components/add.js rename packages/ipfs/test/core/{add.spec.js => add-all.spec.js} (95%) diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index 1e35c92002..024b8239c1 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -9,77 +9,81 @@ _Explore the Mutable File System through interactive coding challenges in our [P - [Parameters](#parameters) - [Options](#options) - [Returns](#returns) + - [`ipfs.addAll(source, [options])`](#ipfsaddallsource-options) + - [Parameters](#parameters-1) + - [Options](#options-1) + - [Returns](#returns-1) - [Notes](#notes) - [Chunking options](#chunking-options) - [Hash algorithms](#hash-algorithms) - [Importing files from the file system](#importing-files-from-the-file-system) - [Importing a file from a URL](#importing-a-file-from-a-url) - [`ipfs.cat(ipfsPath, [options])`](#ipfscatipfspath-options) - - [Parameters](#parameters-1) - - [Options](#options-1) - - [Returns](#returns-1) - - [Example](#example) - - [`ipfs.get(ipfsPath, [options])`](#ipfsgetipfspath-options) - [Parameters](#parameters-2) - [Options](#options-2) - [Returns](#returns-2) - - [`ipfs.ls(ipfsPath)`](#ipfslsipfspath) + - [Example](#example) + - [`ipfs.get(ipfsPath, [options])`](#ipfsgetipfspath-options) - [Parameters](#parameters-3) - [Options](#options-3) - [Returns](#returns-3) - - [Example](#example-1) -- [The Mutable Files API](#the-mutable-files-api) - - [`ipfs.files.chmod(path, mode, [options])`](#ipfsfileschmodpath-mode-options) + - [`ipfs.ls(ipfsPath)`](#ipfslsipfspath) - [Parameters](#parameters-4) - [Options](#options-4) - [Returns](#returns-4) - - [Example](#example-2) - - [`ipfs.files.cp(...from, to, [options])`](#ipfsfilescpfrom-to-options) + - [Example](#example-1) +- [The Mutable Files API](#the-mutable-files-api) + - [`ipfs.files.chmod(path, mode, [options])`](#ipfsfileschmodpath-mode-options) - [Parameters](#parameters-5) - [Options](#options-5) - [Returns](#returns-5) - - [Example](#example-3) - - [Notes](#notes-1) - - [`ipfs.files.mkdir(path, [options])`](#ipfsfilesmkdirpath-options) + - [Example](#example-2) + - [`ipfs.files.cp(...from, to, [options])`](#ipfsfilescpfrom-to-options) - [Parameters](#parameters-6) - [Options](#options-6) - [Returns](#returns-6) - - [Example](#example-4) - - [`ipfs.files.stat(path, [options])`](#ipfsfilesstatpath-options) + - [Example](#example-3) + - [Notes](#notes-1) + - [`ipfs.files.mkdir(path, [options])`](#ipfsfilesmkdirpath-options) - [Parameters](#parameters-7) - [Options](#options-7) - [Returns](#returns-7) - - [Example](#example-5) - - [`ipfs.files.touch(path, [options])`](#ipfsfilestouchpath-options) + - [Example](#example-4) + - [`ipfs.files.stat(path, [options])`](#ipfsfilesstatpath-options) - [Parameters](#parameters-8) - [Options](#options-8) - [Returns](#returns-8) - - [Example](#example-6) - - [`ipfs.files.rm(...paths, [options])`](#ipfsfilesrmpaths-options) + - [Example](#example-5) + - [`ipfs.files.touch(path, [options])`](#ipfsfilestouchpath-options) - [Parameters](#parameters-9) - [Options](#options-9) - - [Example](#example-7) - - [`ipfs.files.read(path, [options])`](#ipfsfilesreadpath-options) + - [Returns](#returns-9) + - [Example](#example-6) + - [`ipfs.files.rm(...paths, [options])`](#ipfsfilesrmpaths-options) - [Parameters](#parameters-10) - [Options](#options-10) - - [Returns](#returns-9) - - [`ipfs.files.write(path, content, [options])`](#ipfsfileswritepath-content-options) + - [Example](#example-7) + - [`ipfs.files.read(path, [options])`](#ipfsfilesreadpath-options) - [Parameters](#parameters-11) - [Options](#options-11) - [Returns](#returns-10) - - [`ipfs.files.mv(...from, to, [options])`](#ipfsfilesmvfrom-to-options) + - [`ipfs.files.write(path, content, [options])`](#ipfsfileswritepath-content-options) - [Parameters](#parameters-12) - [Options](#options-12) - [Returns](#returns-11) - - [Example](#example-8) - - [`ipfs.files.flush(path, [options])`](#ipfsfilesflushpath-options) + - [`ipfs.files.mv(...from, to, [options])`](#ipfsfilesmvfrom-to-options) - [Parameters](#parameters-13) - [Options](#options-13) - [Returns](#returns-12) - - [`ipfs.files.ls(path, [options])`](#ipfsfileslspath-options) + - [Example](#example-8) + - [`ipfs.files.flush(path, [options])`](#ipfsfilesflushpath-options) - [Parameters](#parameters-14) - [Options](#options-14) - [Returns](#returns-13) + - [`ipfs.files.ls(path, [options])`](#ipfsfileslspath-options) + - [Parameters](#parameters-15) + - [Options](#options-15) + - [Returns](#returns-14) - [Example](#example-9) ## The Regular API @@ -87,7 +91,7 @@ The regular, top-level API for add, cat, get and ls Files on IPFS ### `ipfs.add(data, [options])` -> Import files and data into IPFS. +> Import a file or data into IPFS. #### Parameters @@ -97,19 +101,132 @@ The regular, top-level API for add, cat, get and ls Files on IPFS `data` may be: -* `Bytes` (alias for `Buffer`|`ArrayBuffer`|`TypedArray`) [single file] -* `Bloby` (alias for: `Blob`|`File`) [single file] -* `string` [single file] -* `FileObject` (see below for definition) [single file] -* `Iterable` [single file] -* `Iterable` [single file] -* `Iterable` [multiple files] -* `Iterable` [multiple files] -* `Iterable` [multiple files] -* `AsyncIterable` [single file] -* `AsyncIterable` [multiple files] -* `AsyncIterable` [multiple files] -* `AsyncIterable` [multiple files] +* `Blob` +* `String` +* `Uint8Array` +* `FileObject` (see below for definition) +* `Iterable` +* `AsyncIterable` + +`FileObject` is a plain JS object of the following form: + +```js +{ + // The path you want to the file to be accessible at from the root CID _after_ it has been added + path?: string + // The contents of the file (see below for definition) + content?: FileContent + // File mode to store the entry with (see https://en.wikipedia.org/wiki/File_system_permissions#Numeric_notation) + mode?: number | string + // The modification time of the entry (see below for definition) + mtime?: UnixTime +} +``` + +If no `path` is specified, then the item will be added to the root level and will be given a name according to it's CID. + +If no `content` is passed, then the item is treated as an empty directory. + +One of `path` or `content` _must_ be passed. + +`FileContent` is one of the following types: + +```js +Uint8Array | Blob | String | Iterable | AsyncIterable +``` + +`UnixTime` is one of the following types: + +```js +Date | { secs: number, nsecs?: number } | number[] +``` + +As an object, `secs` is the number of seconds since (positive) or before (negative) the Unix Epoch began and `nsecs` is the number of nanoseconds since the last full second. + +As an array of numbers, it must have two elements, as per the output of [`process.hrtime()`](https://nodejs.org/dist/latest/docs/api/process.html#process_process_hrtime_time). + +#### Options + +An optional object which may have the following keys: + +| Name | Type | Default | Description | +| ---- | ---- | ------- | ----------- | +| chunker | `String` | `'size-262144` | chunking algorithm used to build ipfs DAGs | +| cidVersion | `Number` | `0` | the CID version to use when storing the data | +| hashAlg | `String` | `'sha2-256'` | multihash hashing algorithm to use | +| onlyHash | `boolean` | `false` | If true, will not add blocks to the blockstore | +| pin | `boolean` | `true` | pin this object when adding | +| progress | function | `undefined` | a function that will be called with the byte length of chunks as a file is added to ipfs | +| rawLeaves | `boolean` | `false` | if true, DAG leaves will contain raw file data and not be wrapped in a protobuf | +| trickle | `boolean` | `false` | if true will use the [trickle DAG](https://godoc.org/github.com/ipsn/go-ipfs/gxlibs/github.com/ipfs/go-unixfs/importer/trickle) format for DAG generation | +| wrapWithDirectory | `boolean` | `false` | Adds a wrapping node around the content | +| timeout | `Number` | `undefined` | A timeout in ms | +| signal | [AbortSignal][] | `undefined` | Can be used to cancel any long running requests started as a result of this call | + +#### Returns + +| Type | Description | +| -------- | -------- | +| `UnixFSEntry` | A object describing the added data | + +Each yielded object is of the form: + +```JavaScript +{ + path: '/tmp/myfile.txt', + cid: CID('QmHash'), + mode: Number, + mtime: { secs: Number, nsecs: Number }, + size: 123 +} +``` + +#### Example + +```js +const file = { + path: '/tmp/myfile.txt', + content: 'ABC' +} + +const result of await ipfs.add(content) + +console.info(result) + +/* +Prints: +{ + "path": "tmp", + "cid": CID("QmWXdjNC362aPDtwHPUE9o2VMqPeNeCQuTBTv1NsKtwypg"), + "mode": 493, + "mtime": { secs: Number, nsecs: Number }, + "size": 67 +} +*/ +``` + +Now [ipfs.io/ipfs/Qm..pg/myfile.txt](https://ipfs.io/ipfs/QmWXdjNC362aPDtwHPUE9o2VMqPeNeCQuTBTv1NsKtwypg/myfile.txt) returns the "ABC" string. + +### `ipfs.addAll(source, [options])` + +> Import multiple files and data into IPFS. + +#### Parameters + +| Name | Type | Description | +| ---- | ---- | ----------- | +| source | Object | Data to import (see below) | + +`source` may be: + +* `Iterable` +* `Iterable` +* `Iterable` +* `Iterable` +* `AsyncIterable` +* `AsyncIterable` +* `AsyncIterable` +* `AsyncIterable` `FileObject` is a plain JS object of the following form: @@ -135,7 +252,7 @@ One of `path` or `content` _must_ be passed. `FileContent` is one of the following types: ```js -Bytes | Bloby | string | Iterable | Iterable | AsyncIterable +Uint8Array | Blob | String | Iterable | AsyncIterable ``` `UnixTime` is one of the following types: @@ -172,7 +289,7 @@ An optional object which may have the following keys: | Type | Description | | -------- | -------- | -| `AsyncIterable` | An async iterable that yields objects describing the added data | +| `AsyncIterable` | An async iterable that yields objects describing the added data | Each yielded object is of the form: diff --git a/examples/browser-add-readable-stream/index.js b/examples/browser-add-readable-stream/index.js index 7abfb83554..6db3128f7b 100644 --- a/examples/browser-add-readable-stream/index.js +++ b/examples/browser-add-readable-stream/index.js @@ -52,13 +52,13 @@ const streamFiles = async (ipfs, directory, files) => { } }) - for await (const data of ipfs.add(stream)) { - log(`Added ${data.path} hash: ${data.hash}`) + const data = await ipfs.add(stream) - // The last data event will contain the directory hash - if (data.path === directory) { - return data.cid - } + log(`Added ${data.path} hash: ${data.hash}`) + + // The last data event will contain the directory hash + if (data.path === directory) { + return data.cid } } diff --git a/examples/browser-browserify/src/index.js b/examples/browser-browserify/src/index.js index 4fdbf533bf..7397c159c3 100644 --- a/examples/browser-browserify/src/index.js +++ b/examples/browser-browserify/src/index.js @@ -10,13 +10,11 @@ document.addEventListener('DOMContentLoaded', async () => { async function store () { const toStore = document.getElementById('source').value - for await (const file of node.add(toStore)) { - if (file && file.cid) { - console.log('successfully stored', file.cid) + const file = await node.add(toStore) - await display(file.cid) - } - } + console.log('successfully stored', file.cid) + + await display(file.cid) } async function display (cid) { diff --git a/examples/browser-parceljs/public/index.js b/examples/browser-parceljs/public/index.js index 803553a3fa..c0d6f4b6fa 100644 --- a/examples/browser-parceljs/public/index.js +++ b/examples/browser-parceljs/public/index.js @@ -22,18 +22,17 @@ document.addEventListener('DOMContentLoaded', async () => { log(`The IPFS node version is ${version.version}`) - for await (const entry of node.add({ + const entry = await node.add({ path: 'hello-parcel.txt', content: 'Hello from parcel.js bundled ipfs example' - })) { - log(`This page deployed ${entry.path} to IPFS and its CID is ${entry.cid}`) + }) + log(`This page deployed ${entry.path} to IPFS and its CID is ${entry.cid}`) - const buffers = [] + const buffers = [] - for await (const buf of node.cat(entry.cid)) { - buffers.push(buf) - } - - log(`The contents of the file was: ${Buffer.concat(buffers).toString()}`) + for await (const buf of node.cat(entry.cid)) { + buffers.push(buf) } + + log(`The contents of the file was: ${Buffer.concat(buffers).toString()}`) }) diff --git a/examples/browser-readablestream/utils.js b/examples/browser-readablestream/utils.js index 3579a4882c..f01bd38689 100644 --- a/examples/browser-readablestream/utils.js +++ b/examples/browser-readablestream/utils.js @@ -45,7 +45,7 @@ const dragDrop = (ipfs) => { const progress = log(`IPFS: Adding...`) - for await (const added of ipfs.add(files, { + for await (const added of ipfs.addAll(files, { progress: (addedBytes) => { progress.textContent = `IPFS: Adding ${addedBytes} bytes\r\n` } diff --git a/examples/browser-script-tag/index.html b/examples/browser-script-tag/index.html index 6bb9fb3ba3..2ead69df6b 100644 --- a/examples/browser-script-tag/index.html +++ b/examples/browser-script-tag/index.html @@ -31,9 +31,8 @@

Some suggestions

async function addFile () { - for await (const { cid } of node.add('Hello world!')) { - console.log('successfully stored', cid) - } + const { cid } = await node.add('Hello world!')) + console.log('successfully stored', cid) } addFile() diff --git a/examples/browser-webpack/src/components/app.js b/examples/browser-webpack/src/components/app.js index 9a6ce2f38e..a06a2e398d 100644 --- a/examples/browser-webpack/src/components/app.js +++ b/examples/browser-webpack/src/components/app.js @@ -30,18 +30,17 @@ class App extends React.Component { this.setState({ id, agentVersion, protocolVersion }) - for await (const { cid } of node.add(stringToUse)) { - this.setState({ addedFileHash: cid.toString() }) + const { cid } = await node.add(stringToUse) + this.setState({ addedFileHash: cid.toString() }) - let bufs = [] + let bufs = [] - for await (const buf of node.cat(cid)) { - bufs.push(buf) - } - - const data = Buffer.concat(bufs) - this.setState({ addedFileContents: data.toString('utf8') }) + for await (const buf of node.cat(cid)) { + bufs.push(buf) } + + const data = Buffer.concat(bufs) + this.setState({ addedFileContents: data.toString('utf8') }) } render () { diff --git a/examples/custom-ipfs-repo/index.js b/examples/custom-ipfs-repo/index.js index 058bbd2ec3..6640aa773d 100644 --- a/examples/custom-ipfs-repo/index.js +++ b/examples/custom-ipfs-repo/index.js @@ -80,19 +80,18 @@ async function main () { console.log('Version:', version) // Once we have the version, let's add a file to IPFS - for await (const file of node.add({ + const file = await node.add({ path: 'test-data.txt', content: Buffer.from('We are using a customized repo!') - })) { - // Log out the added files metadata and cat the file from IPFS - console.log('\nAdded file:', file.path, file.cid) + }) + // Log out the added files metadata and cat the file from IPFS + console.log('\nAdded file:', file.path, file.cid) - const data = Buffer.concat(await all(node.cat(file.cid))) + const data = Buffer.concat(await all(node.cat(file.cid))) - // Print out the files contents to console - console.log('\nFetched file content:') - process.stdout.write(data) - } + // Print out the files contents to console + console.log('\nFetched file content:') + process.stdout.write(data) // After everything is done, shut the node down console.log('\n\nStopping the node') diff --git a/examples/exchange-files-in-browser/public/app.js b/examples/exchange-files-in-browser/public/app.js index 2af870ebcd..3a7f05d6c0 100644 --- a/examples/exchange-files-in-browser/public/app.js +++ b/examples/exchange-files-in-browser/public/app.js @@ -3,7 +3,6 @@ const IPFS = require('ipfs') const all = require('it-all') -const last = require('it-last') const { Buffer } = IPFS // Node @@ -249,13 +248,13 @@ async function onDrop (event) { for (const file of files) { fileSize = file.size // Note: fileSize is used by updateProgress - const fileAdded = await last(node.add({ + const fileAdded = await node.add({ path: file.name, content: file }, { wrapWithDirectory: true, progress: updateProgress - })) + }) // As we are wrapping the content we use that hash to keep // the original file name when adding it to the table diff --git a/examples/exchange-files-in-browser/test.js b/examples/exchange-files-in-browser/test.js index 30d7582e10..c190c1efae 100644 --- a/examples/exchange-files-in-browser/test.js +++ b/examples/exchange-files-in-browser/test.js @@ -59,11 +59,7 @@ async function runTest () { } }) - let cid - - for await (const imported of relay.api.add(FILE_CONTENT)) { - cid = imported.cid - } + const { cid } = await relay.api.add(FILE_CONTENT) const server1 = await startServer(__dirname) const server2 = await startServer(__dirname) diff --git a/examples/ipfs-101/1.js b/examples/ipfs-101/1.js index 314f2dabbe..67cc05e79b 100644 --- a/examples/ipfs-101/1.js +++ b/examples/ipfs-101/1.js @@ -9,16 +9,16 @@ async function main () { console.log('Version:', version.version) - for await (const file of await node.add({ + const file = await node.add({ path: 'hello.txt', content: Buffer.from('Hello World 101') - })) { - console.log('Added file:', file.path, file.cid.toString()) + }) - const data = Buffer.concat(await all(node.cat(file.cid))) + console.log('Added file:', file.path, file.cid.toString()) - console.log('Added file contents:', data.toString()) - } + const data = Buffer.concat(await all(node.cat(file.cid))) + + console.log('Added file contents:', data.toString()) } main() diff --git a/examples/ipfs-101/README.md b/examples/ipfs-101/README.md index 4476bb81c1..2db1113ef3 100644 --- a/examples/ipfs-101/README.md +++ b/examples/ipfs-101/README.md @@ -63,7 +63,6 @@ You can learn about the IPFS File API at [interface-ipfs-core](https://github.co ```js const IPFS = require('ipfs') -const last = require('it-last') async function main () { const node = await IPFS.create() @@ -71,10 +70,10 @@ async function main () { console.log('Version:', version.version) - const fileAdded = await last(node.add({ + const fileAdded = await node.add({ path: 'hello.txt', content: 'Hello World 101' - })) + }) console.log('Added file:', fileAdded.path, fileAdded.cid) // ... @@ -98,7 +97,6 @@ The last step of this tutorial is retrieving the file back using the `cat` 😺 ```js const IPFS = require('ipfs') -const last = require('it-last') async function main () { const node = await IPFS.create() @@ -106,10 +104,10 @@ async function main () { console.log('Version:', version.version) - const fileAdded = await last(node.add({ + const fileAdded = await node.add({ path: 'hello.txt', content: 'Hello World 101' - })) + }) console.log('Added file:', fileAdded.path, fileAdded.cid) diff --git a/packages/interface-ipfs-core/src/add-all.js b/packages/interface-ipfs-core/src/add-all.js new file mode 100644 index 0000000000..1ad872d7e7 --- /dev/null +++ b/packages/interface-ipfs-core/src/add-all.js @@ -0,0 +1,381 @@ +/* eslint-env mocha, browser */ +'use strict' + +const { Buffer } = require('buffer') +const { fixtures } = require('./utils') +const { Readable } = require('readable-stream') +const all = require('it-all') +const last = require('it-last') +const drain = require('it-drain') +const fs = require('fs') +const os = require('os') +const path = require('path') +const { supportsFileReader } = require('ipfs-utils/src/supports') +const globSource = require('ipfs-utils/src/files/glob-source') +const { isNode } = require('ipfs-utils/src/env') +const { getDescribe, getIt, expect } = require('./utils/mocha') +const testTimeout = require('./utils/test-timeout') + +/** @typedef { import("ipfsd-ctl/src/factory") } Factory */ +/** + * @param {Factory} common + * @param {Object} options + */ +module.exports = (common, options) => { + const describe = getDescribe(options) + const it = getIt(options) + + describe('.addAll', function () { + this.timeout(40 * 1000) + + let ipfs + + async function testMode (mode, expectedMode) { + const content = String(Math.random() + Date.now()) + const files = await all(ipfs.addAll([{ + content: Buffer.from(content), + mode + }])) + expect(files).to.have.length(1) + expect(files).to.have.nested.property('[0].mode', expectedMode) + + const stats = await ipfs.files.stat(`/ipfs/${files[0].cid}`) + expect(stats).to.have.property('mode', expectedMode) + } + + async function testMtime (mtime, expectedMtime) { + const content = String(Math.random() + Date.now()) + const files = await all(ipfs.addAll({ + content: Buffer.from(content), + mtime + })) + expect(files).to.have.length(1) + expect(files).to.have.deep.nested.property('[0].mtime', expectedMtime) + + const stats = await ipfs.files.stat(`/ipfs/${files[0].cid}`) + expect(stats).to.have.deep.property('mtime', expectedMtime) + } + + before(async () => { ipfs = (await common.spawn()).api }) + + after(() => common.clean()) + + it('should respect timeout option when adding files', () => { + return testTimeout(() => drain(ipfs.addAll(Buffer.from('Hello'), { + timeout: 1 + }))) + }) + + it('should add a File as array of tuples', async function () { + if (!supportsFileReader) return this.skip('skip in node') + + const tuple = { + path: 'filename.txt', + content: new self.File(['should add a File'], 'filename.txt', { type: 'text/plain' }) + } + + const filesAdded = await all(ipfs.addAll([tuple])) + expect(filesAdded[0].cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') + }) + + it('should add a Buffer as array of tuples', async () => { + const tuple = { path: 'testfile.txt', content: fixtures.smallFile.data } + + const filesAdded = await all(ipfs.addAll([tuple])) + expect(filesAdded).to.have.length(1) + + const file = filesAdded[0] + expect(file.cid.toString()).to.equal(fixtures.smallFile.cid) + expect(file.path).to.equal('testfile.txt') + }) + + it('should add array of objects with readable stream content', async function () { + if (!isNode) this.skip() + const expectedCid = 'QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS' + + const rs = new Readable() + rs.push(Buffer.from('some data')) + rs.push(null) + + const tuple = { path: 'data.txt', content: rs } + + const filesAdded = await all(ipfs.addAll([tuple])) + expect(filesAdded).to.be.length(1) + + const file = filesAdded[0] + expect(file.path).to.equal('data.txt') + expect(file.size).to.equal(17) + expect(file.cid.toString()).to.equal(expectedCid) + }) + + it('should add a nested directory as array of tupples', async function () { + const content = (name) => ({ + path: `test-folder/${name}`, + content: fixtures.directory.files[name] + }) + + const emptyDir = (name) => ({ path: `test-folder/${name}` }) + + const dirs = [ + content('pp.txt'), + content('holmes.txt'), + content('jungle.txt'), + content('alice.txt'), + emptyDir('empty-folder'), + content('files/hello.txt'), + content('files/ipfs.txt'), + emptyDir('files/empty') + ] + + const root = await last(ipfs.addAll(dirs)) + + expect(root.path).to.equal('test-folder') + expect(root.cid.toString()).to.equal(fixtures.directory.cid) + }) + + it('should add a nested directory as array of tupples with progress', async function () { + const content = (name) => ({ + path: `test-folder/${name}`, + content: fixtures.directory.files[name] + }) + + const emptyDir = (name) => ({ path: `test-folder/${name}` }) + + const dirs = [ + content('pp.txt'), + content('holmes.txt'), + content('jungle.txt'), + content('alice.txt'), + emptyDir('empty-folder'), + content('files/hello.txt'), + content('files/ipfs.txt'), + emptyDir('files/empty') + ] + + const total = dirs.reduce((i, entry) => { + return i + (entry.content ? entry.content.length : 0) + }, 0) + + let progCalled = false + let accumProgress = 0 + const handler = (p) => { + progCalled = true + accumProgress += p + } + + const root = await last(ipfs.addAll(dirs, { progress: handler })) + expect(progCalled).to.be.true() + expect(accumProgress).to.be.at.least(total) + expect(root.path).to.equal('test-folder') + expect(root.cid.toString()).to.equal(fixtures.directory.cid) + }) + + it('should add files to a directory non sequentially', async function () { + const content = path => ({ + path: `test-dir/${path}`, + content: fixtures.directory.files[path.split('/').pop()] + }) + + const input = [ + content('a/pp.txt'), + content('a/holmes.txt'), + content('b/jungle.txt'), + content('a/alice.txt') + ] + + const filesAdded = await all(ipfs.addAll(input)) + + const toPath = ({ path }) => path + const nonSeqDirFilePaths = input.map(toPath).filter(p => p.includes('/a/')) + const filesAddedPaths = filesAdded.map(toPath) + + expect(nonSeqDirFilePaths.every(p => filesAddedPaths.includes(p))).to.be.true() + }) + + it('should fail when passed invalid input', async () => { + const nonValid = 138 + + await expect(all(ipfs.addAll(nonValid))).to.eventually.be.rejected() + }) + + it('should wrap content in a directory', async () => { + const data = { path: 'testfile.txt', content: fixtures.smallFile.data } + + const filesAdded = await all(ipfs.addAll(data, { wrapWithDirectory: true })) + expect(filesAdded).to.have.length(2) + + const file = filesAdded[0] + const wrapped = filesAdded[1] + expect(file.cid.toString()).to.equal(fixtures.smallFile.cid) + expect(file.path).to.equal('testfile.txt') + expect(wrapped.path).to.equal('') + }) + + it('should add a directory with only-hash=true', async function () { + this.slow(10 * 1000) + const content = String(Math.random() + Date.now()) + + const files = await all(ipfs.addAll([{ + path: '/foo/bar.txt', + content: Buffer.from(content) + }, { + path: '/foo/baz.txt', + content: Buffer.from(content) + }], { onlyHash: true })) + expect(files).to.have.length(3) + + await Promise.all( + files.map(file => expect(ipfs.object.get(file.cid, { timeout: 4000 })) + .to.eventually.be.rejected() + .and.to.have.property('name').that.equals('TimeoutError') + ) + ) + }) + + it('should add with mode as string', async function () { + this.slow(10 * 1000) + const mode = '0777' + await testMode(mode, parseInt(mode, 8)) + }) + + it('should add with mode as number', async function () { + this.slow(10 * 1000) + const mode = parseInt('0777', 8) + await testMode(mode, mode) + }) + + it('should add with mtime as Date', async function () { + this.slow(10 * 1000) + const mtime = new Date(5000) + await testMtime(mtime, { + secs: 5, + nsecs: 0 + }) + }) + + it('should add with mtime as { nsecs, secs }', async function () { + this.slow(10 * 1000) + const mtime = { + secs: 5, + nsecs: 0 + } + await testMtime(mtime, mtime) + }) + + it('should add with mtime as timespec', async function () { + this.slow(10 * 1000) + await testMtime({ + Seconds: 5, + FractionalNanoseconds: 0 + }, { + secs: 5, + nsecs: 0 + }) + }) + + it('should add with mtime as hrtime', async function () { + this.slow(10 * 1000) + const mtime = process.hrtime() + await testMtime(mtime, { + secs: mtime[0], + nsecs: mtime[1] + }) + }) + + it('should add a directory from the file system', async function () { + if (!isNode) this.skip() + const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'test-folder') + + const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true }))) + expect(result.length).to.be.above(8) + }) + + it('should add a directory from the file system with an odd name', async function () { + if (!isNode) this.skip() + + const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'weird name folder [v0]') + + const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true }))) + expect(result.length).to.be.above(8) + }) + + it('should ignore a directory from the file system', async function () { + if (!isNode) this.skip() + + const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'test-folder') + + const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true, ignore: ['files/**'] }))) + expect(result.length).to.be.below(9) + }) + + it('should add a file from the file system', async function () { + if (!isNode) this.skip() + + const filePath = path.join(__dirname, '..', 'test', 'fixtures', 'testfile.txt') + + const result = await all(ipfs.addAll(globSource(filePath))) + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('testfile.txt') + }) + + it('should add a hidden file in a directory from the file system', async function () { + if (!isNode) this.skip() + + const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'hidden-files-folder') + + const result = await all(ipfs.addAll(globSource(filesPath, { recursive: true, hidden: true }))) + expect(result.length).to.be.above(10) + expect(result.map(object => object.path)).to.include('hidden-files-folder/.hiddenTest.txt') + expect(result.map(object => object.cid.toString())).to.include('QmdbAjVmLRdpFyi8FFvjPfhTGB2cVXvWLuK7Sbt38HXrtt') + }) + + it('should add a file from the file system with only-hash=true', async function () { + if (!isNode) this.skip() + + this.slow(10 * 1000) + + const content = String(Math.random() + Date.now()) + const filepath = path.join(os.tmpdir(), `${content}.txt`) + fs.writeFileSync(filepath, content) + + const out = await all(ipfs.addAll(globSource(filepath), { onlyHash: true })) + + fs.unlinkSync(filepath) + + await expect(ipfs.object.get(out[0].cid, { timeout: 500 })) + .to.eventually.be.rejected() + .and.to.have.property('name').that.equals('TimeoutError') + }) + + it('should respect raw leaves when file is smaller than one block and no metadata is present', async () => { + const files = await all(ipfs.addAll(Buffer.from([0, 1, 2]), { + cidVersion: 1, + rawLeaves: true + })) + + expect(files.length).to.equal(1) + expect(files[0].cid.toString()).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') + expect(files[0].cid.codec).to.equal('raw') + expect(files[0].size).to.equal(3) + }) + + it('should override raw leaves when file is smaller than one block and metadata is present', async () => { + const files = await all(ipfs.addAll({ + content: Buffer.from([0, 1, 2]), + mode: 0o123, + mtime: { + secs: 1000, + nsecs: 0 + } + }, { + cidVersion: 1, + rawLeaves: true + })) + + expect(files.length).to.equal(1) + expect(files[0].cid.toString()).to.equal('bafybeifmayxiu375ftlgydntjtffy5cssptjvxqw6vyuvtymntm37mpvua') + expect(files[0].cid.codec).to.equal('dag-pb') + expect(files[0].size).to.equal(18) + }) + }) +} diff --git a/packages/interface-ipfs-core/src/add.js b/packages/interface-ipfs-core/src/add.js index 70d86b9c85..25a5a683d9 100644 --- a/packages/interface-ipfs-core/src/add.js +++ b/packages/interface-ipfs-core/src/add.js @@ -4,14 +4,7 @@ const { Buffer } = require('buffer') const { fixtures } = require('./utils') const { Readable } = require('readable-stream') -const all = require('it-all') -const last = require('it-last') -const drain = require('it-drain') -const fs = require('fs') -const os = require('os') -const path = require('path') const { supportsFileReader } = require('ipfs-utils/src/supports') -const globSource = require('ipfs-utils/src/files/glob-source') const urlSource = require('ipfs-utils/src/files/url-source') const { isNode } = require('ipfs-utils/src/env') const { getDescribe, getIt, expect } = require('./utils/mocha') @@ -35,27 +28,25 @@ module.exports = (common, options) => { async function testMode (mode, expectedMode) { const content = String(Math.random() + Date.now()) - const files = await all(ipfs.add({ + const file = await ipfs.add({ content: Buffer.from(content), mode - })) - expect(files).to.have.length(1) - expect(files).to.have.nested.property('[0].mode', expectedMode) + }) + expect(file).to.have.property('mode', expectedMode) - const stats = await ipfs.files.stat(`/ipfs/${files[0].cid}`) + const stats = await ipfs.files.stat(`/ipfs/${file.cid}`) expect(stats).to.have.property('mode', expectedMode) } async function testMtime (mtime, expectedMtime) { const content = String(Math.random() + Date.now()) - const files = await all(ipfs.add({ + const file = await ipfs.add({ content: Buffer.from(content), mtime - })) - expect(files).to.have.length(1) - expect(files).to.have.deep.nested.property('[0].mtime', expectedMtime) + }) + expect(file).to.have.deep.property('mtime', expectedMtime) - const stats = await ipfs.files.stat(`/ipfs/${files[0].cid}`) + const stats = await ipfs.files.stat(`/ipfs/${file.cid}`) expect(stats).to.have.deep.property('mtime', expectedMtime) } @@ -64,16 +55,16 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when adding files', () => { - return testTimeout(() => drain(ipfs.add(Buffer.from('Hello'), { + return testTimeout(() => ipfs.add(Buffer.from('Hello'), { timeout: 1 - }))) + })) }) it('should add a File', async function () { if (!supportsFileReader) return this.skip('skip in node') - const filesAdded = await all(ipfs.add(new self.File(['should add a File'], 'filename.txt', { type: 'text/plain' }))) - expect(filesAdded[0].cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') + const fileAdded = await ipfs.add(new self.File(['should add a File'], 'filename.txt', { type: 'text/plain' })) + expect(fileAdded.cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') }) it('should add a File as tuple', async function () { @@ -84,27 +75,13 @@ module.exports = (common, options) => { content: new self.File(['should add a File'], 'filename.txt', { type: 'text/plain' }) } - const filesAdded = await all(ipfs.add(tuple)) - expect(filesAdded[0].cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') - }) - - it('should add a File as array of tuple', async function () { - if (!supportsFileReader) return this.skip('skip in node') - - const tuple = { - path: 'filename.txt', - content: new self.File(['should add a File'], 'filename.txt', { type: 'text/plain' }) - } - - const filesAdded = await all(ipfs.add([tuple])) - expect(filesAdded[0].cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') + const fileAdded = await ipfs.add(tuple) + expect(fileAdded.cid.toString()).to.be.eq('QmTVfLxf3qXiJgr4KwG6UBckcNvTqBp93Rwy5f7h3mHsVC') }) it('should add a Buffer', async () => { - const filesAdded = await all(ipfs.add(fixtures.smallFile.data)) - expect(filesAdded).to.have.length(1) + const file = await ipfs.add(fixtures.smallFile.data) - const file = filesAdded[0] expect(file.cid.toString()).to.equal(fixtures.smallFile.cid) expect(file.path).to.equal(fixtures.smallFile.cid) // file.size counts the overhead by IPLD nodes and unixfs protobuf @@ -112,10 +89,8 @@ module.exports = (common, options) => { }) it('should add a BIG Buffer', async () => { - const filesAdded = await all(ipfs.add(fixtures.bigFile.data)) - expect(filesAdded).to.have.length(1) + const file = await ipfs.add(fixtures.bigFile.data) - const file = filesAdded[0] expect(file.cid.toString()).to.equal(fixtures.bigFile.cid) expect(file.path).to.equal(fixtures.bigFile.cid) // file.size counts the overhead by IPLD nodes and unixfs protobuf @@ -130,10 +105,8 @@ module.exports = (common, options) => { accumProgress = p } - const filesAdded = await all(ipfs.add(fixtures.bigFile.data, { progress: handler })) - expect(filesAdded).to.have.length(1) + const file = await ipfs.add(fixtures.bigFile.data, { progress: handler }) - const file = filesAdded[0] expect(file.cid.toString()).to.equal(fixtures.bigFile.cid) expect(file.path).to.equal(fixtures.bigFile.cid) expect(progCalled).to.be.true() @@ -143,10 +116,8 @@ module.exports = (common, options) => { it('should add a Buffer as tuple', async () => { const tuple = { path: 'testfile.txt', content: fixtures.smallFile.data } - const filesAdded = await all(ipfs.add([tuple])) - expect(filesAdded).to.have.length(1) + const file = await ipfs.add(tuple) - const file = filesAdded[0] expect(file.cid.toString()).to.equal(fixtures.smallFile.cid) expect(file.path).to.equal('testfile.txt') }) @@ -155,26 +126,22 @@ module.exports = (common, options) => { const data = 'a string' const expectedCid = 'QmQFRCwEpwQZ5aQMqCsCaFbdjNLLHoyZYDjr92v1F7HeqX' - const filesAdded = await all(ipfs.add(data)) - expect(filesAdded).to.be.length(1) + const file = await ipfs.add(data) - const { path, size, cid } = filesAdded[0] - expect(path).to.equal(expectedCid) - expect(size).to.equal(16) - expect(cid.toString()).to.equal(expectedCid) + expect(file).to.have.property('path', expectedCid) + expect(file).to.have.property('size', 16) + expect(`${file.cid}`).to.equal(expectedCid) }) it('should add a TypedArray', async () => { const data = Uint8Array.from([1, 3, 8]) const expectedCid = 'QmRyUEkVCuHC8eKNNJS9BDM9jqorUvnQJK1DM81hfngFqd' - const filesAdded = await all(ipfs.add(data)) - expect(filesAdded).to.be.length(1) + const file = await ipfs.add(data) - const { path, size, cid } = filesAdded[0] - expect(path).to.equal(expectedCid) - expect(size).to.equal(11) - expect(cid.toString()).to.equal(expectedCid) + expect(file).to.have.property('path', expectedCid) + expect(file).to.have.property('size', 11) + expect(`${file.cid}`).to.equal(expectedCid) }) it('should add readable stream', async function () { @@ -185,170 +152,41 @@ module.exports = (common, options) => { rs.push(Buffer.from('some data')) rs.push(null) - const filesAdded = await all(ipfs.add(rs)) - expect(filesAdded).to.be.length(1) + const file = await ipfs.add(rs) - const file = filesAdded[0] - expect(file.path).to.equal(expectedCid) - expect(file.size).to.equal(17) - expect(file.cid.toString()).to.equal(expectedCid) - }) - - it('should add array of objects with readable stream content', async function () { - if (!isNode) this.skip() - const expectedCid = 'QmVv4Wz46JaZJeH5PMV4LGbRiiMKEmszPYY3g6fjGnVXBS' - - const rs = new Readable() - rs.push(Buffer.from('some data')) - rs.push(null) - - const tuple = { path: 'data.txt', content: rs } - - const filesAdded = await all(ipfs.add([tuple])) - expect(filesAdded).to.be.length(1) - - const file = filesAdded[0] - expect(file.path).to.equal('data.txt') - expect(file.size).to.equal(17) - expect(file.cid.toString()).to.equal(expectedCid) - }) - - it('should add a nested directory as array of tupples', async function () { - const content = (name) => ({ - path: `test-folder/${name}`, - content: fixtures.directory.files[name] - }) - - const emptyDir = (name) => ({ path: `test-folder/${name}` }) - - const dirs = [ - content('pp.txt'), - content('holmes.txt'), - content('jungle.txt'), - content('alice.txt'), - emptyDir('empty-folder'), - content('files/hello.txt'), - content('files/ipfs.txt'), - emptyDir('files/empty') - ] - - const root = await last(ipfs.add(dirs)) - - expect(root.path).to.equal('test-folder') - expect(root.cid.toString()).to.equal(fixtures.directory.cid) - }) - - it('should add a nested directory as array of tupples with progress', async function () { - const content = (name) => ({ - path: `test-folder/${name}`, - content: fixtures.directory.files[name] - }) - - const emptyDir = (name) => ({ path: `test-folder/${name}` }) - - const dirs = [ - content('pp.txt'), - content('holmes.txt'), - content('jungle.txt'), - content('alice.txt'), - emptyDir('empty-folder'), - content('files/hello.txt'), - content('files/ipfs.txt'), - emptyDir('files/empty') - ] - - const total = dirs.reduce((i, entry) => { - return i + (entry.content ? entry.content.length : 0) - }, 0) - - let progCalled = false - let accumProgress = 0 - const handler = (p) => { - progCalled = true - accumProgress += p - } - - const root = await last(ipfs.add(dirs, { progress: handler })) - expect(progCalled).to.be.true() - expect(accumProgress).to.be.at.least(total) - expect(root.path).to.equal('test-folder') - expect(root.cid.toString()).to.equal(fixtures.directory.cid) - }) - - it('should add files to a directory non sequentially', async function () { - const content = path => ({ - path: `test-dir/${path}`, - content: fixtures.directory.files[path.split('/').pop()] - }) - - const input = [ - content('a/pp.txt'), - content('a/holmes.txt'), - content('b/jungle.txt'), - content('a/alice.txt') - ] - - const filesAdded = await all(ipfs.add(input)) - - const toPath = ({ path }) => path - const nonSeqDirFilePaths = input.map(toPath).filter(p => p.includes('/a/')) - const filesAddedPaths = filesAdded.map(toPath) - - expect(nonSeqDirFilePaths.every(p => filesAddedPaths.includes(p))).to.be.true() + expect(file).to.have.property('path', expectedCid) + expect(file).to.have.property('size', 17) + expect(`${file.cid}`).to.equal(expectedCid) }) it('should fail when passed invalid input', async () => { const nonValid = 138 - await expect(all(ipfs.add(nonValid))).to.eventually.be.rejected() + await expect(ipfs.add(nonValid)).to.eventually.be.rejected() }) it('should wrap content in a directory', async () => { const data = { path: 'testfile.txt', content: fixtures.smallFile.data } - const filesAdded = await all(ipfs.add(data, { wrapWithDirectory: true })) - expect(filesAdded).to.have.length(2) + const wrapper = await ipfs.add(data, { wrapWithDirectory: true }) + expect(wrapper.path).to.equal('') - const file = filesAdded[0] - const wrapped = filesAdded[1] - expect(file.cid.toString()).to.equal(fixtures.smallFile.cid) - expect(file.path).to.equal('testfile.txt') - expect(wrapped.path).to.equal('') + const stats = await ipfs.files.stat(`/ipfs/${wrapper.cid}/testfile.txt`) + + expect(`${stats.cid}`).to.equal(fixtures.smallFile.cid) }) it('should add with only-hash=true', async function () { this.slow(10 * 1000) const content = String(Math.random() + Date.now()) - const files = await all(ipfs.add(Buffer.from(content), { onlyHash: true })) - expect(files).to.have.length(1) + const file = await ipfs.add(Buffer.from(content), { onlyHash: true }) - await expect(ipfs.object.get(files[0].cid, { timeout: 4000 })) + await expect(ipfs.object.get(file.cid, { timeout: 4000 })) .to.eventually.be.rejected() .and.to.have.property('name').that.equals('TimeoutError') }) - it('should add a directory with only-hash=true', async function () { - this.slow(10 * 1000) - const content = String(Math.random() + Date.now()) - - const files = await all(ipfs.add([{ - path: '/foo/bar.txt', - content: Buffer.from(content) - }, { - path: '/foo/baz.txt', - content: Buffer.from(content) - }], { onlyHash: true })) - expect(files).to.have.length(3) - - await Promise.all( - files.map(file => expect(ipfs.object.get(file.cid, { timeout: 4000 })) - .to.eventually.be.rejected() - .and.to.have.property('name').that.equals('TimeoutError') - ) - ) - }) - it('should add with mode as string', async function () { this.slow(10 * 1000) const mode = '0777' @@ -399,84 +237,19 @@ module.exports = (common, options) => { }) }) - it('should add a directory from the file system', async function () { - if (!isNode) this.skip() - const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'test-folder') - - const result = await all(ipfs.add(globSource(filesPath, { recursive: true }))) - expect(result.length).to.be.above(8) - }) - - it('should add a directory from the file system with an odd name', async function () { - if (!isNode) this.skip() - - const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'weird name folder [v0]') - - const result = await all(ipfs.add(globSource(filesPath, { recursive: true }))) - expect(result.length).to.be.above(8) - }) - - it('should ignore a directory from the file system', async function () { - if (!isNode) this.skip() - - const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'test-folder') - - const result = await all(ipfs.add(globSource(filesPath, { recursive: true, ignore: ['files/**'] }))) - expect(result.length).to.be.below(9) - }) - - it('should add a file from the file system', async function () { - if (!isNode) this.skip() - - const filePath = path.join(__dirname, '..', 'test', 'fixtures', 'testfile.txt') - - const result = await all(ipfs.add(globSource(filePath))) - expect(result.length).to.equal(1) - expect(result[0].path).to.equal('testfile.txt') - }) - - it('should add a hidden file in a directory from the file system', async function () { - if (!isNode) this.skip() - - const filesPath = path.join(__dirname, '..', 'test', 'fixtures', 'hidden-files-folder') - - const result = await all(ipfs.add(globSource(filesPath, { recursive: true, hidden: true }))) - expect(result.length).to.be.above(10) - expect(result.map(object => object.path)).to.include('hidden-files-folder/.hiddenTest.txt') - expect(result.map(object => object.cid.toString())).to.include('QmdbAjVmLRdpFyi8FFvjPfhTGB2cVXvWLuK7Sbt38HXrtt') - }) - - it('should add a file from the file system with only-hash=true', async function () { - if (!isNode) this.skip() - - this.slow(10 * 1000) - - const content = String(Math.random() + Date.now()) - const filepath = path.join(os.tmpdir(), `${content}.txt`) - fs.writeFileSync(filepath, content) - - const out = await all(ipfs.add(globSource(filepath), { onlyHash: true })) - - fs.unlinkSync(filepath) - - await expect(ipfs.object.get(out[0].cid, { timeout: 500 })) - .to.eventually.be.rejected() - .and.to.have.property('name').that.equals('TimeoutError') - }) - it('should add from a HTTP URL', async () => { const text = `TEST${Math.random()}` const url = echoUrl(text) const [result, expectedResult] = await Promise.all([ - all(ipfs.add(urlSource(url))), - all(ipfs.add(Buffer.from(text))) + ipfs.add(urlSource(url)), + ipfs.add(Buffer.from(text)) ]) expect(result.err).to.not.exist() expect(expectedResult.err).to.not.exist() - expect(result[0].cid.toString()).to.equal(expectedResult[0].cid.toString()) - expect(result[0].size).to.equal(expectedResult[0].size) + expect(result.cid.toString()).to.equal(expectedResult.cid.toString()) + expect(result.size).to.equal(expectedResult.size) }) it('should add from a HTTP URL with redirection', async () => { @@ -484,23 +257,23 @@ module.exports = (common, options) => { const url = echoUrl(text) const [result, expectedResult] = await Promise.all([ - all(ipfs.add(urlSource(redirectUrl(url)))), - all(ipfs.add(Buffer.from(text))) + ipfs.add(urlSource(redirectUrl(url))), + ipfs.add(Buffer.from(text)) ]) expect(result.err).to.not.exist() expect(expectedResult.err).to.not.exist() - expect(result[0].cid.toString()).to.equal(expectedResult[0].cid.toString()) - expect(result[0].size).to.equal(expectedResult[0].size) + expect(result.cid.toString()).to.equal(expectedResult.cid.toString()) + expect(result.size).to.equal(expectedResult.size) }) it('should add from a URL with only-hash=true', async function () { const text = `TEST${Math.random()}` const url = echoUrl(text) - const res = await all(ipfs.add(urlSource(url), { onlyHash: true })) + const res = await ipfs.add(urlSource(url), { onlyHash: true }) - await expect(ipfs.object.get(res[0].cid, { timeout: 500 })) + await expect(ipfs.object.get(res.cid, { timeout: 500 })) .to.eventually.be.rejected() .and.to.have.property('name').that.equals('TimeoutError') }) @@ -511,8 +284,8 @@ module.exports = (common, options) => { const addOpts = { wrapWithDirectory: true } const [result, expectedResult] = await Promise.all([ - all(ipfs.add(urlSource(url), addOpts)), - all(ipfs.add([{ path: 'download', content: Buffer.from(filename) }], addOpts)) + ipfs.add(urlSource(url), addOpts), + ipfs.add({ path: 'download', content: Buffer.from(filename) }, addOpts) ]) expect(result.err).to.not.exist() expect(expectedResult.err).to.not.exist() @@ -525,8 +298,8 @@ module.exports = (common, options) => { const addOpts = { wrapWithDirectory: true } const [result, expectedResult] = await Promise.all([ - all(ipfs.add(urlSource(url), addOpts)), - all(ipfs.add([{ path: 'download', content: Buffer.from(filename) }], addOpts)) + ipfs.add(urlSource(url), addOpts), + ipfs.add([{ path: 'download', content: Buffer.from(filename) }], addOpts) ]) expect(result.err).to.not.exist() @@ -535,23 +308,22 @@ module.exports = (common, options) => { }) it('should not add from an invalid url', () => { - return expect(all(ipfs.add(urlSource('123http://invalid')))).to.eventually.be.rejected() + return expect(ipfs.add(urlSource('123http://invalid'))).to.eventually.be.rejected() }) it('should respect raw leaves when file is smaller than one block and no metadata is present', async () => { - const files = await all(ipfs.add(Buffer.from([0, 1, 2]), { + const file = await ipfs.add(Buffer.from([0, 1, 2]), { cidVersion: 1, rawLeaves: true - })) + }) - expect(files.length).to.equal(1) - expect(files[0].cid.toString()).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') - expect(files[0].cid.codec).to.equal('raw') - expect(files[0].size).to.equal(3) + expect(file.cid.toString()).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') + expect(file.cid.codec).to.equal('raw') + expect(file.size).to.equal(3) }) it('should override raw leaves when file is smaller than one block and metadata is present', async () => { - const files = await all(ipfs.add({ + const file = await ipfs.add({ content: Buffer.from([0, 1, 2]), mode: 0o123, mtime: { @@ -561,12 +333,11 @@ module.exports = (common, options) => { }, { cidVersion: 1, rawLeaves: true - })) + }) - expect(files.length).to.equal(1) - expect(files[0].cid.toString()).to.equal('bafybeifmayxiu375ftlgydntjtffy5cssptjvxqw6vyuvtymntm37mpvua') - expect(files[0].cid.codec).to.equal('dag-pb') - expect(files[0].size).to.equal(18) + expect(file.cid.toString()).to.equal('bafybeifmayxiu375ftlgydntjtffy5cssptjvxqw6vyuvtymntm37mpvua') + expect(file.cid.codec).to.equal('dag-pb') + expect(file.size).to.equal(18) }) }) } diff --git a/packages/interface-ipfs-core/src/dht/get.js b/packages/interface-ipfs-core/src/dht/get.js index 21efdf240d..2a02c26d44 100644 --- a/packages/interface-ipfs-core/src/dht/get.js +++ b/packages/interface-ipfs-core/src/dht/get.js @@ -3,7 +3,6 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') -const all = require('it-all') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -27,7 +26,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when getting a value from the DHT', async () => { - const [data] = await all(nodeA.add('should put a value to the DHT')) + const data = await nodeA.add('should put a value to the DHT') const publish = await nodeA.name.publish(data.cid) await testTimeout(() => nodeB.dht.get(`/ipns/${publish.name}`, { @@ -41,7 +40,7 @@ module.exports = (common, options) => { }) it('should get a value after it was put on another node', async () => { - const [data] = await all(nodeA.add('should put a value to the DHT')) + const data = await nodeA.add('should put a value to the DHT') const publish = await nodeA.name.publish(data.cid) const record = await nodeA.dht.get(`/ipns/${publish.name}`) diff --git a/packages/interface-ipfs-core/src/dht/provide.js b/packages/interface-ipfs-core/src/dht/provide.js index 699a3cdfb8..425e37f960 100644 --- a/packages/interface-ipfs-core/src/dht/provide.js +++ b/packages/interface-ipfs-core/src/dht/provide.js @@ -6,7 +6,6 @@ const CID = require('cids') const all = require('it-all') const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') -const last = require('it-last') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -31,7 +30,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when providing a value on the DHT', async () => { - const res = await last(ipfs.add(Buffer.from('test'))) + const res = await ipfs.add(Buffer.from('test')) await testTimeout(() => ipfs.dht.provide(res.cid, { timeout: 1 @@ -39,9 +38,9 @@ module.exports = (common, options) => { }) it('should provide local CID', async () => { - const res = await all(ipfs.add(Buffer.from('test'))) + const res = await ipfs.add(Buffer.from('test')) - await all(ipfs.dht.provide(res[0].cid)) + await all(ipfs.dht.provide(res.cid)) }) it('should not provide if block not found locally', () => { @@ -54,7 +53,7 @@ module.exports = (common, options) => { }) it('should allow multiple CIDs to be passed', async () => { - const res = await all(ipfs.add([ + const res = await all(ipfs.addAll([ { content: Buffer.from('t0') }, { content: Buffer.from('t1') } ])) @@ -63,8 +62,8 @@ module.exports = (common, options) => { }) it('should provide a CIDv1', async () => { - const res = await all(ipfs.add(Buffer.from('test'), { cidVersion: 1 })) - await all(ipfs.dht.provide(res[0].cid)) + const res = await ipfs.add(Buffer.from('test'), { cidVersion: 1 }) + await all(ipfs.dht.provide(res.cid)) }) it('should error on non CID arg', () => { diff --git a/packages/interface-ipfs-core/src/dht/put.js b/packages/interface-ipfs-core/src/dht/put.js index 27c7facc48..4721fd3a28 100644 --- a/packages/interface-ipfs-core/src/dht/put.js +++ b/packages/interface-ipfs-core/src/dht/put.js @@ -4,7 +4,6 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const testTimeout = require('../utils/test-timeout') const all = require('it-all') -const last = require('it-last') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ /** @@ -28,7 +27,7 @@ module.exports = (common, options) => { after(() => common.clean()) it('should respect timeout option when putting a value into the DHT', async () => { - const { cid } = await last(nodeA.add('should respect timeout option when putting a value into the DH')) + const { cid } = await nodeA.add('should respect timeout option when putting a value into the DH') const publish = await nodeA.name.publish(cid) const record = await nodeA.dht.get(`/ipns/${publish.name}`) @@ -38,7 +37,7 @@ module.exports = (common, options) => { }) it('should put a value to the DHT', async function () { - const { cid } = await last(nodeA.add('should put a value to the DHT')) + const { cid } = await nodeA.add('should put a value to the DHT') const publish = await nodeA.name.publish(cid) const record = await nodeA.dht.get(`/ipns/${publish.name}`) const value = await all(nodeA.dht.put(`/ipns/${publish.name}`, record, { verbose: true })) diff --git a/packages/interface-ipfs-core/src/files/cp.js b/packages/interface-ipfs-core/src/files/cp.js index 0fbcd0f6ae..2fffabba5c 100644 --- a/packages/interface-ipfs-core/src/files/cp.js +++ b/packages/interface-ipfs-core/src/files/cp.js @@ -426,13 +426,13 @@ module.exports = (common, options) => { nsecs: (mtime - (seconds * 1000)) * 1000 } - const [{ + const { cid - }] = await all(ipfs.add({ + } = await ipfs.add({ content: fixtures.smallFile.data, mode, mtime - })) + }) await ipfs.files.cp(`/ipfs/${cid}`, testDestPath) const stats = await ipfs.files.stat(testDestPath) diff --git a/packages/interface-ipfs-core/src/files/read.js b/packages/interface-ipfs-core/src/files/read.js index adfbfde7a4..baa313143e 100644 --- a/packages/interface-ipfs-core/src/files/read.js +++ b/packages/interface-ipfs-core/src/files/read.js @@ -3,7 +3,6 @@ const { Buffer } = require('buffer') const concat = require('it-concat') -const all = require('it-all') const drain = require('it-drain') const { fixtures } = require('../utils') const { getDescribe, getIt, expect } = require('../utils/mocha') @@ -153,7 +152,7 @@ module.exports = (common, options) => { }) it('should read from outside of mfs', async () => { - const [{ cid }] = await all(ipfs.add(fixtures.smallFile.data)) + const { cid } = await ipfs.add(fixtures.smallFile.data) const testFileData = await concat(ipfs.files.read(`/ipfs/${cid}`)) expect(testFileData.slice()).to.eql(fixtures.smallFile.data) }) diff --git a/packages/interface-ipfs-core/src/files/stat.js b/packages/interface-ipfs-core/src/files/stat.js index 072d2a4e58..ab96ee337d 100644 --- a/packages/interface-ipfs-core/src/files/stat.js +++ b/packages/interface-ipfs-core/src/files/stat.js @@ -3,7 +3,6 @@ const { Buffer } = require('buffer') const { nanoid } = require('nanoid') -const all = require('it-all') const { fixtures } = require('../utils') const { getDescribe, getIt, expect } = require('../utils/mocha') const createShardedDirectory = require('../utils/create-sharded-directory') @@ -36,7 +35,7 @@ module.exports = (common, options) => { })).api }) - before(async () => { await all(ipfs.add(fixtures.smallFile.data)) }) + before(async () => { await ipfs.add(fixtures.smallFile.data) }) after(() => common.clean()) diff --git a/packages/interface-ipfs-core/src/index.js b/packages/interface-ipfs-core/src/index.js index 3e8bd774b6..6ce8693ad5 100644 --- a/packages/interface-ipfs-core/src/index.js +++ b/packages/interface-ipfs-core/src/index.js @@ -4,6 +4,7 @@ const { createSuite } = require('./utils/suite') exports.root = createSuite({ add: require('./add'), + addAll: require('./add-all'), cat: require('./cat'), get: require('./get'), ls: require('./ls'), diff --git a/packages/interface-ipfs-core/src/ls.js b/packages/interface-ipfs-core/src/ls.js index 9d237ac1fe..d1ba733631 100644 --- a/packages/interface-ipfs-core/src/ls.js +++ b/packages/interface-ipfs-core/src/ls.js @@ -56,7 +56,7 @@ module.exports = (common, options) => { emptyDir('files/empty') ] - const res = await all(ipfs.add(dirs)) + const res = await all(ipfs.addAll(dirs)) const root = res[res.length - 1] expect(root.path).to.equal('test-folder') @@ -117,7 +117,7 @@ module.exports = (common, options) => { { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } ] - const res = await all(ipfs.add(input, { cidVersion: 0 })) + const res = await all(ipfs.addAll(input, { cidVersion: 0 })) const cidv0 = res[res.length - 1].cid expect(cidv0.version).to.equal(0) @@ -140,7 +140,7 @@ module.exports = (common, options) => { { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } ] - const res = await all(ipfs.add(input, { cidVersion: 1, rawLeaves: false })) + const res = await all(ipfs.addAll(input, { cidVersion: 1, rawLeaves: false })) const cidv1 = res[res.length - 1].cid expect(cidv1.version).to.equal(1) @@ -171,7 +171,7 @@ module.exports = (common, options) => { { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')) } ] - const res = await all(ipfs.add(input)) + const res = await all(ipfs.addAll(input)) const output = await all(ipfs.ls(`/ipfs/${res[res.length - 1].cid}`)) expect(output.length).to.equal(input.length) @@ -195,7 +195,7 @@ module.exports = (common, options) => { { path: `${dir}/${randomName('F1')}`, content: Buffer.from(randomName('D1')), mode, mtime } ] - const res = await all(ipfs.add(input)) + const res = await all(ipfs.addAll(input)) const output = await all(ipfs.ls(`/ipfs/${res[res.length - 1].cid}`)) expect(output).to.have.lengthOf(input.length) @@ -210,12 +210,10 @@ module.exports = (common, options) => { const subdir = randomName('F0') const subfile = randomName('F1') - const input = [ - { path: `${dir}/${subdir}/${subfile}`, content: Buffer.from(randomName('D1')) } - ] + const input = { path: `${dir}/${subdir}/${subfile}`, content: Buffer.from(randomName('D1')) } - const res = await all(ipfs.add(input)) - const path = `${res[res.length - 1].cid}/${subdir}` + const res = await ipfs.add(input) + const path = `${res.cid}/${subdir}` const output = await all(ipfs.ls(path)) expect(output).to.have.lengthOf(1) diff --git a/packages/interface-ipfs-core/src/miscellaneous/resolve.js b/packages/interface-ipfs-core/src/miscellaneous/resolve.js index f3d7250bf5..a773529491 100644 --- a/packages/interface-ipfs-core/src/miscellaneous/resolve.js +++ b/packages/interface-ipfs-core/src/miscellaneous/resolve.js @@ -39,13 +39,13 @@ module.exports = (common, options) => { it('should resolve an IPFS hash', async () => { const content = loadFixture('test/fixtures/testfile.txt', 'interface-ipfs-core') - const [{ cid }] = await all(ipfs.add(content)) + const { cid } = await ipfs.add(content) const path = await ipfs.resolve(`/ipfs/${cid}`) expect(path).to.equal(`/ipfs/${cid}`) }) it('should resolve an IPFS hash and return a base64url encoded CID in path', async () => { - const [{ cid }] = await all(ipfs.add(Buffer.from('base64url encoded'))) + const { cid } = await ipfs.add(Buffer.from('base64url encoded')) const path = await ipfs.resolve(`/ipfs/${cid}`, { cidBase: 'base64url' }) const [,, cidStr] = path.split('/') @@ -56,7 +56,7 @@ module.exports = (common, options) => { it('should resolve an IPFS path link', async () => { const path = 'path/to/testfile.txt' const content = loadFixture('test/fixtures/testfile.txt', 'interface-ipfs-core') - const [{ cid: fileCid }, , , { cid: rootCid }] = await all(ipfs.add([{ path, content }], { wrapWithDirectory: true })) + const [{ cid: fileCid }, , , { cid: rootCid }] = await all(ipfs.addAll([{ path, content }], { wrapWithDirectory: true })) const resolve = await ipfs.resolve(`/ipfs/${rootCid}/${path}`) expect(resolve).to.equal(`/ipfs/${fileCid}`) @@ -94,7 +94,7 @@ module.exports = (common, options) => { // webworkers are not dialable because webrtc is not available const node = (await common.spawn({ type: isWebWorker ? 'go' : undefined })).api await ipfs.swarm.connect(node.peerId.addresses[0]) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record recursive === true'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === true')) const { id: keyId } = await ipfs.key.gen('key-name', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) diff --git a/packages/interface-ipfs-core/src/name/publish.js b/packages/interface-ipfs-core/src/name/publish.js index b860411cc7..c2af18be63 100644 --- a/packages/interface-ipfs-core/src/name/publish.js +++ b/packages/interface-ipfs-core/src/name/publish.js @@ -5,7 +5,6 @@ const { nanoid } = require('nanoid') const { Buffer } = require('buffer') const { fixture } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') -const all = require('it-all') const last = require('it-last') const testTimeout = require('../utils/test-timeout') @@ -26,7 +25,7 @@ module.exports = (common, options) => { before(async () => { ipfs = (await common.spawn()).api nodeId = ipfs.peerId.id - await all(ipfs.add(fixture.data, { pin: false })) + await ipfs.add(fixture.data, { pin: false }) }) after(() => common.clean()) @@ -50,7 +49,7 @@ module.exports = (common, options) => { }) it('should publish correctly with the lifetime option and resolve', async () => { - const [{ path }] = await all(ipfs.add(Buffer.from('should publish correctly with the lifetime option and resolve'))) + const { path } = await ipfs.add(Buffer.from('should publish correctly with the lifetime option and resolve')) await ipfs.name.publish(path, { allowOffline: true, resolve: false, lifetime: '2h' }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}`))).to.eq(`/ipfs/${path}`) }) diff --git a/packages/interface-ipfs-core/src/name/resolve.js b/packages/interface-ipfs-core/src/name/resolve.js index f805d35ffc..8465bdc6bf 100644 --- a/packages/interface-ipfs-core/src/name/resolve.js +++ b/packages/interface-ipfs-core/src/name/resolve.js @@ -5,7 +5,6 @@ const { Buffer } = require('buffer') const { getDescribe, getIt, expect } = require('../utils/mocha') const delay = require('delay') const CID = require('cids') -const all = require('it-all') const last = require('it-last') /** @typedef { import("ipfsd-ctl/src/factory") } Factory */ @@ -31,7 +30,7 @@ module.exports = (common, options) => { it('should resolve a record default options', async function () { this.timeout(20 * 1000) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record default options'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record default options')) const { id: keyId } = await ipfs.key.gen('key-name-default', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -43,7 +42,7 @@ module.exports = (common, options) => { it('should resolve a record from peerid as cidv1 in base32', async function () { this.timeout(20 * 1000) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record from cidv1b32'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record from cidv1b32')) const { id: peerId } = await ipfs.id() await ipfs.name.publish(path, { allowOffline: true }) @@ -56,7 +55,7 @@ module.exports = (common, options) => { }) it('should resolve a record recursive === false', async () => { - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record recursive === false'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === false')) await ipfs.name.publish(path, { allowOffline: true }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}`, { recursive: false }))) .to.eq(`/ipfs/${path}`) @@ -65,7 +64,7 @@ module.exports = (common, options) => { it('should resolve a record recursive === true', async function () { this.timeout(20 * 1000) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record recursive === true'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record recursive === true')) const { id: keyId } = await ipfs.key.gen('key-name', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -78,7 +77,7 @@ module.exports = (common, options) => { it('should resolve a record default options with remainder', async function () { this.timeout(20 * 1000) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record default options with remainder'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record default options with remainder')) const { id: keyId } = await ipfs.key.gen('key-name-remainder-default', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -89,7 +88,7 @@ module.exports = (common, options) => { }) it('should resolve a record recursive === false with remainder', async () => { - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record recursive = false with remainder'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record recursive = false with remainder')) await ipfs.name.publish(path, { allowOffline: true }) expect(await last(ipfs.name.resolve(`/ipns/${nodeId}/remainder/file.txt`, { recursive: false }))) .to.eq(`/ipfs/${path}/remainder/file.txt`) @@ -98,7 +97,7 @@ module.exports = (common, options) => { it('should resolve a record recursive === true with remainder', async function () { this.timeout(20 * 1000) - const [{ path }] = await all(ipfs.add(Buffer.from('should resolve a record recursive = true with remainder'))) + const { path } = await ipfs.add(Buffer.from('should resolve a record recursive = true with remainder')) const { id: keyId } = await ipfs.key.gen('key-name-remainder', { type: 'rsa', size: 2048 }) await ipfs.name.publish(path, { allowOffline: true }) @@ -116,7 +115,7 @@ module.exports = (common, options) => { } // we add new data instead of re-using fixture to make sure lifetime handling doesn't break - const [{ path }] = await all(ipfs.add(Buffer.from('should not get the entry if its validity time expired'))) + const { path } = await ipfs.add(Buffer.from('should not get the entry if its validity time expired')) await ipfs.name.publish(path, publishOptions) await delay(500) // go only has 1 possible error https://github.com/ipfs/go-ipfs/blob/master/namesys/interface.go#L51 diff --git a/packages/interface-ipfs-core/src/object/get.js b/packages/interface-ipfs-core/src/object/get.js index 22ae7bc6ce..85a8246a3b 100644 --- a/packages/interface-ipfs-core/src/object/get.js +++ b/packages/interface-ipfs-core/src/object/get.js @@ -9,7 +9,6 @@ const { getDescribe, getIt, expect } = require('../utils/mocha') const UnixFs = require('ipfs-unixfs') const randomBytes = require('iso-random-stream/src/random') const { asDAGLink } = require('./utils') -const all = require('it-all') const testTimeout = require('../utils/test-timeout') const CID = require('cids') @@ -142,12 +141,12 @@ module.exports = (common, options) => { // has to be big enough to span several DAGNodes const data = randomBytes(1024 * 3000) - const result = await all(ipfs.add({ + const result = await ipfs.add({ path: '', content: data - })) + }) - const node = await ipfs.object.get(result[0].cid) + const node = await ipfs.object.get(result.cid) const meta = UnixFs.unmarshal(node.Data) expect(meta.fileSize()).to.equal(data.length) diff --git a/packages/interface-ipfs-core/src/object/links.js b/packages/interface-ipfs-core/src/object/links.js index ee055d66ae..f2dd984c80 100644 --- a/packages/interface-ipfs-core/src/object/links.js +++ b/packages/interface-ipfs-core/src/object/links.js @@ -7,7 +7,6 @@ const DAGNode = dagPB.DAGNode const { nanoid } = require('nanoid') const { getDescribe, getIt, expect } = require('../utils/mocha') const { asDAGLink } = require('./utils') -const all = require('it-all') const testTimeout = require('../utils/test-timeout') const CID = require('cids') @@ -96,11 +95,11 @@ module.exports = (common, options) => { it('should get links from CBOR object', async () => { const hashes = [] - const res1 = await all(ipfs.add(Buffer.from('test data'))) - hashes.push(res1[0].cid) + const res1 = await ipfs.add(Buffer.from('test data')) + hashes.push(res1.cid) - const res2 = await all(ipfs.add(Buffer.from('more test data'))) - hashes.push(res2[0].cid) + const res2 = await ipfs.add(Buffer.from('more test data')) + hashes.push(res2.cid) const obj = { some: 'data', diff --git a/packages/interface-ipfs-core/src/pin/add.js b/packages/interface-ipfs-core/src/pin/add.js index df8a4a6fb2..ce60210d4f 100644 --- a/packages/interface-ipfs-core/src/pin/add.js +++ b/packages/interface-ipfs-core/src/pin/add.js @@ -3,7 +3,6 @@ const { fixtures } = require('./utils') const { getDescribe, getIt, expect } = require('../utils/mocha') -const all = require('it-all') const testTimeout = require('../utils/test-timeout') const CID = require('cids') @@ -23,7 +22,7 @@ module.exports = (common, options) => { before(async () => { ipfs = (await common.spawn()).api await Promise.all(fixtures.files.map(file => { - return all(ipfs.add(file.data, { pin: false })) + return ipfs.add(file.data, { pin: false }) })) }) diff --git a/packages/interface-ipfs-core/src/pin/ls.js b/packages/interface-ipfs-core/src/pin/ls.js index 7ca8f1c565..0957942e25 100644 --- a/packages/interface-ipfs-core/src/pin/ls.js +++ b/packages/interface-ipfs-core/src/pin/ls.js @@ -24,13 +24,13 @@ module.exports = (common, options) => { ipfs = (await common.spawn()).api // two files wrapped in directories, only root CID pinned recursively const dir = fixtures.directory.files.map((file) => ({ path: file.path, content: file.data })) - await all(ipfs.add(dir, { pin: false, cidVersion: 0 })) + await all(ipfs.addAll(dir, { pin: false, cidVersion: 0 })) await ipfs.pin.add(fixtures.directory.cid, { recursive: true }) // a file (CID pinned recursively) - await all(ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 })) + await ipfs.add(fixtures.files[0].data, { pin: false, cidVersion: 0 }) await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) // a single CID (pinned directly) - await all(ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 })) + await ipfs.add(fixtures.files[1].data, { pin: false, cidVersion: 0 }) await ipfs.pin.add(fixtures.files[1].cid, { recursive: false }) }) diff --git a/packages/interface-ipfs-core/src/pin/rm.js b/packages/interface-ipfs-core/src/pin/rm.js index e89e70f697..6923b914c7 100644 --- a/packages/interface-ipfs-core/src/pin/rm.js +++ b/packages/interface-ipfs-core/src/pin/rm.js @@ -21,9 +21,9 @@ module.exports = (common, options) => { let ipfs beforeEach(async () => { ipfs = (await common.spawn()).api - await all(ipfs.add(fixtures.files[0].data, { pin: false })) + await ipfs.add(fixtures.files[0].data, { pin: false }) await ipfs.pin.add(fixtures.files[0].cid, { recursive: true }) - await all(ipfs.add(fixtures.files[1].data, { pin: false })) + await ipfs.add(fixtures.files[1].data, { pin: false }) await ipfs.pin.add(fixtures.files[1].cid, { recursive: false }) }) diff --git a/packages/interface-ipfs-core/src/repo/gc.js b/packages/interface-ipfs-core/src/repo/gc.js index ce305573b5..b8ee709f2a 100644 --- a/packages/interface-ipfs-core/src/repo/gc.js +++ b/packages/interface-ipfs-core/src/repo/gc.js @@ -33,16 +33,16 @@ module.exports = (common, options) => { }) it('should run garbage collection', async () => { - const res = await all(ipfs.add(Buffer.from('apples'))) + const res = await ipfs.add(Buffer.from('apples')) const pinset = await all(ipfs.pin.ls()) - expect(pinset.map(obj => obj.cid.toString())).includes(res[0].cid.toString()) + expect(pinset.map(obj => obj.cid.toString())).includes(res.cid.toString()) - await ipfs.pin.rm(res[0].cid) + await ipfs.pin.rm(res.cid) await all(ipfs.repo.gc()) const finalPinset = await all(ipfs.pin.ls()) - expect(finalPinset.map(obj => obj.cid.toString())).not.includes(res[0].cid.toString()) + expect(finalPinset.map(obj => obj.cid.toString())).not.includes(res.cid.toString()) }) it('should clean up unpinned data', async () => { @@ -52,8 +52,8 @@ module.exports = (common, options) => { // Add some data. Note: this will implicitly pin the data, which causes // some blocks to be added for the data itself and for the pinning // information that refers to the blocks - const addRes = await all(ipfs.add(Buffer.from('apples'))) - const cid = addRes[0].cid + const addRes = await ipfs.add(Buffer.from('apples')) + const cid = addRes.cid // Get the list of local blocks after the add, should be bigger than // the initial list and contain hash @@ -128,8 +128,8 @@ module.exports = (common, options) => { const block = await ipfs.block.get(mfsFileCid) // Add the data to IPFS (which implicitly pins the data) - const addRes = await all(ipfs.add(block.data)) - const dataCid = addRes[0].cid + const addRes = await ipfs.add(block.data) + const dataCid = addRes.cid // Get the list of local blocks after the add, should be bigger than // the initial list and contain the data hash @@ -174,8 +174,8 @@ module.exports = (common, options) => { const refsBeforeAdd = await all(ipfs.refs.local()) // Add some data - const addRes = await all(ipfs.add(Buffer.from('pears'))) - const dataCid = addRes[0].cid + const addRes = await ipfs.add(Buffer.from('pears')) + const dataCid = addRes.cid // Unpin the data await ipfs.pin.rm(dataCid) @@ -184,7 +184,7 @@ module.exports = (common, options) => { const obj = await new DAGNode(Buffer.from('fruit'), [{ Name: 'p', Hash: dataCid, - Tsize: addRes[0].size + Tsize: addRes.size }]) // Put the object into IPFS diff --git a/packages/interface-ipfs-core/src/utils/create-sharded-directory.js b/packages/interface-ipfs-core/src/utils/create-sharded-directory.js index 4d995fdd58..26c9d5f0ab 100644 --- a/packages/interface-ipfs-core/src/utils/create-sharded-directory.js +++ b/packages/interface-ipfs-core/src/utils/create-sharded-directory.js @@ -8,7 +8,7 @@ const last = require('it-last') module.exports = async (ipfs, files = 1001) => { const dirPath = `/sharded-dir-${Math.random()}` - const result = await last(ipfs.add(function * () { + const result = await last(ipfs.addAll(function * () { for (let i = 0; i < files; i++) { yield { path: `${dirPath}/file-${i}`, diff --git a/packages/interface-ipfs-core/src/utils/create-two-shards.js b/packages/interface-ipfs-core/src/utils/create-two-shards.js index dc19a02fae..73728e9755 100644 --- a/packages/interface-ipfs-core/src/utils/create-two-shards.js +++ b/packages/interface-ipfs-core/src/utils/create-two-shards.js @@ -21,11 +21,11 @@ const createTwoShards = async (ipfs, fileCount) => { })) const nextFile = someFiles.pop() - const { cid: dirWithAllFiles } = await last(ipfs.add(allFiles, { + const { cid: dirWithAllFiles } = await last(ipfs.addAll(allFiles, { // for js-ipfs - go-ipfs shards everything when sharding is turned on shardSplitThreshold: files.length - 1 })) - const { cid: dirWithSomeFiles } = await last(ipfs.add(someFiles, { + const { cid: dirWithSomeFiles } = await last(ipfs.addAll(someFiles, { // for js-ipfs - go-ipfs shards everything when sharding is turned on shardSplitThreshold: files.length - 1 })) diff --git a/packages/ipfs-http-client/README.md b/packages/ipfs-http-client/README.md index 1a9b837ff9..4bfad08055 100644 --- a/packages/ipfs-http-client/README.md +++ b/packages/ipfs-http-client/README.md @@ -147,9 +147,9 @@ const IpfsHttpClient = require('ipfs-http-client') const { globSource } = IpfsHttpClient const ipfs = IpfsHttpClient() -for await (const file of ipfs.add(globSource('./docs', { recursive: true }))) { - console.log(file) -} +const file = await ipfs.add(globSource('./docs', { recursive: true })) +console.log(file) + /* { path: 'docs/assets/anchor.js', @@ -182,9 +182,9 @@ const IpfsHttpClient = require('ipfs-http-client') const { urlSource } = IpfsHttpClient const ipfs = IpfsHttpClient() -for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg'))) { - console.log(file) -} +const file = await ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg')) +console.log(file) + /* { path: 'ipfs-logo.svg', diff --git a/packages/ipfs-http-client/examples/bundle-webpack/src/App.js b/packages/ipfs-http-client/examples/bundle-webpack/src/App.js index bc25b5b810..c1099f367e 100644 --- a/packages/ipfs-http-client/examples/bundle-webpack/src/App.js +++ b/packages/ipfs-http-client/examples/bundle-webpack/src/App.js @@ -24,19 +24,17 @@ class App extends React.Component { protocol_version: id.protocolVersion }) - const source = ipfs.add(stringToUse) - for await (const file of source) { - console.log("TCL: App -> forawait -> file", file) - const hash = file.path - this.setState({ added_file_hash: hash }) + const file = ipfs.add(stringToUse) + console.log("TCL: App -> forawait -> file", file) + const cid = file.cid + this.setState({ added_file_hash: `${cid}` }) - const source = ipfs.cat(hash) - const data = [] - for await (const chunk of source) { - data.push(chunk) - } - this.setState({ added_file_contents: Buffer.concat(data).toString() }) + const source = ipfs.cat(cid) + const data = [] + for await (const chunk of source) { + data.push(chunk) } + this.setState({ added_file_contents: Buffer.concat(data).toString() }) } render () { return
diff --git a/packages/ipfs-http-client/examples/name-api/index.js b/packages/ipfs-http-client/examples/name-api/index.js index 10d7758b43..7c64821d8d 100644 --- a/packages/ipfs-http-client/examples/name-api/index.js +++ b/packages/ipfs-http-client/examples/name-api/index.js @@ -58,13 +58,12 @@ const addAndPublish = async (e) => { showStatus('adding to IPFS...', COLORS.active) try { - for await (const file of ipfs.add(buffer)) { - showStatus('success!', COLORS.success) + const file = await ipfs.add(buffer) + showStatus('success!', COLORS.success) - publish(file.path) + publish(file.path) - input.value = '' - } + input.value = '' } catch (err) { showStatus('failed to add the data', COLORS.error) console.error(err) diff --git a/packages/ipfs-http-client/examples/upload-file-via-browser/src/App.js b/packages/ipfs-http-client/examples/upload-file-via-browser/src/App.js index 16e48ef43b..427e46fce3 100644 --- a/packages/ipfs-http-client/examples/upload-file-via-browser/src/App.js +++ b/packages/ipfs-http-client/examples/upload-file-via-browser/src/App.js @@ -30,17 +30,15 @@ class App extends React.Component { // Example #1 // Add file to IPFS and return a CID async saveToIpfs (files) { - const source = this.ipfs.add( - [...files], - { - progress: (prog) => console.log(`received: ${prog}`) - } - ) try { - for await (const file of source) { - console.log(file) - this.setState({ added_file_hash: file.path }) - } + const file = this.ipfs.add( + [...files], + { + progress: (prog) => console.log(`received: ${prog}`) + } + ) + console.log(file) + this.setState({ added_file_hash: file.path }) } catch (err) { console.error(err) } @@ -59,12 +57,10 @@ class App extends React.Component { progress: (prog) => console.log(`received: ${prog}`) } - const source = this.ipfs.add(fileDetails, options) try { - for await (const file of source) { - console.log(file) - this.setState({ added_file_hash: file.cid.toString() }) - } + const file = await this.ipfs.add(fileDetails, options) + console.log(file) + this.setState({ added_file_hash: file.cid.toString() }) } catch (err) { console.error(err) } diff --git a/packages/ipfs-http-client/src/add-all.js b/packages/ipfs-http-client/src/add-all.js new file mode 100644 index 0000000000..7e819ebd24 --- /dev/null +++ b/packages/ipfs-http-client/src/add-all.js @@ -0,0 +1,63 @@ +'use strict' + +const CID = require('cids') +const toCamel = require('./lib/object-to-camel') +const configure = require('./lib/configure') +const multipartRequest = require('./lib/multipart-request') +const toUrlSearchParams = require('./lib/to-url-search-params') +const anySignal = require('any-signal') +const AbortController = require('abort-controller') + +module.exports = configure((api) => { + return async function * addAll (input, options = {}) { + const progressFn = options.progress + + // allow aborting requests on body errors + const controller = new AbortController() + const signal = anySignal([controller.signal, options.signal]) + + const res = await api.post('add', { + searchParams: toUrlSearchParams({ + 'stream-channels': true, + ...options, + progress: Boolean(progressFn) + }), + timeout: options.timeout, + signal, + ...( + await multipartRequest(input, controller, options.headers) + ) + }) + + for await (let file of res.ndjson()) { + file = toCamel(file) + + if (progressFn && file.bytes) { + progressFn(file.bytes) + } else { + yield toCoreInterface(file) + } + } + } +}) + +function toCoreInterface ({ name, hash, size, mode, mtime, mtimeNsecs }) { + const output = { + path: name, + cid: new CID(hash), + size: parseInt(size) + } + + if (mode != null) { + output.mode = parseInt(mode, 8) + } + + if (mtime != null) { + output.mtime = { + secs: mtime, + nsecs: mtimeNsecs || 0 + } + } + + return output +} diff --git a/packages/ipfs-http-client/src/add.js b/packages/ipfs-http-client/src/add.js index d399abb4a1..3190b9a7d9 100644 --- a/packages/ipfs-http-client/src/add.js +++ b/packages/ipfs-http-client/src/add.js @@ -1,63 +1,15 @@ 'use strict' -const CID = require('cids') -const toCamel = require('./lib/object-to-camel') +const addAll = require('./add-all') +const last = require('it-last') const configure = require('./lib/configure') -const multipartRequest = require('./lib/multipart-request') -const toUrlSearchParams = require('./lib/to-url-search-params') -const anySignal = require('any-signal') -const AbortController = require('abort-controller') -module.exports = configure((api) => { - return async function * add (input, options = {}) { - const progressFn = options.progress +module.exports = (options) => { + const all = addAll(options) - // allow aborting requests on body errors - const controller = new AbortController() - const signal = anySignal([controller.signal, options.signal]) - - const res = await api.post('add', { - searchParams: toUrlSearchParams({ - 'stream-channels': true, - ...options, - progress: Boolean(progressFn) - }), - timeout: options.timeout, - signal, - ...( - await multipartRequest(input, controller, options.headers) - ) - }) - - for await (let file of res.ndjson()) { - file = toCamel(file) - - if (progressFn && file.bytes) { - progressFn(file.bytes) - } else { - yield toCoreInterface(file) - } - } - } -}) - -function toCoreInterface ({ name, hash, size, mode, mtime, mtimeNsecs }) { - const output = { - path: name, - cid: new CID(hash), - size: parseInt(size) - } - - if (mode != null) { - output.mode = parseInt(mode, 8) - } - - if (mtime != null) { - output.mtime = { - secs: mtime, - nsecs: mtimeNsecs || 0 + return configure(() => { + return async function add (input, options = {}) { // eslint-disable-line require-await + return last(all(input, options)) } - } - - return output + })(options) } diff --git a/packages/ipfs-http-client/src/index.js b/packages/ipfs-http-client/src/index.js index 850575efc5..aeb82ee703 100644 --- a/packages/ipfs-http-client/src/index.js +++ b/packages/ipfs-http-client/src/index.js @@ -21,6 +21,7 @@ const urlSource = require('ipfs-utils/src/files/url-source') function ipfsClient (options = {}) { return { add: require('./add')(options), + addAll: require('./add-all')(options), bitswap: require('./bitswap')(options), block: require('./block')(options), bootstrap: require('./bootstrap')(options), diff --git a/packages/ipfs-http-client/test/get.spec.js b/packages/ipfs-http-client/test/get.spec.js index 41bdeb401c..dd2513c450 100644 --- a/packages/ipfs-http-client/test/get.spec.js +++ b/packages/ipfs-http-client/test/get.spec.js @@ -27,7 +27,7 @@ describe('.get (specific go-ipfs features)', function () { before(async () => { ipfs = (await f.spawn()).api - await all(ipfs.add(smallFile.data)) + await ipfs.add(smallFile.data) }) after(() => f.clean()) @@ -65,7 +65,7 @@ describe('.get (specific go-ipfs features)', function () { const subdir = 'tmp/c++files' const expectedCid = 'QmPkmARcqjo5fqK1V1o8cFsuaXxWYsnwCNLJUYS4KeZyff' const path = `${subdir}/${filename}` - const files = await all(ipfs.add([{ + const files = await all(ipfs.addAll([{ path, content: Buffer.from(path) }])) diff --git a/packages/ipfs-http-client/test/log.spec.js b/packages/ipfs-http-client/test/log.spec.js index b408f75519..e41ea9b03c 100644 --- a/packages/ipfs-http-client/test/log.spec.js +++ b/packages/ipfs-http-client/test/log.spec.js @@ -3,7 +3,6 @@ 'use strict' const { expect } = require('interface-ipfs-core/src/utils/mocha') -const all = require('it-all') const { Buffer } = require('buffer') const f = require('./utils/factory')() @@ -21,7 +20,7 @@ describe('.log', function () { it('.log.tail', async () => { const i = setInterval(async () => { try { - await all(ipfs.add(Buffer.from('just adding some data to generate logs'))) + await ipfs.add(Buffer.from('just adding some data to generate logs')) } catch (_) { // this can error if the test has finished and we're shutting down the node } diff --git a/packages/ipfs/src/cli/commands/add.js b/packages/ipfs/src/cli/commands/add.js index e764e3f30c..035ec481b8 100644 --- a/packages/ipfs/src/cli/commands/add.js +++ b/packages/ipfs/src/cli/commands/add.js @@ -259,7 +259,7 @@ module.exports = { let finalCid try { - for await (const added of ipfs.add(source, options)) { + for await (const added of ipfs.addAll(source, options)) { if (silent) { continue } diff --git a/packages/ipfs/src/core/components/add/index.js b/packages/ipfs/src/core/components/add-all/index.js similarity index 97% rename from packages/ipfs/src/core/components/add/index.js rename to packages/ipfs/src/core/components/add-all/index.js index 86098cd730..f72c337439 100644 --- a/packages/ipfs/src/core/components/add/index.js +++ b/packages/ipfs/src/core/components/add-all/index.js @@ -9,7 +9,7 @@ const { withTimeoutOption } = require('../../utils') module.exports = ({ block, gcLock, preload, pin, options: constructorOptions }) => { const isShardingEnabled = constructorOptions.EXPERIMENTAL && constructorOptions.EXPERIMENTAL.sharding - return withTimeoutOption(async function * add (source, options) { + return withTimeoutOption(async function * addAll (source, options) { options = options || {} const opts = { diff --git a/packages/ipfs/src/core/components/add/utils.js b/packages/ipfs/src/core/components/add-all/utils.js similarity index 100% rename from packages/ipfs/src/core/components/add/utils.js rename to packages/ipfs/src/core/components/add-all/utils.js diff --git a/packages/ipfs/src/core/components/add.js b/packages/ipfs/src/core/components/add.js new file mode 100644 index 0000000000..89a240df63 --- /dev/null +++ b/packages/ipfs/src/core/components/add.js @@ -0,0 +1,9 @@ +'use strict' + +const last = require('it-last') + +module.exports = ({ addAll }) => { + return async function add (source, options) { // eslint-disable-line require-await + return last(addAll(source, options)) + } +} diff --git a/packages/ipfs/src/core/components/index.js b/packages/ipfs/src/core/components/index.js index 6b59eaf151..c5257df661 100644 --- a/packages/ipfs/src/core/components/index.js +++ b/packages/ipfs/src/core/components/index.js @@ -1,6 +1,7 @@ 'use strict' exports.add = require('./add') +exports.addAll = require('./add-all') exports.block = { get: require('./block/get'), put: require('./block/put'), diff --git a/packages/ipfs/src/core/components/init.js b/packages/ipfs/src/core/components/init.js index 40dc7c2363..3474d14bd4 100644 --- a/packages/ipfs/src/core/components/init.js +++ b/packages/ipfs/src/core/components/init.js @@ -127,14 +127,14 @@ module.exports = ({ stat: Components.block.stat({ blockService, preload }) } - const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions }) + const addAll = Components.addAll({ block, preload, pin, gcLock, options: constructorOptions }) if (!isInitialized && !options.emptyRepo) { // add empty unixfs dir object (go-ipfs assumes this exists) const emptyDirCid = await addEmptyDir({ dag }) log('adding default assets') - await initAssets({ add, print }) + await initAssets({ addAll, print }) log('initializing IPNS keyspace') // Setup the offline routing for IPNS. @@ -145,7 +145,8 @@ module.exports = ({ } const api = createApi({ - add, + add: Components.add({ addAll }), + addAll, apiManager, constructorOptions, block, diff --git a/packages/ipfs/src/core/components/start.js b/packages/ipfs/src/core/components/start.js index aa543bfacd..24d59fd381 100644 --- a/packages/ipfs/src/core/components/start.js +++ b/packages/ipfs/src/core/components/start.js @@ -191,7 +191,7 @@ function createApi ({ stat: Components.object.stat({ ipld, preload }) } - const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions }) + const addAll = Components.addAll({ block, preload, pin, gcLock, options: constructorOptions }) const isOnline = Components.isOnline({ libp2p }) const dhtNotEnabled = async () => { // eslint-disable-line require-await @@ -240,7 +240,8 @@ function createApi ({ } const api = { - add, + add: Components.add({ addAll }), + addAll, bitswap: { stat: Components.bitswap.stat({ bitswap }), unwant: Components.bitswap.unwant({ bitswap }), diff --git a/packages/ipfs/src/core/components/stop.js b/packages/ipfs/src/core/components/stop.js index 5463a160c0..41dea112bb 100644 --- a/packages/ipfs/src/core/components/stop.js +++ b/packages/ipfs/src/core/components/stop.js @@ -113,7 +113,7 @@ function createApi ({ stat: Components.block.stat({ blockService, preload }) } - const add = Components.add({ block, preload, pin, gcLock, options: constructorOptions }) + const addAll = Components.addAll({ block, preload, pin, gcLock, options: constructorOptions }) const resolve = Components.resolve({ ipld }) const refs = Components.refs({ ipld, resolve, preload }) refs.local = Components.refs.local({ repo }) @@ -123,7 +123,8 @@ function createApi ({ } const api = { - add, + add: Components.add({ addAll }), + addAll, bitswap: { stat: notStarted, unwant: notStarted, diff --git a/packages/ipfs/src/core/runtime/init-assets-nodejs.js b/packages/ipfs/src/core/runtime/init-assets-nodejs.js index 3844f77827..62b0747b27 100644 --- a/packages/ipfs/src/core/runtime/init-assets-nodejs.js +++ b/packages/ipfs/src/core/runtime/init-assets-nodejs.js @@ -5,9 +5,9 @@ const globSource = require('ipfs-utils/src/files/glob-source') const all = require('it-all') // Add the default assets to the repo. -module.exports = async function initAssets ({ add, print }) { +module.exports = async function initAssets ({ addAll, print }) { const initDocsPath = path.join(__dirname, '..', '..', 'init-files', 'init-docs') - const results = await all(add(globSource(initDocsPath, { recursive: true }), { preload: false })) + const results = await all(addAll(globSource(initDocsPath, { recursive: true }), { preload: false })) const dir = results.filter(file => file.path === 'init-docs').pop() print('to get started, enter:\n') diff --git a/packages/ipfs/src/http/api/resources/files-regular.js b/packages/ipfs/src/http/api/resources/files-regular.js index 59b8833ae8..805b06011e 100644 --- a/packages/ipfs/src/http/api/resources/files-regular.js +++ b/packages/ipfs/src/http/api/resources/files-regular.js @@ -285,7 +285,7 @@ exports.add = { } }, function (source) { - return ipfs.add(source, { + return ipfs.addAll(source, { cidVersion, rawLeaves, progress: progress ? progressHandler : () => {}, diff --git a/packages/ipfs/test/cli/add.js b/packages/ipfs/test/cli/add.js index ab690c7689..459f59fefc 100644 --- a/packages/ipfs/test/cli/add.js +++ b/packages/ipfs/test/cli/add.js @@ -46,14 +46,14 @@ describe('add', () => { beforeEach(() => { ipfs = { - add: sinon.stub() + addAll: sinon.stub() } }) it('should add a file', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid: new CID(cid), path: 'readme' }]) @@ -66,7 +66,7 @@ describe('add', () => { it('adds a file path with progress', async () => { const cid = 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB' - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid: new CID(cid), path: 'readme' }]) @@ -79,7 +79,7 @@ describe('add', () => { const cid1 = 'QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB' const cid2 = 'QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o' - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, progress: sinon.match.func, wrapWithDirectory: true @@ -99,7 +99,7 @@ describe('add', () => { it('add with cid-version=1', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB').toV1() - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, cidVersion: 1 }).returns([{ @@ -114,7 +114,7 @@ describe('add', () => { it('add with cid-version=1 and raw-leaves=false', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB').toV1() - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, cidVersion: 1, rawLeaves: false @@ -130,7 +130,7 @@ describe('add', () => { it('add with cid-version=1 and raw-leaves=true', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB').toV1() - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, cidVersion: 1, rawLeaves: true @@ -146,7 +146,7 @@ describe('add', () => { it('add from pipe', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(sinon.match({ + ipfs.addAll.withArgs(sinon.match({ content: matchIterable() }), defaultOptions).returns([{ cid, @@ -167,7 +167,7 @@ describe('add', () => { it('add from pipe with mtime=100', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(sinon.match({ + ipfs.addAll.withArgs(sinon.match({ content: matchIterable(), mtime: { secs: 100 } }), defaultOptions).returns([{ @@ -189,7 +189,7 @@ describe('add', () => { it('add --quiet', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -201,7 +201,7 @@ describe('add', () => { it('add --quiet (short option)', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -213,7 +213,7 @@ describe('add', () => { it('add --quieter', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -225,7 +225,7 @@ describe('add', () => { it('add --quieter (short option)', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -237,7 +237,7 @@ describe('add', () => { it('add --silent', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -249,7 +249,7 @@ describe('add', () => { it('add --only-hash outputs correct hash', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, onlyHash: true }).returns([{ @@ -264,7 +264,7 @@ describe('add', () => { it('add does not pin with --pin=false', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, pin: false }).returns([{ @@ -279,7 +279,7 @@ describe('add', () => { it('add with mtime', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -287,7 +287,7 @@ describe('add', () => { const out = await cli('add --mtime 5 src/init-files/init-docs/readme', { ipfs }) expect(out).to.equal(`added ${cid} readme\n`) - const source = ipfs.add.getCall(0).args[0] + const source = ipfs.addAll.getCall(0).args[0] const input = await first(source) expect(input).to.have.nested.property('mtime.secs', 5) }) @@ -295,7 +295,7 @@ describe('add', () => { it('add with mtime-nsecs', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -303,7 +303,7 @@ describe('add', () => { const out = await cli('add --mtime 5 --mtime-nsecs 100 src/init-files/init-docs/readme', { ipfs }) expect(out).to.equal(`added ${cid} readme\n`) - const source = ipfs.add.getCall(0).args[0] + const source = ipfs.addAll.getCall(0).args[0] const input = await first(source) expect(input).to.have.nested.property('mtime.secs', 5) expect(input).to.have.nested.property('mtime.nsecs', 100) @@ -312,7 +312,7 @@ describe('add', () => { it('add with mode', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -320,7 +320,7 @@ describe('add', () => { const out = await cli('add --mode 0655 src/init-files/init-docs/readme', { ipfs }) expect(out).to.equal(`added ${cid} readme\n`) - const source = ipfs.add.getCall(0).args[0] + const source = ipfs.addAll.getCall(0).args[0] const input = await first(source) expect(input).to.have.property('mode', '0655') }) @@ -329,7 +329,7 @@ describe('add', () => { it(`add with hash=${name} and raw-leaves=false`, async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, hashAlg: name, rawLeaves: false @@ -346,7 +346,7 @@ describe('add', () => { it('should add and print CID encoded in specified base', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ cid, path: 'readme' }]) @@ -358,7 +358,7 @@ describe('add', () => { it('should add with a timeout', async () => { const cid = new CID('QmPZ9gcCEpqKTo6aq61g2nXGUhM4iCL3ewB6LDXZCtioEB') - ipfs.add.withArgs(matchIterable(), { + ipfs.addAll.withArgs(matchIterable(), { ...defaultOptions, timeout: 1000 }).returns([{ diff --git a/packages/ipfs/test/core/add.spec.js b/packages/ipfs/test/core/add-all.spec.js similarity index 95% rename from packages/ipfs/test/core/add.spec.js rename to packages/ipfs/test/core/add-all.spec.js index 007ccfad80..c2bb48930a 100644 --- a/packages/ipfs/test/core/add.spec.js +++ b/packages/ipfs/test/core/add-all.spec.js @@ -3,9 +3,9 @@ 'use strict' const { expect } = require('interface-ipfs-core/src/utils/mocha') -const utils = require('../../src/core/components/add/utils') +const utils = require('../../src/core/components/add-all/utils') -describe('add/utils', () => { +describe('add-all/utils', () => { describe('parseChunkerString', () => { it('handles an empty string', () => { const options = utils.parseChunkerString('') diff --git a/packages/ipfs/test/core/circuit-relay.spec.js b/packages/ipfs/test/core/circuit-relay.spec.js index 39dbc96d24..2fed2390f2 100644 --- a/packages/ipfs/test/core/circuit-relay.spec.js +++ b/packages/ipfs/test/core/circuit-relay.spec.js @@ -2,7 +2,6 @@ 'use strict' const { expect } = require('interface-ipfs-core/src/utils/mocha') -const all = require('it-all') const concat = require('it-concat') const randomBytes = require('iso-random-stream/src/random') const factory = require('../utils/factory') @@ -61,8 +60,8 @@ describe('circuit relay', () => { it('should transfer via relay', async () => { const data = randomBytes(128) - const res = await all(nodeA.add(data)) - const buffer = await concat(nodeB.cat(res[0].cid)) + const res = await nodeA.add(data) + const buffer = await concat(nodeB.cat(res.cid)) expect(buffer.slice()).to.deep.equal(data) }) }) diff --git a/packages/ipfs/test/core/files-sharding.spec.js b/packages/ipfs/test/core/files-sharding.spec.js index bc8232c6d8..4d3270d656 100644 --- a/packages/ipfs/test/core/files-sharding.spec.js +++ b/packages/ipfs/test/core/files-sharding.spec.js @@ -30,7 +30,7 @@ describe('files directory (sharding tests)', function () { after(() => df.clean()) it('should be able to add dir without sharding', async () => { - const { path, cid } = await last(ipfs.add(testFiles)) + const { path, cid } = await last(ipfs.addAll(testFiles)) expect(path).to.eql('test-folder') expect(cid.toString()).to.eql('QmWWM8ZV6GPhqJ46WtKcUaBPNHN5yQaFsKDSQ1RE73w94Q') }) @@ -50,7 +50,7 @@ describe('files directory (sharding tests)', function () { after(() => df.clean()) it('should be able to add dir with sharding', async () => { - const { path, cid } = await last(ipfs.add(testFiles)) + const { path, cid } = await last(ipfs.addAll(testFiles)) expect(path).to.eql('test-folder') expect(cid.toString()).to.eql('Qmb3JNLq2KcvDTSGT23qNQkMrr4Y4fYMktHh6DtC7YatLa') }) diff --git a/packages/ipfs/test/core/files.spec.js b/packages/ipfs/test/core/files.spec.js index 51be5103f0..2b50ad1e28 100644 --- a/packages/ipfs/test/core/files.spec.js +++ b/packages/ipfs/test/core/files.spec.js @@ -31,17 +31,16 @@ describe('files', function () { describe('add', () => { it('should not error when passed null options', async () => { - await all(ipfs.add(Buffer.from(nanoid()), null)) + await ipfs.add(Buffer.from(nanoid()), null) }) it('should add a file with a v1 CID', async () => { - const files = await all(ipfs.add(Buffer.from([0, 1, 2]), { + const file = await ipfs.add(Buffer.from([0, 1, 2]), { cidVersion: 1 - })) + }) - expect(files.length).to.equal(1) - expect(files[0].cid.toString()).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') - expect(files[0].size).to.equal(3) + expect(file.cid.toString()).to.equal('bafkreifojmzibzlof6xyh5auu3r5vpu5l67brf3fitaf73isdlglqw2t7q') + expect(file.size).to.equal(3) }) }) }) diff --git a/packages/ipfs/test/core/gc.spec.js b/packages/ipfs/test/core/gc.spec.js index 89e9903835..fce8319778 100644 --- a/packages/ipfs/test/core/gc.spec.js +++ b/packages/ipfs/test/core/gc.spec.js @@ -3,7 +3,6 @@ 'use strict' const { expect } = require('interface-ipfs-core/src/utils/mocha') -const last = require('it-last') const { Buffer } = require('buffer') const factory = require('../utils/factory') const pEvent = require('p-event') @@ -72,8 +71,8 @@ describe.skip('gc', function () { const blockAddTests = [{ name: 'add', - add1: () => last(ipfs.add(fixtures[0], { pin: false })), - add2: () => last(ipfs.add(fixtures[1], { pin: false })), + add1: () => ipfs.add(fixtures[0], { pin: false }), + add2: () => ipfs.add(fixtures[1], { pin: false }), resToCid: (res) => res[0].cid.toString() }, { name: 'object put', diff --git a/packages/ipfs/test/core/kad-dht.node.js b/packages/ipfs/test/core/kad-dht.node.js index 1e72601eb0..e04403a1ac 100644 --- a/packages/ipfs/test/core/kad-dht.node.js +++ b/packages/ipfs/test/core/kad-dht.node.js @@ -4,7 +4,6 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const path = require('path') -const all = require('it-all') const { Buffer } = require('buffer') const concat = require('it-concat') @@ -60,8 +59,8 @@ describe.skip('kad-dht is routing content and peers correctly', () => { content: Buffer.from('hello kad 1') } - const filesAdded = await all(nodeB.add(file)) - const data = await concat(nodeA.cat(filesAdded[0].cid)) + const fileAdded = await nodeB.add(file) + const data = await concat(nodeA.cat(fileAdded.cid)) expect(data.slice()).to.eql(file.content) }) @@ -73,8 +72,8 @@ describe.skip('kad-dht is routing content and peers correctly', () => { content: Buffer.from('hello kad 2') } - const filesAdded = await all(nodeC.add(file)) - const data = await concat(nodeA.cat(filesAdded[0].cid)) + const fileAdded = await nodeC.add(file) + const data = await concat(nodeA.cat(fileAdded.cid)) expect(data.slice()).to.eql(file.content) }) diff --git a/packages/ipfs/test/core/name-pubsub.js b/packages/ipfs/test/core/name-pubsub.js index eb02a3dd5f..93cb421184 100644 --- a/packages/ipfs/test/core/name-pubsub.js +++ b/packages/ipfs/test/core/name-pubsub.js @@ -90,7 +90,7 @@ describe('name-pubsub', function () { it('should self resolve, publish and then resolve correctly', async function () { this.timeout(6000) const emptyDirCid = '/ipfs/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn' - const { path } = await last(nodeA.add(Buffer.from('pubsub records'))) + const { path } = await nodeA.add(Buffer.from('pubsub records')) const resolvesEmpty = await last(nodeB.name.resolve(idB.id)) expect(resolvesEmpty).to.be.eq(emptyDirCid) diff --git a/packages/ipfs/test/core/pin.js b/packages/ipfs/test/core/pin.js index 01f45f30ad..c9f79d68d6 100644 --- a/packages/ipfs/test/core/pin.js +++ b/packages/ipfs/test/core/pin.js @@ -86,7 +86,7 @@ describe('pin', function () { }) pin = ipfs.pin - await all(ipfs.add(fixtures)) + await ipfs.add(fixtures) }) after(function () { diff --git a/packages/ipfs/test/core/preload.spec.js b/packages/ipfs/test/core/preload.spec.js index 8e57233e81..65c6dfcecd 100644 --- a/packages/ipfs/test/core/preload.spec.js +++ b/packages/ipfs/test/core/preload.spec.js @@ -5,7 +5,6 @@ const { nanoid } = require('nanoid') const { Buffer } = require('buffer') const { expect } = require('interface-ipfs-core/src/utils/mocha') const all = require('it-all') -const last = require('it-last') const MockPreloadNode = require('../utils/mock-preload-node-utils') const IPFS = require('../../src') const createTempRepo = require('../utils/create-repo-nodejs') @@ -38,14 +37,14 @@ describe('preload', () => { it('should preload content added with add', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add(Buffer.from(nanoid()))) - await MockPreloadNode.waitForCids(res[0].cid) + const res = await ipfs.add(Buffer.from(nanoid())) + await MockPreloadNode.waitForCids(res.cid) }) it('should preload multiple content added with add', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add([{ + const res = await all(ipfs.addAll([{ content: Buffer.from(nanoid()) }, { content: Buffer.from(nanoid()) @@ -59,7 +58,7 @@ describe('preload', () => { it('should preload multiple content and intermediate dirs added with add', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add([{ + const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', content: Buffer.from(nanoid()) }, { @@ -79,7 +78,7 @@ describe('preload', () => { it('should preload multiple content and wrapping dir for content added with add and wrapWithDirectory option', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add([{ + const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', content: Buffer.from(nanoid()) }, { @@ -98,22 +97,22 @@ describe('preload', () => { it('should preload content retrieved with cat', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add(Buffer.from(nanoid()), { preload: false })) - await all(ipfs.cat(res[0].cid)) - await MockPreloadNode.waitForCids(res[0].cid) + const res = await ipfs.add(Buffer.from(nanoid()), { preload: false }) + await all(ipfs.cat(res.cid)) + await MockPreloadNode.waitForCids(res.cid) }) it('should preload content retrieved with get', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add(Buffer.from(nanoid()), { preload: false })) - await all(ipfs.get(res[0].cid)) - await MockPreloadNode.waitForCids(res[0].cid) + const res = await ipfs.add(Buffer.from(nanoid()), { preload: false }) + await all(ipfs.get(res.cid)) + await MockPreloadNode.waitForCids(res.cid) }) it('should preload content retrieved with ls', async function () { this.timeout(50 * 1000) - const res = await all(ipfs.add([{ + const res = await all(ipfs.addAll([{ path: 'dir0/dir1/file0', content: Buffer.from(nanoid()) }, { @@ -243,8 +242,8 @@ describe('preload', () => { }) it('should preload content retrieved with files.ls', async () => { - const res = await all(ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) })) - const dirCid = res[res.length - 1].cid + const res = await ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) }) + const dirCid = res.cid await MockPreloadNode.waitForCids(dirCid) await MockPreloadNode.clearPreloadCids() await all(ipfs.files.ls(`/ipfs/${dirCid}`)) @@ -252,8 +251,8 @@ describe('preload', () => { }) it('should preload content retrieved with files.ls by CID', async () => { - const res = await all(ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) })) - const dirCid = res[res.length - 1].cid + const res = await ipfs.add({ path: `/t/${nanoid()}`, content: Buffer.from(nanoid()) }) + const dirCid = res.cid await MockPreloadNode.waitForCids(dirCid) await MockPreloadNode.clearPreloadCids() await all(ipfs.files.ls(dirCid)) @@ -261,15 +260,15 @@ describe('preload', () => { }) it('should preload content retrieved with files.read', async () => { - const fileCid = (await all(ipfs.add(Buffer.from(nanoid()))))[0].cid - await MockPreloadNode.waitForCids(fileCid) + const { cid } = await ipfs.add(Buffer.from(nanoid())) + await MockPreloadNode.waitForCids(cid) await MockPreloadNode.clearPreloadCids() - await ipfs.files.read(`/ipfs/${fileCid}`) - await MockPreloadNode.waitForCids(`/ipfs/${fileCid}`) + await ipfs.files.read(`/ipfs/${cid}`) + await MockPreloadNode.waitForCids(`/ipfs/${cid}`) }) it('should preload content retrieved with files.stat', async () => { - const fileCid = (await all(ipfs.add(Buffer.from(nanoid()))))[0].cid + const { cid: fileCid } = await ipfs.add(Buffer.from(nanoid())) await MockPreloadNode.waitForCids(fileCid) await MockPreloadNode.clearPreloadCids() await ipfs.files.stat(`/ipfs/${fileCid}`) @@ -305,7 +304,7 @@ describe('preload disabled', function () { after(() => repo.teardown()) it('should not preload if disabled', async () => { - const { cid } = await last(ipfs.add(Buffer.from(nanoid()))) + const { cid } = await ipfs.add(Buffer.from(nanoid())) return expect(MockPreloadNode.waitForCids(cid)) .to.eventually.be.rejected() diff --git a/packages/ipfs/test/core/utils.js b/packages/ipfs/test/core/utils.js index ee83781de4..941f8b3042 100644 --- a/packages/ipfs/test/core/utils.js +++ b/packages/ipfs/test/core/utils.js @@ -5,7 +5,6 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const fs = require('fs') const { fromB58String } = require('multihashing-async').multihash -const all = require('it-all') // This gets replaced by `create-repo-browser.js` in the browser const createTempRepo = require('../utils/create-repo-nodejs.js') @@ -43,7 +42,7 @@ describe('utils', () => { }, preload: { enabled: false } }) - await all(node.add(fixtures)) + await node.add(fixtures) }) after(() => node.stop()) diff --git a/packages/ipfs/test/gateway/index.js b/packages/ipfs/test/gateway/index.js index 892b2d207d..f4bf98061d 100644 --- a/packages/ipfs/test/gateway/index.js +++ b/packages/ipfs/test/gateway/index.js @@ -66,7 +66,7 @@ describe('HTTP Gateway', function () { gateway = http.api._httpApi._gatewayServers[0] // QmbQD7EMEL1zeebwBsWEfA3ndgSS6F7S6iTuwuqasPgVRi - await all(http.api._ipfs.add([ + await all(http.api._ipfs.addAll([ content('index.html'), emptyDir('empty-folder'), content('nested-folder/hello.txt'), @@ -75,18 +75,18 @@ describe('HTTP Gateway', function () { emptyDir('nested-folder/empty') ])) // Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq - await all(http.api._ipfs.add(bigFile)) + await http.api._ipfs.add(bigFile) // QmT78zSuBmuS4z925WZfrqQ1qHaJ56DQaTfyMUF7F8ff5o - await all(http.api._ipfs.add(Buffer.from('hello world' + '\n'), { cidVersion: 0 })) + await http.api._ipfs.add(Buffer.from('hello world' + '\n'), { cidVersion: 0 }) // QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ - await all(http.api._ipfs.add([content('cat-folder/cat.jpg')])) + await http.api._ipfs.add([content('cat-folder/cat.jpg')]) // QmVZoGxDvKM9KExc8gaL4uTbhdNtWhzQR7ndrY7J1gWs3F - await all(http.api._ipfs.add([ + await all(http.api._ipfs.addAll([ content('unsniffable-folder/hexagons-xml.svg'), content('unsniffable-folder/hexagons.svg') ])) // QmaRdtkDark8TgXPdDczwBneadyF44JvFGbrKLTkmTUhHk - await all(http.api._ipfs.add([content('utf8/cat-with-óąśśł-and-أعظم._.jpg')])) + await http.api._ipfs.add(content('utf8/cat-with-óąśśł-and-أعظم._.jpg')) // Publish QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ to IPNS using self key await http.api._ipfs.name.publish('QmW2WQi7j6c7UgJTarActp7tDNikE4B2qXtFCfLPdsgaTQ', { resolve: false }) }) diff --git a/packages/ipfs/test/http-api/inject/files.js b/packages/ipfs/test/http-api/inject/files.js index 5165cd735c..c4fd21b274 100644 --- a/packages/ipfs/test/http-api/inject/files.js +++ b/packages/ipfs/test/http-api/inject/files.js @@ -27,7 +27,7 @@ describe('/files', () => { beforeEach(() => { ipfs = { - add: sinon.stub(), + addAll: sinon.stub(), cat: sinon.stub(), get: sinon.stub(), ls: sinon.stub(), @@ -40,7 +40,7 @@ describe('/files', () => { async function assertAddArgs (url, fn) { const content = Buffer.from('TEST\n') - ipfs.add.callsFake(async function * (source, opts) { + ipfs.addAll.callsFake(async function * (source, opts) { expect(fn(opts)).to.be.true() const input = await first(source) @@ -123,7 +123,7 @@ describe('/files', () => { it('should add data and return a base64 encoded CID', async () => { const content = Buffer.from('TEST' + Date.now()) - ipfs.add.withArgs(matchIterable(), defaultOptions).returns([{ + ipfs.addAll.withArgs(matchIterable(), defaultOptions).returns([{ path: cid.toString(), cid, size: content.byteLength, @@ -153,7 +153,7 @@ describe('/files', () => { it('should add data without pinning and return a base64 encoded CID', async () => { const content = Buffer.from('TEST' + Date.now()) - ipfs.add.callsFake(async function * (source, opts) { + ipfs.addAll.callsFake(async function * (source, opts) { expect(opts).to.have.property('pin', false) const input = await first(source) From 1b875f817c44f80f0faea0314470947b336dda28 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Jul 2020 09:19:15 +0100 Subject: [PATCH 2/7] chore: add missing dep --- packages/ipfs-http-client/package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/ipfs-http-client/package.json b/packages/ipfs-http-client/package.json index 8a58a239d8..903072e8f5 100644 --- a/packages/ipfs-http-client/package.json +++ b/packages/ipfs-http-client/package.json @@ -53,6 +53,7 @@ "ipld-dag-pb": "^0.18.5", "ipld-raw": "^5.0.0", "iso-url": "^0.4.7", + "it-last": "^1.0.1", "it-tar": "^1.2.2", "it-to-buffer": "^1.0.0", "it-to-stream": "^0.1.1", From 37a20f71094bf9ce0fd03243233382dac22130ce Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Jul 2020 13:08:16 +0100 Subject: [PATCH 3/7] chore: update interop tests --- packages/ipfs/package.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 7f433c02b2..467b13cbaf 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -160,7 +160,7 @@ "progress": "^2.0.1", "protons": "^1.2.0", "semver": "^7.3.2", - "stream-to-it": "^0.2.0", + "stream-to-it": "^0.2.1", "streaming-iterables": "^5.0.0", "temp": "^0.9.0", "timeout-abort-controller": "^1.1.0", @@ -180,7 +180,7 @@ "form-data": "^3.0.0", "go-ipfs-dep": "^0.6.0", "interface-ipfs-core": "^0.137.0", - "ipfs-interop": "^1.0.4", + "ipfs-interop": "ipfs/interop#refactor/make-add-only-work-on-single-items", "ipfsd-ctl": "^4.1.1", "iso-random-stream": "^1.1.1", "it-to-buffer": "^1.0.0", From 3b44f9db093c197180d06fd4894c3e1c2f0939d6 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Jul 2020 14:58:55 +0100 Subject: [PATCH 4/7] chore: update bitswap test --- packages/ipfs/test/core/bitswap.spec.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/ipfs/test/core/bitswap.spec.js b/packages/ipfs/test/core/bitswap.spec.js index 443eeb8e93..933f70b879 100644 --- a/packages/ipfs/test/core/bitswap.spec.js +++ b/packages/ipfs/test/core/bitswap.spec.js @@ -7,7 +7,6 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha') const Block = require('ipld-block') const multihashing = require('multihashing-async') const CID = require('cids') -const all = require('it-all') const concat = require('it-concat') const { Buffer } = require('buffer') const factory = require('../utils/factory') @@ -71,8 +70,8 @@ describe('bitswap', function () { const proc = (await df.spawn({ type: 'proc' })).api proc.swarm.connect(remote.peerId.addresses[0]) - const files = await all(remote.add([{ path: 'awesome.txt', content: file }])) - const data = await concat(proc.cat(files[0].cid)) + const file = await remote.add({ path: 'awesome.txt', content: file }) + const data = await concat(proc.cat(file.cid)) expect(data.slice()).to.eql(file) await df.clean() }) From a200a875972ff28dc41acd15c685d8994def6df1 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Jul 2020 13:27:42 +0100 Subject: [PATCH 5/7] chore: fix linting --- packages/ipfs/test/core/bitswap.spec.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ipfs/test/core/bitswap.spec.js b/packages/ipfs/test/core/bitswap.spec.js index 933f70b879..c59a249fea 100644 --- a/packages/ipfs/test/core/bitswap.spec.js +++ b/packages/ipfs/test/core/bitswap.spec.js @@ -65,12 +65,12 @@ describe('bitswap', function () { it('2 peers', async () => { // TODO make this test more interesting (10Mb file) // TODO remove randomness from the test - const file = Buffer.from(`I love IPFS <3 ${nanoid()}`) + const content = Buffer.from(`I love IPFS <3 ${nanoid()}`) const remote = (await df.spawn({ type: 'js' })).api const proc = (await df.spawn({ type: 'proc' })).api proc.swarm.connect(remote.peerId.addresses[0]) - const file = await remote.add({ path: 'awesome.txt', content: file }) + const file = await remote.add({ path: 'awesome.txt', content }) const data = await concat(proc.cat(file.cid)) expect(data.slice()).to.eql(file) await df.clean() From 3c31a3629a23c558b894d007d47b7c7e50159f63 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 15 Jul 2020 14:16:33 +0100 Subject: [PATCH 6/7] chore: fix failing test --- docs/core-api/FILES.md | 9 ++++----- packages/ipfs/docs/MODULE.md | 9 +++++---- packages/ipfs/test/core/bitswap.spec.js | 2 +- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/core-api/FILES.md b/docs/core-api/FILES.md index 024b8239c1..550ddd62ed 100644 --- a/docs/core-api/FILES.md +++ b/docs/core-api/FILES.md @@ -311,7 +311,7 @@ const files = [{ content: 'ABC' }] -for await (const result of ipfs.add(content)) { +for await (const result of ipfs.addAll(content)) { console.log(result) } @@ -376,7 +376,7 @@ const addOptions = { timeout: 10000 }; -for await (const file of ipfs.add(globSource('./docs', globSourceOptions), addOptions)) { +for await (const file of ipfs.addAll(globSource('./docs', globSourceOptions), addOptions)) { console.log(file) } @@ -405,9 +405,8 @@ const { urlSource } = IPFS const ipfs = await IPFS.create() -for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg'))) { - console.log(file) -} +const file = await ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg')) +console.log(file) /* { diff --git a/packages/ipfs/docs/MODULE.md b/packages/ipfs/docs/MODULE.md index 41b8e4e9f7..9e05865ba9 100644 --- a/packages/ipfs/docs/MODULE.md +++ b/packages/ipfs/docs/MODULE.md @@ -388,7 +388,7 @@ Returns an async iterable that yields `{ path, content }` objects suitable for p const IPFS = require('ipfs') const { globSource } = IPFS const ipfs = await IPFS.create() -for await (const file of ipfs.add(globSource('./docs', { recursive: true }))) { +for await (const file of ipfs.addAll(globSource('./docs', { recursive: true }))) { console.log(file) } /* @@ -422,9 +422,10 @@ Returns an async iterable that yields `{ path, content }` objects suitable for p const IPFS = require('ipfs') const { urlSource } = IPFS const ipfs = await IPFS.create() -for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg'))) { - console.log(file) -} + +const file = await ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg')) +console.log(file) + /* { path: 'ipfs-logo.svg', diff --git a/packages/ipfs/test/core/bitswap.spec.js b/packages/ipfs/test/core/bitswap.spec.js index c59a249fea..da1b4fa123 100644 --- a/packages/ipfs/test/core/bitswap.spec.js +++ b/packages/ipfs/test/core/bitswap.spec.js @@ -72,7 +72,7 @@ describe('bitswap', function () { const file = await remote.add({ path: 'awesome.txt', content }) const data = await concat(proc.cat(file.cid)) - expect(data.slice()).to.eql(file) + expect(data.slice()).to.eql(content) await df.clean() }) }) From 0789c40691606b96262000ac740b24ac922adfef Mon Sep 17 00:00:00 2001 From: achingbrain Date: Thu, 16 Jul 2020 09:23:11 +0100 Subject: [PATCH 7/7] chore: fix broken example --- .../http-client-bundle-webpack/src/App.js | 28 +++++++++---------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/examples/http-client-bundle-webpack/src/App.js b/examples/http-client-bundle-webpack/src/App.js index 843e3dae99..d68aaa855f 100644 --- a/examples/http-client-bundle-webpack/src/App.js +++ b/examples/http-client-bundle-webpack/src/App.js @@ -29,25 +29,23 @@ class App extends React.Component { protocol_version: id.protocolVersion }) - const source = ipfs.add(stringToUse) - for await (const file of source) { - const hash = file.cid - this.setState({ added_file_hash: hash.toString() }) + const file = await ipfs.add(stringToUse) + const hash = file.cid + this.setState({ added_file_hash: hash.toString() }) - const source = ipfs.cat(hash) - let contents = '' - const decoder = new TextDecoder('utf-8') + const source = ipfs.cat(hash) + let contents = '' + const decoder = new TextDecoder('utf-8') - for await (const chunk of source) { - contents += decoder.decode(chunk, { - stream: true - }) - } + for await (const chunk of source) { + contents += decoder.decode(chunk, { + stream: true + }) + } - contents += decoder.decode() + contents += decoder.decode() - this.setState({ added_file_contents: contents }) - } + this.setState({ added_file_contents: contents }) } render () {