feat: storeDirectory accepts iter of file objects (#1924)

* storeDirectory accepts iterables of FileObject, not just dom File * cleanup * adjust types so storeDirectory doesnt have to cast FileObject to File before passing to encodeDirectory, which now takes a FilesSource
nftstorage · May 19, 2022 · 377b045 · 377b045
1 parent eb65633
commit 377b045
Show file tree

Hide file tree

Showing 3 changed files with 75 additions and 26 deletions.
diff --git a/packages/client/src/lib.js b/packages/client/src/lib.js
@@ -36,6 +36,8 @@ const RATE_LIMIT_PERIOD = 10 * 1000
  * @typedef {import('./lib/interface.js').Service} Service
  * @typedef {import('./lib/interface.js').CIDString} CIDString
  * @typedef {import('./lib/interface.js').Deal} Deal
+ * @typedef {import('./lib/interface.js').FileObject} FileObject
+ * @typedef {import('./lib/interface.js').FilesSource} FilesSource
  * @typedef {import('./lib/interface.js').Pin} Pin
  * @typedef {import('./lib/interface.js').CarReader} CarReader
  * @typedef {import('ipfs-car/blockstore').Blockstore} BlockstoreI
@@ -216,15 +218,14 @@ class NFTStorage {
    * `foo/bla/baz.json` is ok but `foo/bar.png`, `bla/baz.json` is not.
    *
    * @param {Service} service
-   * @param {Iterable<File>|AsyncIterable<File>} files
+   * @param {FilesSource} filesSource
    * @returns {Promise<CIDString>}
    */
-  static async storeDirectory(service, files) {
+  static async storeDirectory(service, filesSource) {
     const blockstore = new Blockstore()
     let cidString
-
     try {
-      const { cid, car } = await NFTStorage.encodeDirectory(files, {
+      const { cid, car } = await NFTStorage.encodeDirectory(filesSource, {
         blockstore,
       })
       await NFTStorage.storeCar(service, car)
@@ -456,22 +457,19 @@ class NFTStorage {
    * await client.storeCar(car)
    * ```
    *
-   * @param {Iterable<File>|AsyncIterable<File>} files
+   * @param {FilesSource} files
    * @param {object} [options]
    * @param {BlockstoreI} [options.blockstore]
    * @returns {Promise<{ cid: CID, car: CarReader }>}
    */
   static async encodeDirectory(files, { blockstore } = {}) {
     let size = 0
-    const input = pipe(
-      isIterable(files) ? toAsyncIterable(files) : files,
-      async function* (files) {
-        for await (const file of files) {
-          yield toImportCandidate(file.name, file)
-          size += file.size
-        }
+    const input = pipe(files, async function* (files) {
+      for await (const file of files) {
+        yield toImportCandidate(file.name, file)
+        size += file.size
       }
-    )
+    })
     const packed = await packCar(input, {
       blockstore,
       wrapWithDirectory: true,
@@ -561,7 +559,7 @@ class NFTStorage {
    * Argument can be a [FileList](https://developer.mozilla.org/en-US/docs/Web/API/FileList)
    * instance as well, in which case directory structure will be retained.
    *
-   * @param {AsyncIterable<File>|Iterable<File>} files
+   * @param {FilesSource} files
    */
   storeDirectory(files) {
     return NFTStorage.storeDirectory(this, files)
@@ -658,15 +656,6 @@ class NFTStorage {
   }
 }
 
-/**
- * type guard checking for Iterable
- * @param {any} x;
- * @returns {x is Iterable<unknown>}
- */
-function isIterable(x) {
-  return Symbol.iterator in x
-}
-
 /**
  * Cast an iterable to an asyncIterable
  * @template T
@@ -758,10 +747,11 @@ const decodePin = (pin) => ({ ...pin, created: new Date(pin.created) })
  * the stream is created only when needed.
  *
  * @param {string} path
- * @param {Blob} blob
+ * @param {Pick<Blob, 'stream'>|{ stream: () => AsyncIterable<Uint8Array> }} blob
+ * @returns {import('ipfs-core-types/src/utils.js').ImportCandidate}
  */
 function toImportCandidate(path, blob) {
-  /** @type {ReadableStream} */
+  /** @type {AsyncIterable<Uint8Array>} */
   let stream
   return {
     path,

diff --git a/packages/client/src/lib/interface.ts b/packages/client/src/lib/interface.ts
@@ -23,6 +23,18 @@ export interface PublicService {
   rateLimiter?: RateLimiter
 }
 
+export interface FileObject {
+  name: string
+  size: number
+  stream: () => AsyncIterable<any>
+}
+
+export type FilesSource =
+| Iterable<File>
+| Iterable<FileObject>
+| AsyncIterable<File>
+| AsyncIterable<FileObject>
+
 /**
  * CID in string representation
  */
@@ -100,7 +112,7 @@ export interface API {
    * be within the same directory, otherwise error is raised e.g. `foo/bar.png`,
    * `foo/bla/baz.json` is ok but `foo/bar.png`, `bla/baz.json` is not.
    */
-  storeDirectory(service: Service, files: Iterable<File>|AsyncIterable<File>): Promise<CIDString>
+  storeDirectory(service: Service, files: FilesSource): Promise<CIDString>
   /**
    * Returns current status of the stored NFT by its CID. Note the NFT must
    * have previously been stored by this account.

diff --git a/packages/client/test/lib.spec.js b/packages/client/test/lib.spec.js
@@ -241,6 +241,36 @@ describe('client', () => {
       )
     })
 
+    it('upload multiple FileObject from files-from-path as asyncIterable', async () => {
+      const client = new NFTStorage({ token, endpoint })
+      const file1Buffer = new TextEncoder().encode('hello world')
+      const file2Buffer = new TextEncoder().encode(
+        JSON.stringify({ from: 'incognito' }, null, 2)
+      )
+      const cid = await client.storeDirectory(
+        toAsyncIterable([
+          {
+            name: 'hello.txt',
+            size: file1Buffer.length,
+            stream: async function* () {
+              yield file1Buffer
+            },
+          },
+          {
+            name: 'metadata.json',
+            size: file2Buffer.length,
+            stream: async function* () {
+              yield file2Buffer
+            },
+          },
+        ])
+      )
+
+      assert.equal(
+        cid,
+        'bafybeigkms36pnnjsa7t2mq2g4mx77s4no2hilirs4wqx3eebbffy2ay3a'
+      )
+    })
     it('upload empty files', async () => {
       const client = new NFTStorage({ token, endpoint })
       try {
@@ -743,4 +773,21 @@ describe('client', () => {
       }
     })
   })
+
+  describe('static encodeDirectory', () => {
+    it('can encode multiple FileObject as iterable', async () => {
+      const files = [
+        new File(['hello world'], 'hello.txt'),
+        new File(
+          [JSON.stringify({ from: 'incognito' }, null, 2)],
+          'metadata.json'
+        ),
+      ]
+      const { cid } = await NFTStorage.encodeDirectory(files)
+      assert.equal(
+        cid.toString(),
+        'bafybeigkms36pnnjsa7t2mq2g4mx77s4no2hilirs4wqx3eebbffy2ay3a'
+      )
+    })
+  })
 })