refactor!: remove search option (#23)

This PR removes the internal `search` option in favour of a block `filter` option. I also fixed all the types in `index.js`...was having a hard time with `Link` vs `CID` and decided to upgrade everything to use `Link`, seeing as this is going out as a breaking change anyway.
storacha · May 18, 2023 · 9ede86f · 9ede86f
1 parent 3e0d648
commit 9ede86f
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 38 deletions.
diff --git a/bitswap-fetcher.js b/bitswap-fetcher.js
@@ -16,9 +16,9 @@ const log = debug('dagula:bitswapfetcher')
 export class BitswapFetcher {
   /** @type {() => Promise<import('@libp2p/interface-connection').Stream>} */
   #newStream
-  /** @type {Map<string, Array<{ cid: import('multiformats').CID, deferredPromise: import('p-defer').DeferredPromise<Block|undefined> }>>} */
+  /** @type {Map<string, Array<{ cid: import('multiformats').UnknownLink, deferredPromise: import('p-defer').DeferredPromise<Block|undefined> }>>} */
   #wants = new Map()
-  /** @type {import('multiformats').CID[]} */
+  /** @type {import('multiformats').UnknownLink[]} */
   #wantlist = []
   /** @type {number} */
   #outstandingWants = 0
@@ -83,7 +83,7 @@ export class BitswapFetcher {
   }
 
   /**
-   * @param {import('multiformats').CID} cid
+   * @param {import('multiformats').UnknownLink} cid
    * @param {{ signal?: AbortSignal }} [options]
    */
   get (cid, options = {}) {

diff --git a/index.d.ts b/index.d.ts
@@ -1,6 +1,6 @@
 import type { BlockDecoder } from 'multiformats/codecs/interface'
 import type { MultihashHasher } from 'multiformats/hashes/interface'
-import type { CID } from 'multiformats'
+import type { UnknownLink } from 'multiformats'
 import type { UnixFSEntry } from 'ipfs-unixfs-exporter'
 import type { Multiaddr } from '@multiformats/multiaddr'
 import type { AbortOptions } from '@libp2p/interfaces'
@@ -17,12 +17,12 @@ export interface MultihashHashers {
 }
 
 export interface Block {
-  cid: CID
+  cid: UnknownLink
   bytes: Uint8Array
 }
 
 export interface Blockstore {
-  get: (cid: CID, options?: { signal?: AbortSignal }) => Promise<Block | undefined>
+  get: (cid: UnknownLink, options?: { signal?: AbortSignal }) => Promise<Block | undefined>
 }
 
 export interface Network {
@@ -90,45 +90,45 @@ export interface IDagula {
   /**
    * Get a complete DAG by root CID.
    */
-  get (cid: CID|string, options?: AbortOptions & BlockOrderOptions): AsyncIterableIterator<Block>
+  get (cid: UnknownLink|string, options?: AbortOptions & BlockOrderOptions): AsyncIterableIterator<Block>
   /**
    * Get a DAG for a cid+path.
    */
   getPath (cidPath: string, options?: AbortOptions & DagScopeOptions & BlockOrderOptions): AsyncIterableIterator<Block>
   /**
    * Get a single block.
    */
-  getBlock (cid: CID|string, options?: AbortOptions): Promise<Block>
+  getBlock (cid: UnknownLink|string, options?: AbortOptions): Promise<Block>
   /**
    * Get UnixFS files and directories.
    */
-  getUnixfs (path: CID|string, options?: AbortOptions): Promise<UnixFSEntry>
+  getUnixfs (path: UnknownLink|string, options?: AbortOptions): Promise<UnixFSEntry>
   /**
    * Emit nodes for all path segements and get UnixFS files and directories.
    */
-  walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator<UnixFSEntry>
+  walkUnixfsPath (path: UnknownLink|string, options?: AbortOptions): AsyncIterableIterator<UnixFSEntry>
 }
 
 export declare class Dagula implements IDagula {
   constructor (blockstore: Blockstore, options?: { decoders?: BlockDecoders, hashers?: MultihashHashers })
   /**
    * Get a complete DAG by root CID.
    */
-  get (cid: CID|string, options?: AbortOptions & BlockOrderOptions): AsyncIterableIterator<Block>
+  get (cid: UnknownLink|string, options?: AbortOptions & BlockOrderOptions): AsyncIterableIterator<Block>
   /**
    * Get a DAG for a cid+path.
    */
   getPath (cidPath: string, options?: AbortOptions & DagScopeOptions & BlockOrderOptions): AsyncIterableIterator<Block>
   /**
    * Get a single block.
    */
-  getBlock (cid: CID|string, options?: AbortOptions): Promise<Block>
+  getBlock (cid: UnknownLink|string, options?: AbortOptions): Promise<Block>
   /**
    * Get UnixFS files and directories.
    */
-  getUnixfs (path: CID|string, options?: AbortOptions): Promise<UnixFSEntry>
+  getUnixfs (path: UnknownLink|string, options?: AbortOptions): Promise<UnixFSEntry>
   /**
    * Emit nodes for all path segements and get UnixFS files and directories.
    */
-  walkUnixfsPath (path: CID|string, options?: AbortOptions): AsyncIterableIterator<UnixFSEntry>
+  walkUnixfsPath (path: UnknownLink|string, options?: AbortOptions): AsyncIterableIterator<UnixFSEntry>
 }
diff --git a/index.js b/index.js
@@ -7,6 +7,8 @@ import { parallelMap, transform } from 'streaming-iterables'
 import { Decoders, Hashers } from './defaults.js'
 import { identity } from 'multiformats/hashes/identity'
 
+/** @typedef {([name, cid]: [string, import('multiformats').UnknownLink]) => boolean} LinkFilter */
+
 const log = debug('dagula')
 
 export class Dagula {
@@ -31,18 +33,18 @@ export class Dagula {
   }
 
   /**
-   * @param {CID[]|CID|string} cid
+   * @param {import('multiformats').UnknownLink[]|import('multiformats').UnknownLink|string} cid
    * @param {object} [options]
    * @param {AbortSignal} [options.signal]
    * @param {import('./index').BlockOrder} [options.order]
-   * @param {(block: import('multiformats').BlockView) => CID[]} [options.search]
+   * @param {LinkFilter} [options.filter]
    */
   async * get (cid, options = {}) {
     cid = typeof cid === 'string' ? CID.parse(cid) : cid
     const order = options.order ?? 'dfs'
     log('getting DAG %s', cid)
     let cids = Array.isArray(cid) ? cid : [cid]
-    const search = options.search || blockLinks()
+    const getLinks = blockLinks(options.filter)
 
     /** @type {AbortController[]} */
     let aborters = []
@@ -80,7 +82,7 @@ export class Dagula {
         // createUnsafe here.
         const block = await Block.create({ bytes, cid, codec: decoder, hasher })
         yield block
-        const blockCids = search(block)
+        const blockCids = getLinks(block)
         if (order === 'dfs') {
           yield * this.get(blockCids, options)
         } else {
@@ -115,9 +117,9 @@ export class Dagula {
 
     /**
      * The resolved dag root at the terminus of the cidPath
-     * @type {import('ipfs-unixfs-exporter').UnixFSEntry}
+     * @type {import('ipfs-unixfs-exporter').UnixFSEntry?}
      */
-    let base
+    let base = null
 
     /**
      * Cache of blocks required to resove the cidPath
@@ -141,11 +143,13 @@ export class Dagula {
         return block.bytes
       }
     }
+    // @ts-expect-error walkPath wants blockstore with has and put
     for await (const item of walkPath(cidPath, blockstore, { signal: options.signal })) {
       base = item
       yield * traversed
       traversed = []
     }
+    if (!base) throw new Error('walkPath did not yield an entry')
 
     if (dagScope === 'all' || (dagScope === 'entity' && base.type !== 'directory')) {
       const links = getLinks(base, this.#decoders)
@@ -159,16 +163,16 @@ export class Dagula {
       // the single block for the root has already been yielded.
       // For a hamt we must fetch all the blocks of the (current) hamt.
       if (base.unixfs.type === 'hamt-sharded-directory') {
-        const hamtLinks = base.node.Links?.filter(l => l.Name.length === 2).map(l => l.Hash) || []
+        const hamtLinks = base.node.Links?.filter(l => l.Name?.length === 2).map(l => l.Hash) || []
         if (hamtLinks.length) {
-          yield * this.get(hamtLinks, { search: hamtSearch, signal: options.signal, order: options.order })
+          yield * this.get(hamtLinks, { filter: hamtFilter, signal: options.signal, order: options.order })
         }
       }
     }
   }
 
   /**
-   * @param {import('multiformats').CID|string} cid
+   * @param {import('multiformats').UnknownLink|string} cid
    * @param {{ signal?: AbortSignal }} [options]
    */
   async getBlock (cid, options = {}) {
@@ -185,14 +189,14 @@ export class Dagula {
   }
 
   /**
-   * @param {string|import('multiformats').CID} path
+   * @param {string|import('multiformats').UnknownLink} path
    * @param {{ signal?: AbortSignal }} [options]
    */
   async getUnixfs (path, options = {}) {
     log('getting unixfs %s', path)
     const blockstore = {
       /**
-       * @param {CID} cid
+       * @param {import('multiformats').UnknownLink} cid
        * @param {{ signal?: AbortSignal }} [options]
        */
       get: async (cid, options) => {
@@ -212,14 +216,15 @@ export class Dagula {
     log('walking unixfs %s', cidPath)
     const blockstore = {
       /**
-       * @param {CID} cid
+       * @param {import('multiformats').UnknownLink} cid
        * @param {{ signal?: AbortSignal }} [options]
        */
       get: async (cid, options) => {
         const block = await this.getBlock(cid, options)
         return block.bytes
       }
     }
+    // @ts-expect-error walkPath wants blockstore with has and put
     yield * walkPath(cidPath, blockstore, { signal: options.signal })
   }
 }
@@ -228,13 +233,14 @@ export class Dagula {
  * Create a search function that given a decoded Block
  * will return an array of CIDs to fetch next.
  *
- * @param {([name, cid]: [string, Link]) => boolean} linkFilter
+ * @param {LinkFilter} linkFilter
  */
 export function blockLinks (linkFilter = () => true) {
   /**
-   * @param {import('multiformats').BlockView} block
+   * @param {import('multiformats').BlockView<any, any, any, import('multiformats').Version>} block
    */
   return function (block) {
+    /** @type {import('multiformats').UnknownLink[]} */
     const nextCids = []
     if (block.cid.code === dagPB.code) {
       for (const { Name, Hash } of block.value.Links) {
@@ -254,12 +260,13 @@ export function blockLinks (linkFilter = () => true) {
   }
 }
 
-export const hamtSearch = blockLinks(([name]) => name.length === 2)
+/** @type {LinkFilter} */
+export const hamtFilter = ([name]) => name.length === 2
 
 /**
  * Get links as array of CIDs for a UnixFS entry.
  * @param {import('ipfs-unixfs-exporter').UnixFSEntry} entry
- * @param {import('multiformats').BlockDecoder[]} decoders
+ * @param {import('./index').BlockDecoders} decoders
  */
 function getLinks (entry, decoders) {
   if (entry.type === 'file' || entry.type === 'directory') {

diff --git a/message.js b/message.js
@@ -36,7 +36,7 @@ const ADDED_ESTIMATION_PERCENTAGE = 0.1
 
 export class Entry {
   /**
-   * @param {CID} cid
+   * @param {import('multiformats').UnknownLink} cid
    * @param {Object} [options]
    * @param {number} [options.priority]
    * @param {boolean} [options.cancel]
@@ -125,15 +125,13 @@ export class Wantlist {
 
 export class Block {
   /**
-   * @param {Uint8Array|CID} prefixOrCid
+   * @param {Uint8Array|import('multiformats').UnknownLink} prefixOrCid
    * @param {Uint8Array} data
    */
   constructor (prefixOrCid, data) {
-    if (prefixOrCid instanceof CID) {
-      prefixOrCid = Prefix.encode(prefixOrCid)
-    }
-
-    this.prefix = prefixOrCid
+    this.prefix = prefixOrCid instanceof Uint8Array
+      ? prefixOrCid
+      : Prefix.encode(prefixOrCid)
     this.data = data
   }
 
@@ -155,7 +153,7 @@ export class Block {
 
 export class BlockPresence {
   /**
-   * @param {CID} cid
+   * @param {import('multiformats').UnknownLink} cid
    * @param {gen.Message.BlockPresenceType} type
    */
   constructor (cid, type) {