fix: prevent interop timeouts with fast fixture loading

packages/interop/.aegir.js

@@ -1,46 +1,30 @@
-import getPort from 'aegir/get-port'
-import { createServer } from 'ipfsd-ctl'
-import * as kuboRpcClient from 'kubo-rpc-client'
+import { resolve } from 'node:path'
+import { tmpdir } from 'node:os'
+
+const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo')
 
 /** @type {import('aegir').PartialOptions} */
 export default {
   test: {
     files: './dist/src/*.spec.js',
-    before: async (options) => {
-      if (options.runner !== 'node') {
-        const ipfsdPort = await getPort()
-        const ipfsdServer = await createServer({
-          host: '127.0.0.1',
-          port: ipfsdPort
-        }, {
-          ipfsBin: (await import('kubo')).default.path(),
-          kuboRpcModule: kuboRpcClient,
-          ipfsOptions: {
-            config: {
-              Addresses: {
-                Swarm: [
-                  "/ip4/0.0.0.0/tcp/0",
-                  "/ip4/0.0.0.0/tcp/0/ws"
-                ]
-              }
-            }
-          }
-        }).start()
+    before: async () => {
 
-        return {
-          env: {
-            IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}`
-          },
-          ipfsdServer
-        }
-      }
+      const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js')
+      const kuboNode = await createKuboNode(IPFS_PATH)
 
-      return {}
-    },
-    after: async (options, beforeResult) => {
-      if (options.runner !== 'node') {
-        await beforeResult.ipfsdServer.stop()
+      await kuboNode.start()
+
+      // requires aegir build to be run first, which it will by default.
+      const { loadFixtures } = await import('./dist/src/fixtures/load-fixtures.js')
+
+      await loadFixtures(IPFS_PATH)
+
+      return {
+        kuboNode
       }
+    },
+    after: async (_options, beforeResult) => {
+      await beforeResult.kuboNode.stop()
     }
   }
 }

packages/interop/package.json

@@ -59,10 +59,11 @@
   "dependencies": {
     "@helia/verified-fetch": "1.3.14",
     "aegir": "^42.2.5",
-    "ipfsd-ctl": "^13.0.0",
-    "it-drain": "^3.0.5",
+    "execa": "^8.0.1",
+    "fast-glob": "^3.3.2",
+    "ipfsd-ctl": "^14.1.0",
     "kubo": "^0.27.0",
-    "kubo-rpc-client": "^3.0.4",
+    "kubo-rpc-client": "^4.1.1",
     "magic-bytes.js": "^1.10.0",
     "multiformats": "^13.1.0"
   },

packages/interop/src/bin.ts

@@ -1,5 +1,4 @@
 #! /usr/bin/env node
-/* eslint-disable no-console */
 
 import { spawn } from 'node:child_process'
 import { dirname, resolve } from 'node:path'

packages/interop/src/fixtures/create-kubo.ts

@@ -1,14 +1,15 @@
-/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */
-import { createController, type Controller } from 'ipfsd-ctl'
+import { createNode, type KuboNode } from 'ipfsd-ctl'
 import { path as kuboPath } from 'kubo'
-import * as kuboRpcClient from 'kubo-rpc-client'
+import { create } from 'kubo-rpc-client'
 
-export async function createKuboNode (): Promise<Controller> {
-  return createController({
-    kuboRpcModule: kuboRpcClient,
-    ipfsBin: kuboPath(),
+export async function createKuboNode (repoPath = undefined): Promise<KuboNode> {
+  return createNode({
+    type: 'kubo',
+    rpc: create,
+    bin: kuboPath(),
     test: true,
-    ipfsOptions: {
+    repo: repoPath,
+    init: {
       config: {
         Addresses: {
           Swarm: [

packages/interop/src/fixtures/load-fixtures.ts

@@ -0,0 +1,25 @@
+import { $ } from 'execa'
+import fg from 'fast-glob'
+import { path as kuboPath } from 'kubo'
+
+/**
+ * Only callable from node (intended to be consumed by .aegir.js)
+ * but the fixtures loaded by this function are also used by browser tests.
+ */
+export async function loadFixtures (IPFS_PATH = undefined): Promise<void> {
+  const kuboBinary = process.env.KUBO_BINARY ?? kuboPath()
+  /**
+   * fast-glob does not like windows paths, see https://github.com/mrmlnc/fast-glob/issues/237
+   * fast-glob performs search from process.cwd() by default, which will be:
+   * 1. the root of the monorepo when running tests in CI
+   * 2. the package root when running tests in the package directory
+   */
+  let globRoot = process.cwd().replace(/\\/g, '/')
+  if (!globRoot.includes('packages/interop')) {
+    // we only want car files from the interop package
+    globRoot = [...globRoot.split('/'), 'packages/interop'].join('/')
+  }
+  for (const carFile of await fg.glob('src/fixtures/data/*.car', { cwd: globRoot })) {
+    await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}`
+  }
+}

packages/interop/src/json.spec.ts

@@ -2,31 +2,23 @@
 import { createVerifiedFetch } from '@helia/verified-fetch'
 import { expect } from 'aegir/chai'
 import { CID } from 'multiformats/cid'
-import { createKuboNode } from './fixtures/create-kubo.js'
-import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
-import type { Controller } from 'ipfsd-ctl'
 
 describe('@helia/verified-fetch - json', () => {
   describe('unixfs - multiblock', () => {
-    let controller: Controller<'go'>
     let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>
 
     before(async () => {
-      controller = await createKuboNode()
-      await controller.start()
       // As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to:
       // root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr
       // child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON
       // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON
-      await loadFixtureDataCar(controller, 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car')
       verifiedFetch = await createVerifiedFetch({
-        gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
-        routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+        gateways: ['http://127.0.0.1:8180'],
+        routers: []
       })
     })
 
     after(async () => {
-      await controller.stop()
       await verifiedFetch.stop()
     })
 

packages/interop/src/unixfs-dir.spec.ts

@@ -2,35 +2,24 @@
 import { createVerifiedFetch } from '@helia/verified-fetch'
 import { expect } from 'aegir/chai'
 import { filetypemime } from 'magic-bytes.js'
-import { createKuboNode } from './fixtures/create-kubo.js'
-import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
 import type { VerifiedFetch } from '@helia/verified-fetch'
-import type { Controller } from 'ipfsd-ctl'
 
 describe('@helia/verified-fetch - unixfs directory', () => {
-  let controller: Controller
   let verifiedFetch: VerifiedFetch
 
   before(async () => {
-    controller = await createKuboNode()
-    await controller.start()
-
     verifiedFetch = await createVerifiedFetch({
-      gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
-      routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+      gateways: ['http://127.0.0.1:8180'],
+      routers: []
     })
+    verifiedFetch = await createVerifiedFetch()
   })
 
   after(async () => {
-    await controller.stop()
     await verifiedFetch.stop()
   })
 
   describe('unixfs-dir-redirect', () => {
-    before(async () => {
-      await loadFixtureDataCar(controller, 'gateway-conformance-fixtures.car')
-    });
-
     [
       'https://example.com/ipfs/bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q',
       'ipfs://bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q',
@@ -45,12 +34,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
     })
   })
 
+  // This tests the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201
   describe('XKCD Barrel Part 1', () => {
-    before(async () => {
-      // This is the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201
-      await loadFixtureDataCar(controller, 'QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car')
-    })
-
     it('fails to load when passed the root', async () => {
       // The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure
       const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR')
@@ -78,8 +63,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
     before(async () => {
       await verifiedFetch.stop()
       verifiedFetch = await createVerifiedFetch({
-        gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
-        routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+        gateways: ['http://127.0.0.1:8180'],
+        routers: []
       }, {
         contentTypeParser: (bytes) => {
           return filetypemime(bytes)?.[0]
@@ -94,12 +79,8 @@ describe('@helia/verified-fetch - unixfs directory', () => {
     })
   })
 
+  // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
   describe('HAMT-sharded directory', () => {
-    before(async () => {
-      // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car
-      await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car')
-    })
-
     it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => {
       const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt')
       expect(resp).to.be.ok()

packages/interop/src/websites.spec.ts

@@ -1,28 +1,20 @@
 /* eslint-env mocha */
 import { createVerifiedFetch } from '@helia/verified-fetch'
 import { expect } from 'aegir/chai'
-import { createKuboNode } from './fixtures/create-kubo.js'
-import { loadFixtureDataCar } from './fixtures/load-fixture-data.js'
-import type { Controller } from 'ipfsd-ctl'
 
 describe('@helia/verified-fetch - websites', () => {
   describe('helia-identify.on.fleek.co', () => {
-    let controller: Controller<'go'>
     let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>
 
     before(async () => {
-      controller = await createKuboNode()
-      await controller.start()
       // 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co
-      await loadFixtureDataCar(controller, 'QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car')
       verifiedFetch = await createVerifiedFetch({
-        gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
-        routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+        gateways: ['http://127.0.0.1:8180'],
+        routers: []
       })
     })
 
     after(async () => {
-      await controller.stop()
       await verifiedFetch.stop()
     })
 
@@ -55,21 +47,16 @@ describe('@helia/verified-fetch - websites', () => {
    * ```
    */
   describe('fake blog.libp2p.io', () => {
-    let controller: Controller<'go'>
     let verifiedFetch: Awaited<ReturnType<typeof createVerifiedFetch>>
 
     before(async () => {
-      controller = await createKuboNode()
-      await controller.start()
-      await loadFixtureDataCar(controller, 'QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car')
       verifiedFetch = await createVerifiedFetch({
-        gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`],
-        routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`]
+        gateways: ['http://127.0.0.1:8180'],
+        routers: []
       })
     })
 
     after(async () => {
-      await controller.stop()
       await verifiedFetch.stop()
     })