From 2b2e93b7fadc39a1829cc8d0a3309dd9dff1e475 Mon Sep 17 00:00:00 2001 From: Russell Dempsey <1173416+SgtPooki@users.noreply.github.com> Date: Fri, 3 May 2024 12:42:55 -0700 Subject: [PATCH] fix: prevent interop timeouts with fast fixture loading (#72) * deps: update ipfsd-ctl and kubo-rpc-client * chore: apply suggestions from code review * tmp: get some debug logs from CI jobs * fix: disable delegated routing in vfetch-interop tests * fix: test:node on windows * Revert "tmp: get some debug logs from CI jobs" This reverts commit 3654a287653199d8027c2a3b845da952b7cde4dc. --- packages/interop/.aegir.js | 54 +++++++------------ packages/interop/package.json | 7 +-- packages/interop/src/bin.ts | 1 - .../src/fixtures/create-kubo.browser.ts | 30 ----------- packages/interop/src/fixtures/create-kubo.ts | 17 +++--- .../interop/src/fixtures/load-fixture-data.ts | 9 ---- .../interop/src/fixtures/load-fixtures.ts | 25 +++++++++ packages/interop/src/json.spec.ts | 12 +---- packages/interop/src/unixfs-dir.spec.ts | 33 +++--------- packages/interop/src/websites.spec.ts | 21 ++------ 10 files changed, 70 insertions(+), 139 deletions(-) delete mode 100644 packages/interop/src/fixtures/create-kubo.browser.ts delete mode 100644 packages/interop/src/fixtures/load-fixture-data.ts create mode 100644 packages/interop/src/fixtures/load-fixtures.ts diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js index 226c2746..a02b8beb 100644 --- a/packages/interop/.aegir.js +++ b/packages/interop/.aegir.js @@ -1,46 +1,30 @@ -import getPort from 'aegir/get-port' -import { createServer } from 'ipfsd-ctl' -import * as kuboRpcClient from 'kubo-rpc-client' +import { resolve } from 'node:path' +import { tmpdir } from 'node:os' + +const IPFS_PATH = resolve(tmpdir(), 'verified-fetch-interop-ipfs-repo') /** @type {import('aegir').PartialOptions} */ export default { test: { files: './dist/src/*.spec.js', - before: async (options) => { - if (options.runner !== 'node') { - const ipfsdPort = await getPort() - const ipfsdServer = await createServer({ - host: '127.0.0.1', - port: ipfsdPort - }, { - ipfsBin: (await import('kubo')).default.path(), - kuboRpcModule: kuboRpcClient, - ipfsOptions: { - config: { - Addresses: { - Swarm: [ - "/ip4/0.0.0.0/tcp/0", - "/ip4/0.0.0.0/tcp/0/ws" - ] - } - } - } - }).start() + before: async () => { - return { - env: { - IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` - }, - ipfsdServer - } - } + const { createKuboNode } = await import('./dist/src/fixtures/create-kubo.js') + const kuboNode = await createKuboNode(IPFS_PATH) - return {} - }, - after: async (options, beforeResult) => { - if (options.runner !== 'node') { - await beforeResult.ipfsdServer.stop() + await kuboNode.start() + + // requires aegir build to be run first, which it will by default. + const { loadFixtures } = await import('./dist/src/fixtures/load-fixtures.js') + + await loadFixtures(IPFS_PATH) + + return { + kuboNode } + }, + after: async (_options, beforeResult) => { + await beforeResult.kuboNode.stop() } } } diff --git a/packages/interop/package.json b/packages/interop/package.json index 98b5dbe1..2b3b4d98 100644 --- a/packages/interop/package.json +++ b/packages/interop/package.json @@ -59,10 +59,11 @@ "dependencies": { "@helia/verified-fetch": "1.3.14", "aegir": "^42.2.5", - "ipfsd-ctl": "^13.0.0", - "it-drain": "^3.0.5", + "execa": "^8.0.1", + "fast-glob": "^3.3.2", + "ipfsd-ctl": "^14.1.0", "kubo": "^0.27.0", - "kubo-rpc-client": "^3.0.4", + "kubo-rpc-client": "^4.1.1", "magic-bytes.js": "^1.10.0", "multiformats": "^13.1.0" }, diff --git a/packages/interop/src/bin.ts b/packages/interop/src/bin.ts index e2ca51a3..e5bc829b 100644 --- a/packages/interop/src/bin.ts +++ b/packages/interop/src/bin.ts @@ -1,5 +1,4 @@ #! /usr/bin/env node -/* eslint-disable no-console */ import { spawn } from 'node:child_process' import { dirname, resolve } from 'node:path' diff --git a/packages/interop/src/fixtures/create-kubo.browser.ts b/packages/interop/src/fixtures/create-kubo.browser.ts deleted file mode 100644 index 8eacb7da..00000000 --- a/packages/interop/src/fixtures/create-kubo.browser.ts +++ /dev/null @@ -1,30 +0,0 @@ -import { type Controller, createController } from 'ipfsd-ctl' -import * as kuboRpcClient from 'kubo-rpc-client' - -export async function createKuboNode (): Promise { - return createController({ - kuboRpcModule: kuboRpcClient, - test: true, - endpoint: process.env.IPFSD_SERVER, - ipfsOptions: { - config: { - Addresses: { - Swarm: [ - '/ip4/0.0.0.0/tcp/0', - '/ip4/0.0.0.0/tcp/0/ws' - ], - Gateway: '/ip4/127.0.0.1/tcp/8180' - }, - Gateway: { - NoFetch: true, - ExposeRoutingAPI: true, - HTTPHeaders: { - 'Access-Control-Allow-Origin': ['*'], - 'Access-Control-Allow-Methods': ['GET', 'POST', 'PUT', 'OPTIONS'] - } - } - } - }, - args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] - }) -} diff --git a/packages/interop/src/fixtures/create-kubo.ts b/packages/interop/src/fixtures/create-kubo.ts index a632b905..b167d765 100644 --- a/packages/interop/src/fixtures/create-kubo.ts +++ b/packages/interop/src/fixtures/create-kubo.ts @@ -1,14 +1,15 @@ -/* eslint-disable @typescript-eslint/ban-ts-comment,@typescript-eslint/prefer-ts-expect-error */ -import { createController, type Controller } from 'ipfsd-ctl' +import { createNode, type KuboNode } from 'ipfsd-ctl' import { path as kuboPath } from 'kubo' -import * as kuboRpcClient from 'kubo-rpc-client' +import { create } from 'kubo-rpc-client' -export async function createKuboNode (): Promise { - return createController({ - kuboRpcModule: kuboRpcClient, - ipfsBin: kuboPath(), +export async function createKuboNode (repoPath = undefined): Promise { + return createNode({ + type: 'kubo', + rpc: create, + bin: kuboPath(), test: true, - ipfsOptions: { + repo: repoPath, + init: { config: { Addresses: { Swarm: [ diff --git a/packages/interop/src/fixtures/load-fixture-data.ts b/packages/interop/src/fixtures/load-fixture-data.ts deleted file mode 100644 index 12c14c4b..00000000 --- a/packages/interop/src/fixtures/load-fixture-data.ts +++ /dev/null @@ -1,9 +0,0 @@ -import loadFixture from 'aegir/fixtures' -import drain from 'it-drain' -import type { Controller } from 'ipfsd-ctl' - -export async function loadFixtureDataCar (controller: Controller, path: string): Promise { - const fixtureData = `src/fixtures/data/${path}` - const buf = loadFixture(fixtureData) - await drain(controller.api.dag.import([buf])) -} diff --git a/packages/interop/src/fixtures/load-fixtures.ts b/packages/interop/src/fixtures/load-fixtures.ts new file mode 100644 index 00000000..54161499 --- /dev/null +++ b/packages/interop/src/fixtures/load-fixtures.ts @@ -0,0 +1,25 @@ +import { $ } from 'execa' +import fg from 'fast-glob' +import { path as kuboPath } from 'kubo' + +/** + * Only callable from node (intended to be consumed by .aegir.js) + * but the fixtures loaded by this function are also used by browser tests. + */ +export async function loadFixtures (IPFS_PATH = undefined): Promise { + const kuboBinary = process.env.KUBO_BINARY ?? kuboPath() + /** + * fast-glob does not like windows paths, see https://github.com/mrmlnc/fast-glob/issues/237 + * fast-glob performs search from process.cwd() by default, which will be: + * 1. the root of the monorepo when running tests in CI + * 2. the package root when running tests in the package directory + */ + let globRoot = process.cwd().replace(/\\/g, '/') + if (!globRoot.includes('packages/interop')) { + // we only want car files from the interop package + globRoot = [...globRoot.split('/'), 'packages/interop'].join('/') + } + for (const carFile of await fg.glob('src/fixtures/data/*.car', { cwd: globRoot })) { + await $({ env: { IPFS_PATH } })`${kuboBinary} dag import --pin-roots=false --offline ${carFile}` + } +} diff --git a/packages/interop/src/json.spec.ts b/packages/interop/src/json.spec.ts index 4536faab..ce1f519f 100644 --- a/packages/interop/src/json.spec.ts +++ b/packages/interop/src/json.spec.ts @@ -2,31 +2,23 @@ import { createVerifiedFetch } from '@helia/verified-fetch' import { expect } from 'aegir/chai' import { CID } from 'multiformats/cid' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' -import type { Controller } from 'ipfsd-ctl' describe('@helia/verified-fetch - json', () => { describe('unixfs - multiblock', () => { - let controller: Controller<'go'> let verifiedFetch: Awaited> before(async () => { - controller = await createKuboNode() - await controller.start() // As of 2024-01-18, https://cloudflare-ipfs.com/ipns/tokens.uniswap.org resolves to: // root: QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr // child1: QmNik5N4ryNwzzXYq5hCYKGcRjAf9QtigxtiJh9o8aXXbG // partial JSON // child2: QmWNBJX6fZyNTLWNYBHxAHpBctCP43R2zeqV2G8uavqFZn // partial JSON - await loadFixtureDataCar(controller, 'QmQJ8fxavY54CUsxMSx9aE9Rdcmvhx8awJK2jzJp4iAqCr-tokens.uniswap.org-2024-01-18.car') verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] + gateways: ['http://127.0.0.1:8180'], + routers: [] }) }) after(async () => { - await controller.stop() await verifiedFetch.stop() }) diff --git a/packages/interop/src/unixfs-dir.spec.ts b/packages/interop/src/unixfs-dir.spec.ts index 94d109c4..013609d8 100644 --- a/packages/interop/src/unixfs-dir.spec.ts +++ b/packages/interop/src/unixfs-dir.spec.ts @@ -2,35 +2,24 @@ import { createVerifiedFetch } from '@helia/verified-fetch' import { expect } from 'aegir/chai' import { filetypemime } from 'magic-bytes.js' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' import type { VerifiedFetch } from '@helia/verified-fetch' -import type { Controller } from 'ipfsd-ctl' describe('@helia/verified-fetch - unixfs directory', () => { - let controller: Controller let verifiedFetch: VerifiedFetch before(async () => { - controller = await createKuboNode() - await controller.start() - verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] + gateways: ['http://127.0.0.1:8180'], + routers: [] }) + verifiedFetch = await createVerifiedFetch() }) after(async () => { - await controller.stop() await verifiedFetch.stop() }) describe('unixfs-dir-redirect', () => { - before(async () => { - await loadFixtureDataCar(controller, 'gateway-conformance-fixtures.car') - }); - [ 'https://example.com/ipfs/bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q', 'ipfs://bafybeifq2rzpqnqrsdupncmkmhs3ckxxjhuvdcbvydkgvch3ms24k5lo7q', @@ -45,12 +34,8 @@ describe('@helia/verified-fetch - unixfs directory', () => { }) }) + // This tests the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201 describe('XKCD Barrel Part 1', () => { - before(async () => { - // This is the content of https://explore.ipld.io/#/explore/QmdmQXB2mzChmMeKY47C43LxUdg1NDJ5MWcKMKxDu7RgQm/1%20-%20Barrel%20-%20Part%201 - await loadFixtureDataCar(controller, 'QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR-xkcd-Barrel-part-1.car') - }) - it('fails to load when passed the root', async () => { // The spec says we should generate HTML with directory listings, but we don't do that yet, so expect a failure const resp = await verifiedFetch('ipfs://QmbQDovX7wRe9ek7u6QXe9zgCXkTzoUSsTFJEkrYV1HrVR') @@ -78,8 +63,8 @@ describe('@helia/verified-fetch - unixfs directory', () => { before(async () => { await verifiedFetch.stop() verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] + gateways: ['http://127.0.0.1:8180'], + routers: [] }, { contentTypeParser: (bytes) => { return filetypemime(bytes)?.[0] @@ -94,12 +79,8 @@ describe('@helia/verified-fetch - unixfs directory', () => { }) }) + // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car describe('HAMT-sharded directory', () => { - before(async () => { - // from https://github.com/ipfs/gateway-conformance/blob/193833b91f2e9b17daf45c84afaeeae61d9d7c7e/fixtures/trustless_gateway_car/single-layer-hamt-with-multi-block-files.car - await loadFixtureDataCar(controller, 'bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i-single-layer-hamt-with-multi-block-files.car') - }) - it('loads path /ipfs/bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt', async () => { const resp = await verifiedFetch('ipfs://bafybeidbclfqleg2uojchspzd4bob56dqetqjsj27gy2cq3klkkgxtpn4i/685.txt') expect(resp).to.be.ok() diff --git a/packages/interop/src/websites.spec.ts b/packages/interop/src/websites.spec.ts index 287e6638..9ca6bad4 100644 --- a/packages/interop/src/websites.spec.ts +++ b/packages/interop/src/websites.spec.ts @@ -1,28 +1,20 @@ /* eslint-env mocha */ import { createVerifiedFetch } from '@helia/verified-fetch' import { expect } from 'aegir/chai' -import { createKuboNode } from './fixtures/create-kubo.js' -import { loadFixtureDataCar } from './fixtures/load-fixture-data.js' -import type { Controller } from 'ipfsd-ctl' describe('@helia/verified-fetch - websites', () => { describe('helia-identify.on.fleek.co', () => { - let controller: Controller<'go'> let verifiedFetch: Awaited> before(async () => { - controller = await createKuboNode() - await controller.start() // 2024-01-22 CID for _dnslink.helia-identify.on.fleek.co - await loadFixtureDataCar(controller, 'QmbxpRxwKXxnJQjnPqm1kzDJSJ8YgkLxH23mcZURwPHjGv-helia-identify-website.car') verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] + gateways: ['http://127.0.0.1:8180'], + routers: [] }) }) after(async () => { - await controller.stop() await verifiedFetch.stop() }) @@ -55,21 +47,16 @@ describe('@helia/verified-fetch - websites', () => { * ``` */ describe('fake blog.libp2p.io', () => { - let controller: Controller<'go'> let verifiedFetch: Awaited> before(async () => { - controller = await createKuboNode() - await controller.start() - await loadFixtureDataCar(controller, 'QmeiDMLtPUS3RT2xAcUwsNyZz169wPke2q7im9vZpVLSYw-fake-blog.libp2p.io.car') verifiedFetch = await createVerifiedFetch({ - gateways: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`], - routers: [`http://${controller.api.gatewayHost}:${controller.api.gatewayPort}`] + gateways: ['http://127.0.0.1:8180'], + routers: [] }) }) after(async () => { - await controller.stop() await verifiedFetch.stop() })