From dc6f4eb8c1b7fd32f72fc270dec8a783a1b98967 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 16:43:21 +0100 Subject: [PATCH 01/10] feat: initial implementation --- .github/dependabot.yml | 11 + .github/workflows/automerge.yml | 8 + .github/workflows/js-test-and-release.yml | 143 +++++++++ .gitignore | 8 + LICENSE | 4 + LICENSE-APACHE | 5 + LICENSE-MIT | 19 ++ package.json | 50 ++++ packages/interop/.aegir.js | 45 +++ packages/interop/LICENSE | 4 + packages/interop/LICENSE-APACHE | 5 + packages/interop/LICENSE-MIT | 19 ++ packages/interop/README.md | 53 ++++ packages/interop/package.json | 198 +++++++++++++ packages/interop/src/index.ts | 1 + packages/interop/test/dht.spec.ts | 148 ++++++++++ .../test/fixtures/create-helia.browser.ts | 62 ++++ .../interop/test/fixtures/create-helia.ts | 46 +++ packages/interop/test/fixtures/create-kubo.ts | 28 ++ .../interop/test/fixtures/create-peer-ids.ts | 46 +++ packages/interop/test/fixtures/wait-for.ts | 27 ++ packages/interop/test/pubsub.spec.ts | 178 ++++++++++++ packages/interop/tsconfig.json | 15 + packages/ipns/LICENSE | 4 + packages/ipns/LICENSE-APACHE | 5 + packages/ipns/LICENSE-MIT | 19 ++ packages/ipns/README.md | 53 ++++ packages/ipns/package.json | 189 ++++++++++++ packages/ipns/src/index.ts | 272 ++++++++++++++++++ packages/ipns/src/routing/dht.ts | 68 +++++ packages/ipns/src/routing/index.ts | 9 + packages/ipns/src/routing/pubsub.ts | 178 ++++++++++++ packages/ipns/src/utils/local-store.ts | 44 +++ .../src/utils/resolve-dns-link.browser.ts | 61 ++++ packages/ipns/src/utils/resolve-dns-link.ts | 65 +++++ packages/ipns/src/utils/tlru.ts | 52 ++++ packages/ipns/test/publish.spec.ts | 39 +++ packages/ipns/test/resolve.spec.ts | 65 +++++ packages/ipns/tsconfig.json | 10 + 39 files changed, 2256 insertions(+) create mode 100644 .github/dependabot.yml create mode 100644 .github/workflows/automerge.yml create mode 100644 .github/workflows/js-test-and-release.yml create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 LICENSE-APACHE create mode 100644 LICENSE-MIT create mode 100644 package.json create mode 100644 packages/interop/.aegir.js create mode 100644 packages/interop/LICENSE create mode 100644 packages/interop/LICENSE-APACHE create mode 100644 packages/interop/LICENSE-MIT create mode 100644 packages/interop/README.md create mode 100644 packages/interop/package.json create mode 100644 packages/interop/src/index.ts create mode 100644 packages/interop/test/dht.spec.ts create mode 100644 packages/interop/test/fixtures/create-helia.browser.ts create mode 100644 packages/interop/test/fixtures/create-helia.ts create mode 100644 packages/interop/test/fixtures/create-kubo.ts create mode 100644 packages/interop/test/fixtures/create-peer-ids.ts create mode 100644 packages/interop/test/fixtures/wait-for.ts create mode 100644 packages/interop/test/pubsub.spec.ts create mode 100644 packages/interop/tsconfig.json create mode 100644 packages/ipns/LICENSE create mode 100644 packages/ipns/LICENSE-APACHE create mode 100644 packages/ipns/LICENSE-MIT create mode 100644 packages/ipns/README.md create mode 100644 packages/ipns/package.json create mode 100644 packages/ipns/src/index.ts create mode 100644 packages/ipns/src/routing/dht.ts create mode 100644 packages/ipns/src/routing/index.ts create mode 100644 packages/ipns/src/routing/pubsub.ts create mode 100644 packages/ipns/src/utils/local-store.ts create mode 100644 packages/ipns/src/utils/resolve-dns-link.browser.ts create mode 100644 packages/ipns/src/utils/resolve-dns-link.ts create mode 100644 packages/ipns/src/utils/tlru.ts create mode 100644 packages/ipns/test/publish.spec.ts create mode 100644 packages/ipns/test/resolve.spec.ts create mode 100644 packages/ipns/tsconfig.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..0bc3b42 --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,11 @@ +version: 2 +updates: +- package-ecosystem: npm + directory: "/" + schedule: + interval: daily + time: "10:00" + open-pull-requests-limit: 10 + commit-message: + prefix: "deps" + prefix-development: "deps(dev)" diff --git a/.github/workflows/automerge.yml b/.github/workflows/automerge.yml new file mode 100644 index 0000000..d57c2a0 --- /dev/null +++ b/.github/workflows/automerge.yml @@ -0,0 +1,8 @@ +name: Automerge +on: [ pull_request ] + +jobs: + automerge: + uses: protocol/.github/.github/workflows/automerge.yml@master + with: + job: 'automerge' diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml new file mode 100644 index 0000000..27fd45a --- /dev/null +++ b/.github/workflows/js-test-and-release.yml @@ -0,0 +1,143 @@ +name: test & maybe release +on: + push: + branches: + - ${{{ github.default_branch }}} + pull_request: + +jobs: + + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present lint + - run: npm run --if-present dep-check + + test-node: + needs: check + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [windows-latest, ubuntu-latest, macos-latest] + node: [lts/*] + fail-fast: true + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node }} + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:node + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: node + + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: chrome + + test-chrome-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:chrome-webworker + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: chrome-webworker + + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: firefox + + test-firefox-webworker: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npm run --if-present test:firefox-webworker + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: firefox-webworker + + test-electron-main: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-main + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: electron-main + + test-electron-renderer: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - run: npx xvfb-maybe npm run --if-present test:electron-renderer + - uses: codecov/codecov-action@81cd2dc8148241f03f5839d295e000b8f761e378 # v3.1.0 + with: + flags: electron-renderer + + release: + needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] + runs-on: ubuntu-latest + if: github.event_name == 'push' && github.ref == 'refs/heads/${{{ github.default_branch }}}' + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + - uses: actions/setup-node@v3 + with: + node-version: lts/* + - uses: ipfs/aegir/actions/cache-node-modules@master + - uses: ipfs/aegir/actions/docker-login@master + with: + docker-token: ${{ secrets.DOCKER_TOKEN }} + docker-username: ${{ secrets.DOCKER_USERNAME }} + - run: npm run --if-present release + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..910f633 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +node_modules +build +dist +.docs +.coverage +node_modules +package-lock.json +yarn.lock diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..20ce483 --- /dev/null +++ b/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/LICENSE-APACHE b/LICENSE-APACHE new file mode 100644 index 0000000..14478a3 --- /dev/null +++ b/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/LICENSE-MIT b/LICENSE-MIT new file mode 100644 index 0000000..72dc60d --- /dev/null +++ b/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/package.json b/package.json new file mode 100644 index 0000000..0ad2b58 --- /dev/null +++ b/package.json @@ -0,0 +1,50 @@ +{ + "name": "@helia/ipns", + "version": "0.0.0", + "description": "An implementation of IPNS for Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "ipfs" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "private": true, + "scripts": { + "reset": "aegir run clean && aegir clean **/node_modules **/package-lock.json", + "test": "aegir run test", + "test:node": "aegir run test:node", + "test:chrome": "aegir run test:chrome", + "test:chrome-webworker": "aegir run test:chrome-webworker", + "test:firefox": "aegir run test:firefox", + "test:firefox-webworker": "aegir run test:firefox-webworker", + "test:electron-main": "aegir run test:electron-main", + "test:electron-renderer": "aegir run test:electron-renderer", + "clean": "aegir run clean", + "generate": "aegir run generate", + "build": "aegir run build", + "lint": "aegir run lint", + "docs": "NODE_OPTIONS=--max_old_space_size=4096 aegir docs", + "docs:no-publish": "npm run docs -- --publish false", + "dep-check": "aegir run dep-check", + "release": "npm run docs:no-publish && npm run release:npm && npm run docs", + "release:npm": "aegir exec npm -- publish", + "release:rc": "aegir release-rc" + }, + "dependencies": { + "aegir": "^38.1.0" + }, + "type": "module", + "workspaces": [ + "packages/*" + ] +} diff --git a/packages/interop/.aegir.js b/packages/interop/.aegir.js new file mode 100644 index 0000000..498799a --- /dev/null +++ b/packages/interop/.aegir.js @@ -0,0 +1,45 @@ +import getPort from 'aegir/get-port' +import { createServer } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' + +/** @type {import('aegir').PartialOptions} */ +export default { + test: { + before: async (options) => { + if (options.runner !== 'node') { + const ipfsdPort = await getPort() + const ipfsdServer = await createServer({ + host: '127.0.0.1', + port: ipfsdPort + }, { + ipfsBin: (await import('go-ipfs')).default.path(), + kuboRpcModule: kuboRpcClient, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + "/ip4/0.0.0.0/tcp/4001", + "/ip4/0.0.0.0/tcp/4002/ws" + ] + } + } + } + }).start() + + return { + env: { + IPFSD_SERVER: `http://127.0.0.1:${ipfsdPort}` + }, + ipfsdServer + } + } + + return {} + }, + after: async (options, beforeResult) => { + if (options.runner !== 'node') { + await beforeResult.ipfsdServer.stop() + } + } + } +} diff --git a/packages/interop/LICENSE b/packages/interop/LICENSE new file mode 100644 index 0000000..20ce483 --- /dev/null +++ b/packages/interop/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/interop/LICENSE-APACHE b/packages/interop/LICENSE-APACHE new file mode 100644 index 0000000..14478a3 --- /dev/null +++ b/packages/interop/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/interop/LICENSE-MIT b/packages/interop/LICENSE-MIT new file mode 100644 index 0000000..72dc60d --- /dev/null +++ b/packages/interop/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/interop/README.md b/packages/interop/README.md new file mode 100644 index 0000000..c99e295 --- /dev/null +++ b/packages/interop/README.md @@ -0,0 +1,53 @@ +# @helia/ipns-interop + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> Interop tests for @helia/ipns + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/interop/package.json b/packages/interop/package.json new file mode 100644 index 0000000..aebe1cf --- /dev/null +++ b/packages/interop/package.json @@ -0,0 +1,198 @@ +{ + "name": "@helia/ipns-interop", + "version": "0.0.0", + "description": "Interop tests for @helia/ipns", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/interop#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./routing": { + "types": "./dist/src/routing/index.d.ts", + "import": "./dist/src/routing/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "devDependencies": { + "@chainsafe/libp2p-gossipsub": "^6.1.0", + "@chainsafe/libp2p-noise": "^11.0.0", + "@chainsafe/libp2p-yamux": "^3.0.5", + "@helia/interface": "next", + "@helia/ipns": "~0.0.0", + "@libp2p/interface-peer-id": "^2.0.1", + "@libp2p/kad-dht": "^7.0.0", + "@libp2p/peer-id": "^2.0.1", + "@libp2p/peer-id-factory": "^2.0.1", + "@libp2p/tcp": "^6.1.2", + "@libp2p/websockets": "^5.0.3", + "aegir": "^38.1.0", + "blockstore-core": "^3.0.0", + "datastore-core": "^8.0.4", + "go-ipfs": "^0.18.1", + "helia": "next", + "ipfsd-ctl": "^13.0.0", + "ipns": "^5.0.1", + "it-all": "^2.0.0", + "it-last": "^2.0.0", + "it-map": "^2.0.0", + "kubo-rpc-client": "^3.0.0", + "libp2p": "^0.42.2", + "merge-options": "^3.0.4", + "multiformats": "^11.0.1", + "uint8arrays": "^4.0.3", + "wherearewe": "^2.0.1" + }, + "browser": { + "./dist/test/fixtures/create-helia.js": "./dist/test/fixtures/create-helia.browser.js", + "go-ipfs": false + }, + "private": true, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/interop/src/index.ts b/packages/interop/src/index.ts new file mode 100644 index 0000000..336ce12 --- /dev/null +++ b/packages/interop/src/index.ts @@ -0,0 +1 @@ +export {} diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts new file mode 100644 index 0000000..7738ed0 --- /dev/null +++ b/packages/interop/test/dht.spec.ts @@ -0,0 +1,148 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import type { IPNS } from '@helia/ipns' +import { ipns } from '@helia/ipns' +import { dht } from '@helia/ipns/routing' +import last from 'it-last' +import { kadDHT } from '@libp2p/kad-dht' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import { sortClosestPeers } from './fixtures/create-peer-ids.js' +import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' + +describe('dht routing', () => { + let helia: Helia + let kubo: Controller + let name: IPNS + + // the CID we are going to publish + let value: CID + + // the public key we will use to publish the value + let key: PeerId + + /** + * Ensure that for the CID we are going to publish, the resolver has a peer ID that + * is KAD-closer to the routing key so we can predict the the resolver will receive + * the DHT record containing the IPNS record + */ + async function createNodes (resolver: 'kubo' | 'helia'): Promise { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + value = CID.createV1(raw.code, digest) + + helia = await createHeliaNode({ + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }) + }) + kubo = await createKuboNode() + + // find a PeerId that when used as an IPNS key is KAD-close to the resolver + while (true) { + key = await createEd25519PeerId() + const routingKey = uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + key.toBytes() + ]) + + const [closest] = await sortClosestPeers(routingKey, [ + helia.libp2p.peerId, + kubo.peer.id + ]) + + if (resolver === 'kubo' && closest.equals(kubo.peer.id)) { + break + } + + if (resolver === 'helia' && closest.equals(helia.libp2p.peerId)) { + break + } + } + + // connect the two nodes over the KAD-DHT protocol, this should ensure + // both nodes have each other in their KAD buckets + let connected = false + for (const addr of kubo.peer.addresses) { + try { + await helia.libp2p.dialProtocol(addr, '/ipfs/lan/kad/1.0.0') + connected = true + break + } catch { } + } + expect(connected).to.be.true() + + name = ipns(helia, [ + dht(helia) + ]) + } + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should publish on helia and resolve on kubo', async () => { + await createNodes('kubo') + + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.importPeer(keyName, key) + + await name.publish(key, value) + + const resolved = await last(kubo.api.name.resolve(key)) + + if (resolved == null) { + throw new Error('kubo failed to resolve name') + } + + expect(resolved).to.equal(`/ipfs/${value.toString()}`) + }) + + it('should publish on kubo and resolve on helia', async () => { + await createNodes('helia') + + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + + // ensure the key is in the kubo keychain so we can use it to publish the IPNS record + const body = new FormData() + body.append('key', new Blob([key.privateKey ?? new Uint8Array(0)])) + + // can't use the kubo-rpc-api for this call yet + const response = await fetch(`http://${kubo.api.apiHost}:${kubo.api.apiPort}/api/v0/key/import?arg=${keyName}`, { + method: 'POST', + body + }) + + expect(response).to.have.property('status', 200) + + await kubo.api.name.publish(cid, { + key: keyName + }) + + const resolvedCid = await name.resolve(key) + expect(resolvedCid.toString()).to.equal(cid.toString()) + }) +}) diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts new file mode 100644 index 0000000..6311c2a --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -0,0 +1,62 @@ +import { createHelia } from 'helia' +import { createLibp2p } from 'libp2p' +import { webSockets } from '@libp2p/websockets' +import { all } from '@libp2p/websockets/filters' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' +import { kadDHT } from '@libp2p/kad-dht' +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' + +export async function createHeliaNode (): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + // dial-only in the browser until webrtc browser-to-browser arrives + const libp2p = await createLibp2p({ + transports: [ + webSockets({ + filter: all + }) + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + dht: kadDHT({ + validators: { + ipns: ipnsValidator + }, + selectors: { + ipns: ipnsSelector + } + }), + pubsub: gossipsub(), + datastore, + identify: { + host: { + agentVersion: 'helia/0.0.0' + } + }, + nat: { + enabled: false + }, + relay: { + enabled: false + } + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-helia.ts b/packages/interop/test/fixtures/create-helia.ts new file mode 100644 index 0000000..881b545 --- /dev/null +++ b/packages/interop/test/fixtures/create-helia.ts @@ -0,0 +1,46 @@ +import { createHelia } from 'helia' +import { createLibp2p, Libp2pOptions } from 'libp2p' +import { tcp } from '@libp2p/tcp' +import { noise } from '@chainsafe/libp2p-noise' +import { yamux } from '@chainsafe/libp2p-yamux' +import { MemoryBlockstore } from 'blockstore-core' +import { MemoryDatastore } from 'datastore-core' +import type { Helia } from '@helia/interface' + +export async function createHeliaNode (config: Libp2pOptions = {}): Promise { + const blockstore = new MemoryBlockstore() + const datastore = new MemoryDatastore() + + const libp2p = await createLibp2p({ + addresses: { + listen: [ + '/ip4/0.0.0.0/tcp/0' + ] + }, + transports: [ + tcp() + ], + connectionEncryption: [ + noise() + ], + streamMuxers: [ + yamux() + ], + datastore, + nat: { + enabled: false + }, + relay: { + enabled: false + }, + ...config + }) + + const helia = await createHelia({ + libp2p, + blockstore, + datastore + }) + + return helia +} diff --git a/packages/interop/test/fixtures/create-kubo.ts b/packages/interop/test/fixtures/create-kubo.ts new file mode 100644 index 0000000..7fd4872 --- /dev/null +++ b/packages/interop/test/fixtures/create-kubo.ts @@ -0,0 +1,28 @@ + +// @ts-expect-error no types +import * as goIpfs from 'go-ipfs' +import { Controller, ControllerOptions, createController } from 'ipfsd-ctl' +import * as kuboRpcClient from 'kubo-rpc-client' +import { isElectronMain, isNode } from 'wherearewe' +import mergeOptions from 'merge-options' + +export async function createKuboNode (options: ControllerOptions<'go'> = {}): Promise { + const opts = mergeOptions({ + kuboRpcModule: kuboRpcClient, + ipfsBin: isNode || isElectronMain ? goIpfs.path() : undefined, + test: true, + endpoint: process.env.IPFSD_SERVER, + ipfsOptions: { + config: { + Addresses: { + Swarm: [ + '/ip4/0.0.0.0/tcp/4001', + '/ip4/0.0.0.0/tcp/4002/ws' + ] + } + } + } + }, options) + + return await createController(opts) +} diff --git a/packages/interop/test/fixtures/create-peer-ids.ts b/packages/interop/test/fixtures/create-peer-ids.ts new file mode 100644 index 0000000..c38834e --- /dev/null +++ b/packages/interop/test/fixtures/create-peer-ids.ts @@ -0,0 +1,46 @@ +import { xor as uint8ArrayXor } from 'uint8arrays/xor' +import { compare as uint8ArrayCompare } from 'uint8arrays/compare' +import all from 'it-all' +import map from 'it-map' +import type { PeerId } from '@libp2p/interface-peer-id' +import { sha256 } from 'multiformats/hashes/sha2' + +/** + * Sort peers by distance to the KadID of the passed buffer + */ +export async function sortClosestPeers (buf: Uint8Array, peers: PeerId[]): Promise { + const kadId = await convertBuffer(buf) + + const distances = await all( + map(peers, async (peer) => { + const id = await convertPeerId(peer) + + return { + peer, + distance: uint8ArrayXor(id, kadId) + } + }) + ) + + return distances + .sort((a, b) => { + return uint8ArrayCompare(a.distance, b.distance) + }) + .map((d) => d.peer) +} + +/** + * Creates a DHT ID by hashing a Peer ID + */ +export async function convertPeerId (peerId: PeerId): Promise { + return await convertBuffer(peerId.toBytes()) +} + +/** + * Creates a DHT ID by hashing a given Uint8Array + */ +export async function convertBuffer (buf: Uint8Array): Promise { + const multihash = await sha256.digest(buf) + + return multihash.digest +} diff --git a/packages/interop/test/fixtures/wait-for.ts b/packages/interop/test/fixtures/wait-for.ts new file mode 100644 index 0000000..3c1451f --- /dev/null +++ b/packages/interop/test/fixtures/wait-for.ts @@ -0,0 +1,27 @@ + +export interface WaitForOptions { + timeout: number + delay?: number + message?: string +} + +export async function waitFor (fn: () => Promise, options: WaitForOptions): Promise { + const delay = options.delay ?? 100 + const timeoutAt = Date.now() + options.timeout + + while (true) { + const result = await fn() + + if (result) { + return + } + + await new Promise((resolve) => { + setTimeout(() => { resolve() }, delay) + }) + + if (Date.now() > timeoutAt) { + throw new Error(options.message ?? 'WaitFor timed out') + } + } +} diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts new file mode 100644 index 0000000..a0ed278 --- /dev/null +++ b/packages/interop/test/pubsub.spec.ts @@ -0,0 +1,178 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { createHeliaNode } from './fixtures/create-helia.js' +import { createKuboNode } from './fixtures/create-kubo.js' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' +import { sha256 } from 'multiformats/hashes/sha2' +import { CID } from 'multiformats/cid' +import * as raw from 'multiformats/codecs/raw' +import { identity } from 'multiformats/hashes/identity' +import { base36 } from 'multiformats/bases/base36' +import type { IPNS } from '@helia/ipns' +import { ipns } from '@helia/ipns' +import { pubsub } from '@helia/ipns/routing' +import last from 'it-last' +import { peerIdFromKeys } from '@libp2p/peer-id' +import { gossipsub } from '@chainsafe/libp2p-gossipsub' +import { waitFor } from './fixtures/wait-for.js' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { concat as uint8ArrayConcat } from 'uint8arrays/concat' + +const LIBP2P_KEY_CODEC = 0x72 + +describe('pubsub routing', () => { + let helia: Helia + let kubo: Controller + let name: IPNS + + beforeEach(async () => { + helia = await createHeliaNode({ + pubsub: gossipsub() + }) + kubo = await createKuboNode({ + args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] + }) + + // connect the two nodes - serial dial seems more stable? + // connect the two nodes over the KAD-DHT protocol, this should ensure + // both nodes have each other in their KAD buckets + let connected = false + for (const addr of kubo.peer.addresses) { + try { + await helia.libp2p.dialProtocol(addr, '/meshsub/1.1.0') + connected = true + break + } catch { } + } + expect(connected).to.be.true() + + name = ipns(helia, [ + pubsub(helia) + ]) + }) + + afterEach(async () => { + if (helia != null) { + await helia.stop() + } + + if (kubo != null) { + await kubo.stop() + } + }) + + it('should publish on helia and resolve on kubo', async () => { + const input = Uint8Array.from([0, 1, 2, 3, 4]) + const digest = await sha256.digest(input) + const cid = CID.createV1(raw.code, digest) + + const keyName = 'my-ipns-key' + await helia.libp2p.keychain.createKey(keyName, 'Ed25519') + const peerId = await helia.libp2p.keychain.exportPeerId(keyName) + + if (peerId.publicKey == null) { + throw new Error('No public key present') + } + + // first publish should fail because kubo isn't subscribed to key update channel + await expect(name.publish(peerId, cid)).to.eventually.be.rejected() + .with.property('message', 'PublishError.InsufficientPeers') + + // should fail to resolve the first time as kubo was not subscribed to the pubsub channel + await expect(last(kubo.api.name.resolve(peerId, { + timeout: 100 + }))).to.eventually.be.undefined() + + // magic pubsub subscription name + const subscriptionName = `/ipns/${CID.createV1(LIBP2P_KEY_CODEC, identity.digest(peerId.publicKey)).toString(base36)}` + + // wait for kubo to be subscribed to updates + await waitFor(async () => { + const subs = await kubo.api.name.pubsub.subs() + + return subs.includes(subscriptionName) + }, { + timeout: 30000 + }) + + // publish should now succeed + await name.publish(peerId, cid) + + // kubo should now be able to resolve IPNS name + const resolved = await last(kubo.api.name.resolve(peerId, { + timeout: 100 + })) + + expect(resolved).to.equal(`/ipfs/${cid.toString()}`) + }) + + it('should publish on kubo and resolve on helia', async () => { + const keyName = 'my-ipns-key' + const { cid } = await kubo.api.add(Uint8Array.from([0, 1, 2, 3, 4])) + const result = await kubo.api.key.gen(keyName, { + // @ts-expect-error kubo needs this in lower case + type: 'ed25519' + }) + + // the generated id is libp2p-key CID with the public key as an identity multihash + const peerCid = CID.parse(result.id, base36) + const peerId = await peerIdFromKeys(peerCid.multihash.digest) + + // first call to pubsub resolver should fail but we should now be subscribed for updates + await expect(name.resolve(peerId)).to.eventually.be.rejected() + + // actual pubsub subscription name + const subscriptionName = `/record/${uint8ArrayToString(uint8ArrayConcat([ + uint8ArrayFromString('/ipns/'), + peerId.toBytes() + ]), 'base64url')}` + + // wait for helia to be subscribed to the topic for record updates + await waitFor(async () => { + return helia.libp2p.pubsub.getTopics().includes(subscriptionName) + }, { + timeout: 30000, + message: 'Helia did not register for record updates' + }) + + // wait for kubo to see that helia is subscribed to the topic for record updates + await waitFor(async () => { + const peers = await kubo.api.pubsub.peers(subscriptionName) + + return peers.map(p => p.toString()).includes(helia.libp2p.peerId.toString()) + }, { + timeout: 30000, + message: 'Kubo did not see that Helia was registered for record updates' + }) + + // now publish, this should cause a pubsub message on the topic for record updates + await kubo.api.name.publish(cid, { + key: keyName + }) + + let resolvedCid: CID | undefined + + // we should get an update eventually + await waitFor(async () => { + try { + resolvedCid = await name.resolve(peerId) + + return true + } catch { + return false + } + }, { + timeout: 10000, + message: 'Helia could not resolve the IPNS record' + }) + + if (resolvedCid == null) { + throw new Error('Failed to resolve CID') + } + + expect(resolvedCid.toString()).to.equal(cid.toString()) + }) +}) diff --git a/packages/interop/tsconfig.json b/packages/interop/tsconfig.json new file mode 100644 index 0000000..61f84a6 --- /dev/null +++ b/packages/interop/tsconfig.json @@ -0,0 +1,15 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ], + "references": [ + { + "path": "../ipns" + } + ] +} diff --git a/packages/ipns/LICENSE b/packages/ipns/LICENSE new file mode 100644 index 0000000..20ce483 --- /dev/null +++ b/packages/ipns/LICENSE @@ -0,0 +1,4 @@ +This project is dual licensed under MIT and Apache-2.0. + +MIT: https://www.opensource.org/licenses/mit +Apache-2.0: https://www.apache.org/licenses/license-2.0 diff --git a/packages/ipns/LICENSE-APACHE b/packages/ipns/LICENSE-APACHE new file mode 100644 index 0000000..14478a3 --- /dev/null +++ b/packages/ipns/LICENSE-APACHE @@ -0,0 +1,5 @@ +Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. diff --git a/packages/ipns/LICENSE-MIT b/packages/ipns/LICENSE-MIT new file mode 100644 index 0000000..72dc60d --- /dev/null +++ b/packages/ipns/LICENSE-MIT @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/packages/ipns/README.md b/packages/ipns/README.md new file mode 100644 index 0000000..8798097 --- /dev/null +++ b/packages/ipns/README.md @@ -0,0 +1,53 @@ +# @helia/ipns + +[![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) +[![Discuss](https://img.shields.io/discourse/https/discuss.ipfs.tech/posts.svg?style=flat-square)](https://discuss.ipfs.tech) +[![codecov](https://img.shields.io/codecov/c/github/ipfs/helia-ipns.svg?style=flat-square)](https://codecov.io/gh/ipfs/helia-ipns) +[![CI](https://img.shields.io/github/actions/workflow/status/ipfs/helia-ipns/js-test-and-release.yml?branch=main\&style=flat-square)](https://github.com/ipfs/helia-ipns/actions/workflows/js-test-and-release.yml?query=branch%3Amain) + +> An implementation of IPNS for Helia + +## Table of contents + +- [Install](#install) + - [Browser ` +``` + +## API Docs + +- + +## License + +Licensed under either of + +- Apache 2.0, ([LICENSE-APACHE](LICENSE-APACHE) / ) +- MIT ([LICENSE-MIT](LICENSE-MIT) / ) + +## Contribute + +Contributions welcome! Please check out [the issues](https://github.com/ipfs/helia-ipns/issues). + +Also see our [contributing document](https://github.com/ipfs/community/blob/master/CONTRIBUTING_JS.md) for more information on how we work, and about contributing in general. + +Please be aware that all interactions related to this repo are subject to the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +Unless you explicitly state otherwise, any contribution intentionally submitted for inclusion in the work by you, as defined in the Apache-2.0 license, shall be dual licensed as above, without any additional terms or conditions. + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/CONTRIBUTING.md) diff --git a/packages/ipns/package.json b/packages/ipns/package.json new file mode 100644 index 0000000..0105a27 --- /dev/null +++ b/packages/ipns/package.json @@ -0,0 +1,189 @@ +{ + "name": "@helia/ipns", + "version": "0.0.0", + "description": "An implementation of IPNS for Helia", + "license": "Apache-2.0 OR MIT", + "homepage": "https://github.com/ipfs/helia-ipns/tree/master/packages/ipns#readme", + "repository": { + "type": "git", + "url": "git+https://github.com/ipfs/helia-ipns.git" + }, + "bugs": { + "url": "https://github.com/ipfs/helia-ipns/issues" + }, + "keywords": [ + "IPFS" + ], + "engines": { + "node": ">=16.0.0", + "npm": ">=7.0.0" + }, + "type": "module", + "types": "./dist/src/index.d.ts", + "typesVersions": { + "*": { + "*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ], + "src/*": [ + "*", + "dist/*", + "dist/src/*", + "dist/src/*/index" + ] + } + }, + "files": [ + "src", + "dist", + "!dist/test", + "!**/*.tsbuildinfo" + ], + "exports": { + ".": { + "types": "./dist/src/index.d.ts", + "import": "./dist/src/index.js" + }, + "./routing": { + "types": "./dist/src/routing/index.d.ts", + "import": "./dist/src/routing/index.js" + } + }, + "eslintConfig": { + "extends": "ipfs", + "parserOptions": { + "sourceType": "module" + } + }, + "release": { + "branches": [ + "main" + ], + "plugins": [ + [ + "@semantic-release/commit-analyzer", + { + "preset": "conventionalcommits", + "releaseRules": [ + { + "breaking": true, + "release": "major" + }, + { + "revert": true, + "release": "patch" + }, + { + "type": "feat", + "release": "minor" + }, + { + "type": "fix", + "release": "patch" + }, + { + "type": "docs", + "release": "patch" + }, + { + "type": "test", + "release": "patch" + }, + { + "type": "deps", + "release": "patch" + }, + { + "scope": "no-release", + "release": false + } + ] + } + ], + [ + "@semantic-release/release-notes-generator", + { + "preset": "conventionalcommits", + "presetConfig": { + "types": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "chore", + "section": "Trivial Changes" + }, + { + "type": "docs", + "section": "Documentation" + }, + { + "type": "deps", + "section": "Dependencies" + }, + { + "type": "test", + "section": "Tests" + } + ] + } + } + ], + "@semantic-release/changelog", + "@semantic-release/npm", + "@semantic-release/github", + "@semantic-release/git" + ] + }, + "scripts": { + "clean": "aegir clean", + "lint": "aegir lint", + "dep-check": "aegir dep-check", + "build": "aegir build", + "docs": "aegir docs", + "test": "aegir test", + "test:chrome": "aegir test -t browser --cov", + "test:chrome-webworker": "aegir test -t webworker", + "test:firefox": "aegir test -t browser -- --browser firefox", + "test:firefox-webworker": "aegir test -t webworker -- --browser firefox", + "test:node": "aegir test -t node --cov", + "test:electron-main": "aegir test -t electron-main", + "release": "aegir release" + }, + "dependencies": { + "@helia/interface": "next", + "@libp2p/interface-dht": "^2.0.1", + "@libp2p/interface-peer-id": "^2.0.1", + "@libp2p/interface-pubsub": "^3.0.6", + "@libp2p/interfaces": "^3.3.1", + "@libp2p/logger": "^2.0.5", + "@libp2p/peer-id": "^2.0.1", + "@libp2p/record": "^3.0.0", + "hashlru": "^2.3.0", + "interface-datastore": "^7.0.4", + "ipns": "^5.0.1", + "is-ipfs": "^8.0.1", + "multiformats": "^11.0.1", + "p-queue": "^7.3.0", + "uint8arrays": "^4.0.3" + }, + "devDependencies": { + "@libp2p/peer-id-factory": "^2.0.1", + "aegir": "^38.1.0", + "datastore-core": "^8.0.4" + }, + "browser": { + "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" + }, + "typedoc": { + "entryPoint": "./src/index.ts" + } +} diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts new file mode 100644 index 0000000..16365cc --- /dev/null +++ b/packages/ipns/src/index.ts @@ -0,0 +1,272 @@ +/** + * @packageDocumentation + * + * IPNS operations using a Helia node + * + * @example + * + * ```typescript + * import { gossipsub } from '@chainsafe/libp2p' + * import { kadDHT } from '@libp2p/kad-dht' + * import { createLibp2p } from 'libp2p' + * import { createHelia } from 'helia' + * import { ipns, ipnsValidator, ipnsSelector } from '@helia/ipns' + * import { dht, pubsub } from '@helia/ipns/routing' + * import { unixfs } from '@helia/unixfs + * + * const libp2p = await createLibp2p({ + * dht: kadDHT({ + * validators: { + * ipns: ipnsValidator + * }, + * selectors: { + * ipns: ipnsSelector + * } + * }), + * pubsub: gossipsub() + * }) + * + * const helia = await createHelia({ + * libp2p, + * //.. other options + * }) + * const name = ipns(helia, [ + * dht(helia) + * pubsub(helia) + * ]) + * + * // create a public key to publish as an IPNS name + * const keyInfo = await helia.libp2p.keychain.createKey('my-key') + * const peerId = await helia.libp2p.keychain.exportPeerId(keyInfo.name) + * + * // store some data to publish + * const fs = unixfs(helia) + * const cid = await fs.add(Uint8Array.from([0, 1, 2, 3, 4])) + * + * // publish the name + * await name.publish(peerId, cid) + * + * // resolve the name + * const cid = name.resolve(peerId) + * ``` + * + * @example + * + * ```typescript + * // resolve a CID from a TXT record in a DNS zone file, eg: + * // > dig ipfs.io TXT + * // ;; ANSWER SECTION: + * // ipfs.io. 435 IN TXT "dnslink=/ipfs/Qmfoo" + * + * const cid = name.resolveDns('ipfs.io') + * ``` + */ + +import type { AbortOptions } from '@libp2p/interfaces' +import { isPeerId, PeerId } from '@libp2p/interface-peer-id' +import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' +import type { IPNSEntry } from 'ipns' +import type { IPNSRouting } from './routing/index.js' +import { ipnsValidator } from 'ipns/validator' +import { CID } from 'multiformats/cid' +import { resolveDnslink } from './utils/resolve-dns-link.js' +import { logger } from '@libp2p/logger' +import { peerIdFromString } from '@libp2p/peer-id' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import type { Datastore } from 'interface-datastore' +import { localStore } from './utils/local-store.js' +import type { LocalStore } from './utils/local-store.js' + +const log = logger('helia:ipns') + +const MINUTE = 60 * 1000 +const HOUR = 60 * MINUTE + +const DEFAULT_LIFETIME_MS = 24 * HOUR +const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR + +export interface PublishOptions extends AbortOptions { + /** + * Time duration of the record in ms + */ + lifetime?: number +} + +export interface ResolveOptions extends AbortOptions { + /** + * do not use cached entries + */ + nocache?: boolean +} + +export interface RepublishOptions extends AbortOptions { + /** + * The republish interval in ms (default: 24hrs) + */ + interval?: number +} + +export interface IPNS { + /** + * Creates an IPNS record signed by the passed PeerId that will resolve to the passed value + * + * If the valid is a PeerId, a recursive IPNS record will be created. + */ + publish: (key: PeerId, value: CID | PeerId, options?: PublishOptions) => Promise + + /** + * Accepts a public key formatted as a libp2p PeerID and resolves the IPNS record + * corresponding to that public key until a value is found + */ + resolve: (key: PeerId, options?: ResolveOptions) => Promise + + /** + * Resolve a CID from a dns-link style IPNS record + */ + resolveDns: (domain: string, options?: ResolveOptions) => Promise + + /** + * Periodically republish all IPNS records found in the datastore + */ + republish: (options?: RepublishOptions) => void +} + +export type { IPNSRouting } from './routing/index.js' + +export interface IPNSComponents { + datastore: Datastore +} + +class DefaultIPNS implements IPNS { + private readonly routers: IPNSRouting[] + private readonly localStore: LocalStore + private timeout?: ReturnType + + constructor (components: IPNSComponents, routers: IPNSRouting[] = []) { + this.routers = routers + this.localStore = localStore(components.datastore) + } + + async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { + let sequenceNumber = 1n + const routingKey = peerIdToRoutingKey(key) + + if (await this.localStore.has(routingKey, options)) { + // if we have published under this key before, increment the sequence number + const buf = await this.localStore.get(routingKey, options) + const existingRecord = unmarshal(buf) + sequenceNumber = existingRecord.sequence + 1n + } + + let str + + if (isPeerId(value)) { + str = `/ipns/${value.toString()}` + } else { + str = `/ipfs/${value.toString()}` + } + + const bytes = uint8ArrayFromString(str) + + // create record + const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS) + const marshaledRecord = marshal(record) + + await this.localStore.put(routingKey, marshaledRecord, options) + + // publish record to routing + await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) + + return record + } + + async resolve (key: PeerId, options: ResolveOptions = {}): Promise { + const routingKey = peerIdToRoutingKey(key) + const record = await this.#findIpnsRecord(routingKey, options) + const str = uint8ArrayToString(record.value) + + return await this.#resolve(str) + } + + async resolveDns (domain: string, options: ResolveOptions = {}): Promise { + const dnslink = await resolveDnslink(domain, options) + + return await this.#resolve(dnslink) + } + + republish (options: RepublishOptions = {}): void { + if (this.timeout != null) { + throw new Error('Republish is already running') + } + + options.signal?.addEventListener('abort', () => { + clearTimeout(this.timeout) + }) + + async function republish (): Promise { + const startTime = Date.now() + const finishType = Date.now() + const timeTaken = finishType - startTime + let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken + + if (nextInterval < 0) { + nextInterval = options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS + } + + setTimeout(() => { + republish().catch(err => { + log.error('error republishing', err) + }) + }, nextInterval) + } + + this.timeout = setTimeout(() => { + republish().catch(err => { + log.error('error republishing', err) + }) + }, options.interval ?? DEFAULT_REPUBLISH_INTERVAL_MS) + } + + async #resolve (ipfsPath: string): Promise { + const parts = ipfsPath.split('/') + + if (parts.length === 3) { + const scheme = parts[1] + + if (scheme === 'ipns') { + return await this.resolve(peerIdFromString(parts[2])) + } else if (scheme === 'ipfs') { + return CID.parse(parts[2]) + } + } + + log.error('invalid ipfs path %s', ipfsPath) + throw new Error('Invalid value') + } + + async #findIpnsRecord (routingKey: Uint8Array, options: AbortOptions): Promise { + const routers = [ + this.localStore, + ...this.routers + ] + + const unmarshaledRecord = await Promise.any( + routers.map(async (router) => { + const unmarshaledRecord = await router.get(routingKey, options) + await ipnsValidator(routingKey, unmarshaledRecord) + + return unmarshaledRecord + }) + ) + + return unmarshal(unmarshaledRecord) + } +} + +export function ipns (components: IPNSComponents, routers: IPNSRouting[] = []): IPNS { + return new DefaultIPNS(components, routers) +} + +export { ipnsValidator } +export { ipnsSelector } from 'ipns/selector' diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts new file mode 100644 index 0000000..87efdc2 --- /dev/null +++ b/packages/ipns/src/routing/dht.ts @@ -0,0 +1,68 @@ +import { logger } from '@libp2p/logger' +import type { IPNSRouting } from '../index.js' +import type { DHT, QueryEvent } from '@libp2p/interface-dht' +import type { AbortOptions } from '@libp2p/interfaces' + +const log = logger('helia:ipns:routing:dht') + +export interface DHTRoutingComponents { + libp2p: { + dht: DHT + } +} + +export class DHTRouting implements IPNSRouting { + private readonly dht: DHT + + constructor (components: DHTRoutingComponents) { + this.dht = components.libp2p.dht + } + + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: AbortOptions = {}): Promise { + let putValue = false + + for await (const event of this.dht.put(routingKey, marshaledRecord, options)) { + logEvent('DHT put event', event) + + if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') { + putValue = true + } + } + + if (!putValue) { + throw new Error('Could not put value to DHT') + } + } + + async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + for await (const event of this.dht.get(routingKey, options)) { + logEvent('DHT get event', event) + + if (event.name === 'VALUE') { + return event.value + } + } + + throw new Error('Not found') + } +} + +function logEvent (prefix: string, event: QueryEvent): void { + if (event.name === 'SENDING_QUERY') { + log(prefix, event.name, event.messageName, '->', event.to.toString()) + } else if (event.name === 'PEER_RESPONSE') { + log(prefix, event.name, event.messageName, '<-', event.from.toString()) + } else if (event.name === 'FINAL_PEER') { + log(prefix, event.name, event.peer.id.toString()) + } else if (event.name === 'QUERY_ERROR') { + log(prefix, event.name, event.error.message) + } else if (event.name === 'PROVIDER') { + log(prefix, event.name, event.providers.map(p => p.id.toString()).join(', ')) + } else { + log(prefix, event.name) + } +} + +export function dht (components: DHTRoutingComponents): IPNSRouting { + return new DHTRouting(components) +} diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts new file mode 100644 index 0000000..556e4ae --- /dev/null +++ b/packages/ipns/src/routing/index.ts @@ -0,0 +1,9 @@ +import type { AbortOptions } from '@libp2p/interfaces' + +export interface IPNSRouting { + put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: AbortOptions) => Promise + get: (routingKey: Uint8Array, options?: AbortOptions) => Promise +} + +export { dht } from './dht.js' +export { pubsub } from './pubsub.js' diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts new file mode 100644 index 0000000..dff9225 --- /dev/null +++ b/packages/ipns/src/routing/pubsub.ts @@ -0,0 +1,178 @@ +import { peerIdToRoutingKey } from 'ipns' +import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import { logger } from '@libp2p/logger' +import type { PeerId } from '@libp2p/interface-peer-id' +import type { Message, PubSub } from '@libp2p/interface-pubsub' +import type { Datastore } from 'interface-datastore' +import type { AbortOptions } from '@libp2p/interfaces' +import type { IPNSRouting } from './index.js' +import { CodeError } from '@libp2p/interfaces/errors' +import { localStore, LocalStore } from '../utils/local-store.js' +import { ipnsValidator } from 'ipns/validator' +import { ipnsSelector } from 'ipns/selector' +import { equals as uint8ArrayEquals } from 'uint8arrays/equals' + +const log = logger('helia:ipns:routing:pubsub') + +export interface PubsubRoutingComponents { + datastore: Datastore + libp2p: { + peerId: PeerId + pubsub: PubSub + } +} + +/** + * This IPNS routing receives IPNS record updates via dedicated + * pubsub topic. + * + * Note we must first be subscribed to the topic in order to receive + * updated records, so the first call to `.get` should be expected + * to fail! + */ +class PubSubRouting implements IPNSRouting { + private subscriptions: string[] + private readonly localStore: LocalStore + private readonly peerId: PeerId + private readonly pubsub: PubSub + + constructor (components: PubsubRoutingComponents) { + this.subscriptions = [] + this.localStore = localStore(components.datastore) + this.peerId = components.libp2p.peerId + this.pubsub = components.libp2p.pubsub + + this.pubsub.addEventListener('message', (evt) => { + const message = evt.detail + + if (!this.subscriptions.includes(message.topic)) { + return + } + + this.#processPubSubMessage(message).catch(err => { + log.error('Error processing message', err) + }) + }) + } + + async #processPubSubMessage (message: Message): Promise { + log('message received for topic', message.topic) + + if (message.type !== 'signed') { + log.error('unsigned message received, this module can only work with signed messages') + return + } + + if (message.from.equals(this.peerId)) { + log('not storing record from self') + return + } + + const routingKey = topicToKey(message.topic) + + await ipnsValidator(routingKey, message.data) + + if (await this.localStore.has(routingKey)) { + const currentRecord = await this.localStore.get(routingKey) + + if (uint8ArrayEquals(currentRecord, message.data)) { + log('not storing record as we already have it') + return + } + + const records = [currentRecord, message.data] + const index = ipnsSelector(routingKey, records) + + if (index === 0) { + log('not storing record as the one we have is better') + return + } + } + + await this.localStore.put(routingKey, message.data) + } + + /** + * Put a value to the pubsub datastore indexed by the received key properly encoded + */ + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: AbortOptions): Promise { + const topic = keyToTopic(routingKey) + + log('publish value for topic %s', topic) + + const result = await this.pubsub.publish(topic, marshaledRecord) + + log('published record on topic %s to %d recipients', topic, result.recipients) + } + + /** + * Get a value from the pubsub datastore indexed by the received key properly encoded. + * Also, the identifier topic is subscribed to and the pubsub datastore records will be + * updated once new publishes occur + */ + async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const topic = keyToTopic(routingKey) + + // ensure we are subscribed to topic + if (!this.pubsub.getTopics().includes(topic)) { + log('add subscription for topic', topic) + this.pubsub.subscribe(topic) + this.subscriptions.push(topic) + } + + // chain through to local store + return await this.localStore.get(routingKey, options) + } + + /** + * Get pubsub subscriptions related to ipns + */ + getSubscriptions (): string[] { + return this.subscriptions + } + + /** + * Cancel pubsub subscriptions related to ipns + */ + cancel (key: PeerId): void { + const routingKey = peerIdToRoutingKey(key) + const topic = keyToTopic(routingKey) + + // Not found topic + if (!this.subscriptions.includes(topic)) { + return + } + + this.pubsub.unsubscribe(topic) + this.subscriptions = this.subscriptions.filter(t => t !== topic) + } +} + +const PUBSUB_NAMESPACE = '/record/' + +/** + * converts a binary record key to a pubsub topic key + */ +function keyToTopic (key: Uint8Array): string { + const b64url = uint8ArrayToString(key, 'base64url') + + return `${PUBSUB_NAMESPACE}${b64url}` +} + +/** + * converts a pubsub topic key to a binary record key + */ +function topicToKey (topic: string): Uint8Array { + if (topic.substring(0, PUBSUB_NAMESPACE.length) !== PUBSUB_NAMESPACE) { + throw new CodeError('topic received is not from a record', 'ERR_TOPIC_IS_NOT_FROM_RECORD_NAMESPACE') + } + + const key = topic.substring(PUBSUB_NAMESPACE.length) + + return uint8ArrayFromString(key, 'base64url') +} + +export function pubsub (components: PubsubRoutingComponents): IPNSRouting { + return new PubSubRouting(components) +} diff --git a/packages/ipns/src/utils/local-store.ts b/packages/ipns/src/utils/local-store.ts new file mode 100644 index 0000000..3f4d922 --- /dev/null +++ b/packages/ipns/src/utils/local-store.ts @@ -0,0 +1,44 @@ +import type { AbortOptions } from '@libp2p/interfaces' +import { Libp2pRecord } from '@libp2p/record' +import { Datastore, Key } from 'interface-datastore' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { IPNSRouting } from '../routing' + +function dhtRoutingKey (key: Uint8Array): Key { + return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) +} + +export interface LocalStore extends IPNSRouting { + has: (routingKey: Uint8Array, options?: AbortOptions) => Promise +} + +/** + * Returns an IPNSRouting implementation that reads/writes IPNS records to the + * datastore as DHT records. This lets us publish IPNS records offline then + * serve them to the network later in response to DHT queries. + */ +export function localStore (datastore: Datastore): LocalStore { + return { + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: AbortOptions = {}) { + const key = dhtRoutingKey(routingKey) + + // Marshal to libp2p record as the DHT does + const record = new Libp2pRecord(routingKey, marshaledRecord, new Date()) + + await datastore.put(key, record.serialize(), options) + }, + async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const key = dhtRoutingKey(routingKey) + const buf = await datastore.get(key, options) + + // Unmarshal libp2p record as the DHT does + const record = Libp2pRecord.deserialize(buf) + + return record.value + }, + async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const key = dhtRoutingKey(routingKey) + return await datastore.has(key, options) + } + } +} diff --git a/packages/ipns/src/utils/resolve-dns-link.browser.ts b/packages/ipns/src/utils/resolve-dns-link.browser.ts new file mode 100644 index 0000000..459f387 --- /dev/null +++ b/packages/ipns/src/utils/resolve-dns-link.browser.ts @@ -0,0 +1,61 @@ +/* eslint-env browser */ + +import { TLRU } from './tlru.js' +import PQueue from 'p-queue' +import type { AbortOptions } from '@libp2p/interfaces' + +// Avoid sending multiple queries for the same hostname by caching results +const cache = new TLRU<{ Path: string, Message: string }>(1000) +// TODO: /api/v0/dns does not return TTL yet: https://github.com/ipfs/go-ipfs/issues/5884 +// However we know browsers themselves cache DNS records for at least 1 minute, +// which acts a provisional default ttl: https://stackoverflow.com/a/36917902/11518426 +const ttl = 60 * 1000 + +// browsers limit concurrent connections per host, +// we don't want preload calls to exhaust the limit (~6) +const httpQueue = new PQueue({ concurrency: 4 }) + +const ipfsPath = (response: { Path: string, Message: string }): string => { + if (response.Path != null) { + return response.Path + } + throw new Error(response.Message) +} + +export interface ResolveDnsLinkOptions extends AbortOptions { + nocache?: boolean +} + +export async function resolveDnslink (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise { // eslint-disable-line require-await + const resolve = async (fqdn: string, opts: ResolveDnsLinkOptions = {}): Promise => { + // @ts-expect-error - URLSearchParams does not take boolean options, only strings + const searchParams = new URLSearchParams(opts) + searchParams.set('arg', fqdn) + + // try cache first + const query = searchParams.toString() + if (opts.nocache !== true && cache.has(query)) { + const response = cache.get(query) + + if (response != null) { + return ipfsPath(response) + } + } + + // fallback to delegated DNS resolver + const response = await httpQueue.add(async () => { + // Delegated HTTP resolver sending DNSLink queries to ipfs.io + // TODO: replace hardcoded host with configurable DNS over HTTPS: https://github.com/ipfs/js-ipfs/issues/2212 + const res = await fetch(`https://ipfs.io/api/v0/dns?${searchParams}`) + const query = new URL(res.url).search.slice(1) + const json = await res.json() + cache.set(query, json, ttl) + + return json + }) + + return ipfsPath(response) + } + + return await resolve(fqdn, opts) +} diff --git a/packages/ipns/src/utils/resolve-dns-link.ts b/packages/ipns/src/utils/resolve-dns-link.ts new file mode 100644 index 0000000..34ec034 --- /dev/null +++ b/packages/ipns/src/utils/resolve-dns-link.ts @@ -0,0 +1,65 @@ +import dns from 'dns' +import { promisify } from 'util' +import type { AbortOptions } from '@libp2p/interfaces' +import * as isIPFS from 'is-ipfs' + +const MAX_RECURSIVE_DEPTH = 32 + +export async function resolveDnslink (domain: string, options: AbortOptions = {}): Promise { + return await recursiveResolveDnslink(domain, MAX_RECURSIVE_DEPTH, options) +} + +async function recursiveResolveDnslink (domain: string, depth: number, options: AbortOptions = {}): Promise { + if (depth === 0) { + throw new Error('recursion limit exceeded') + } + + let dnslinkRecord + + try { + dnslinkRecord = await resolve(domain) + } catch (err: any) { + // If the code is not ENOTFOUND or ERR_DNSLINK_NOT_FOUND or ENODATA then throw the error + if (err.code !== 'ENOTFOUND' && err.code !== 'ERR_DNSLINK_NOT_FOUND' && err.code !== 'ENODATA') { + throw err + } + + if (domain.startsWith('_dnslink.')) { + // The supplied domain contains a _dnslink component + // Check the non-_dnslink domain + dnslinkRecord = await resolve(domain.replace('_dnslink.', '')) + } else { + // Check the _dnslink subdomain + const _dnslinkDomain = `_dnslink.${domain}` + // If this throws then we propagate the error + dnslinkRecord = await resolve(_dnslinkDomain) + } + } + + const result = dnslinkRecord.replace('dnslink=', '') + const domainOrCID = result.split('/')[2] + const isIPFSCID = isIPFS.cid(domainOrCID) + + if (isIPFSCID || depth === 0) { + return result + } + + return await recursiveResolveDnslink(domainOrCID, depth - 1, options) +} + +async function resolve (domain: string, options: AbortOptions = {}): Promise { + const DNSLINK_REGEX = /^dnslink=.+$/ + const records = await promisify(dns.resolveTxt)(domain) + const dnslinkRecords = records.reduce((rs, r) => rs.concat(r), []) + .filter(record => DNSLINK_REGEX.test(record)) + + const dnslinkRecord = dnslinkRecords[0] + + // we now have dns text entries as an array of strings + // only records passing the DNSLINK_REGEX text are included + if (dnslinkRecord == null) { + throw new Error(`No dnslink records found for domain: ${domain}`) + } + + return dnslinkRecord +} diff --git a/packages/ipns/src/utils/tlru.ts b/packages/ipns/src/utils/tlru.ts new file mode 100644 index 0000000..0556c0e --- /dev/null +++ b/packages/ipns/src/utils/tlru.ts @@ -0,0 +1,52 @@ +import hashlru from 'hashlru' + +/** + * Time Aware Least Recent Used Cache + * + * @see https://arxiv.org/pdf/1801.00390 + */ +export class TLRU { + private readonly lru: ReturnType + + constructor (maxSize: number) { + this.lru = hashlru(maxSize) + } + + get (key: string): T | undefined { + const value = this.lru.get(key) + + if (value != null) { + if (value.expire != null && value.expire < Date.now()) { + this.lru.remove(key) + + return undefined + } + + return value.value + } + + return undefined + } + + set (key: string, value: T, ttl: number): void { + this.lru.set(key, { value, expire: Date.now() + ttl }) + } + + has (key: string): boolean { + const value = this.get(key) + + if (value != null) { + return true + } + + return false + } + + remove (key: string): void { + this.lru.remove(key) + } + + clear (): void { + this.lru.clear() + } +} diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts new file mode 100644 index 0000000..6345a1d --- /dev/null +++ b/packages/ipns/test/publish.spec.ts @@ -0,0 +1,39 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import type { IPNS } from '../src/index.js' +import { ipns } from '../src/index.js' +import { CID } from 'multiformats/cid' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' + +const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + +describe('publish', () => { + let name: IPNS + + before(async () => { + const datastore = new MemoryDatastore() + + name = ipns({ datastore }) + }) + + it('should publish an IPNS record with the default params', async function () { + const key = await createEd25519PeerId() + const ipnsEntry = await name.publish(key, cid) + + expect(ipnsEntry).to.have.property('sequence', 1n) + expect(ipnsEntry).to.have.property('ttl', 8640000000000n) // 24 hours + }) + + it('should publish an IPNS record with a custom ttl params', async function () { + const key = await createEd25519PeerId() + const lifetime = 123000 + const ipnsEntry = await name.publish(key, cid, { + lifetime + }) + + expect(ipnsEntry).to.have.property('sequence', 1n) + expect(ipnsEntry).to.have.property('ttl', BigInt(lifetime) * 100000n) + }) +}) diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts new file mode 100644 index 0000000..a62c9a4 --- /dev/null +++ b/packages/ipns/test/resolve.spec.ts @@ -0,0 +1,65 @@ +/* eslint-env mocha */ + +import { expect } from 'aegir/chai' +import { MemoryDatastore } from 'datastore-core' +import type { IPNS } from '../src/index.js' +import { ipns } from '../src/index.js' +import { CID } from 'multiformats/cid' +import { createEd25519PeerId } from '@libp2p/peer-id-factory' + +const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') + +describe('resolve', () => { + let name: IPNS + + before(async () => { + const datastore = new MemoryDatastore() + + name = ipns({ datastore }) + }) + + it('should resolve a record', async () => { + const key = await createEd25519PeerId() + await name.publish(key, cid) + + const resolvedValue = await name.resolve(key) + + if (resolvedValue == null) { + throw new Error('Did not resolve entry') + } + + expect(resolvedValue.toString()).to.equal(cid.toString()) + }) + + it('should resolve a recursive record', async () => { + const key1 = await createEd25519PeerId() + const key2 = await createEd25519PeerId() + await name.publish(key2, cid) + await name.publish(key1, key2) + + const resolvedValue = await name.resolve(key1) + + if (resolvedValue == null) { + throw new Error('Did not resolve entry') + } + + expect(resolvedValue.toString()).to.equal(cid.toString()) + }) + + it('should resolve /ipns/tableflip.io', async function () { + const domain = 'tableflip.io' + + try { + const resolvedValue = await name.resolveDns(domain) + + expect(resolvedValue).to.be.an.instanceOf(CID) + } catch (err: any) { + // happens when running tests offline + if (err.message.includes(`ECONNREFUSED ${domain}`) === true) { + return this.skip() + } + + throw err + } + }) +}) diff --git a/packages/ipns/tsconfig.json b/packages/ipns/tsconfig.json new file mode 100644 index 0000000..13a3599 --- /dev/null +++ b/packages/ipns/tsconfig.json @@ -0,0 +1,10 @@ +{ + "extends": "aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src", + "test" + ] +} From 97fc97c8dc9781aba2416c0881d439f5b1fcb675 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 17:08:00 +0100 Subject: [PATCH 02/10] chore: update build file --- .github/workflows/js-test-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index 27fd45a..cbc9c02 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -2,7 +2,7 @@ name: test & maybe release on: push: branches: - - ${{{ github.default_branch }}} + - main pull_request: jobs: From 380f9eb9a2aad7b384bd03aa52edb51f76463259 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 17:09:23 +0100 Subject: [PATCH 03/10] chore: update build file --- .github/workflows/js-test-and-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/js-test-and-release.yml b/.github/workflows/js-test-and-release.yml index cbc9c02..7da2cd2 100644 --- a/.github/workflows/js-test-and-release.yml +++ b/.github/workflows/js-test-and-release.yml @@ -124,7 +124,7 @@ jobs: release: needs: [test-node, test-chrome, test-chrome-webworker, test-firefox, test-firefox-webworker, test-electron-main, test-electron-renderer] runs-on: ubuntu-latest - if: github.event_name == 'push' && github.ref == 'refs/heads/${{{ github.default_branch }}}' + if: github.event_name == 'push' && github.ref == 'refs/heads/main' steps: - uses: actions/checkout@v3 with: From e27b5deb0d3ae375910ec4b8c808133e55a1fd5b Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 19:02:09 +0100 Subject: [PATCH 04/10] chore: ensure nodes can find each other --- packages/interop/test/dht.spec.ts | 35 ++++++++++++++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index 7738ed0..e7a4c13 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -20,6 +20,7 @@ import { sortClosestPeers } from './fixtures/create-peer-ids.js' import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' +import { waitFor } from './fixtures/wait-for.js' describe('dht routing', () => { let helia: Helia @@ -54,7 +55,7 @@ describe('dht routing', () => { }) kubo = await createKuboNode() - // find a PeerId that when used as an IPNS key is KAD-close to the resolver + // find a PeerId that is KAD-closer to the resolver than the publisher when used as an IPNS key while (true) { key = await createEd25519PeerId() const routingKey = uint8ArrayConcat([ @@ -88,6 +89,38 @@ describe('dht routing', () => { } expect(connected).to.be.true() + await waitFor(async () => { + let found = false + + for await (const event of helia.libp2p.dht.findPeer(kubo.peer.id)) { + if (event.name === 'FINAL_PEER') { + found = true + } + } + + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Helia could not find Kubo on the DHT' + }) + + await waitFor(async () => { + let found = false + + for await (const event of kubo.api.dht.findPeer(helia.libp2p.peerId)) { + if (event.name === 'FINAL_PEER') { + found = true + } + } + + return found + }, { + timeout: 30000, + delay: 1000, + message: 'Kubo could not find Helia on the DHT' + }) + name = ipns(helia, [ dht(helia) ]) From 155f202f414b11d7b2093b0b69b37a89e95caf9f Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 19:39:21 +0100 Subject: [PATCH 05/10] chore: refactor tests --- packages/interop/test/dht.spec.ts | 11 ++--------- packages/interop/test/fixtures/connect.ts | 18 ++++++++++++++++++ packages/interop/test/pubsub.spec.ts | 15 +++------------ packages/ipns/src/index.ts | 18 +++++++++++++++++- 4 files changed, 40 insertions(+), 22 deletions(-) create mode 100644 packages/interop/test/fixtures/connect.ts diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index e7a4c13..c4fd590 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -21,6 +21,7 @@ import type { PeerId } from 'kubo-rpc-client/dist/src/types.js' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { waitFor } from './fixtures/wait-for.js' +import { connect } from './fixtures/connect.js' describe('dht routing', () => { let helia: Helia @@ -79,15 +80,7 @@ describe('dht routing', () => { // connect the two nodes over the KAD-DHT protocol, this should ensure // both nodes have each other in their KAD buckets - let connected = false - for (const addr of kubo.peer.addresses) { - try { - await helia.libp2p.dialProtocol(addr, '/ipfs/lan/kad/1.0.0') - connected = true - break - } catch { } - } - expect(connected).to.be.true() + await connect(helia, kubo, '/ipfs/lan/kad/1.0.0') await waitFor(async () => { let found = false diff --git a/packages/interop/test/fixtures/connect.ts b/packages/interop/test/fixtures/connect.ts new file mode 100644 index 0000000..e7685a3 --- /dev/null +++ b/packages/interop/test/fixtures/connect.ts @@ -0,0 +1,18 @@ +import { expect } from 'aegir/chai' +import type { Helia } from '@helia/interface' +import type { Controller } from 'ipfsd-ctl' + +/** + * Connect the two nodes by dialing a protocol stream + */ +export async function connect (helia: Helia, kubo: Controller, protocol: string): Promise { + let connected = false + for (const addr of kubo.peer.addresses) { + try { + await helia.libp2p.dialProtocol(addr, protocol) + connected = true + break + } catch { } + } + expect(connected).to.be.true('could not connect hHlia to Kubo') +} diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index a0ed278..9cf4e09 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -20,6 +20,7 @@ import { waitFor } from './fixtures/wait-for.js' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { concat as uint8ArrayConcat } from 'uint8arrays/concat' +import { connect } from './fixtures/connect.js' const LIBP2P_KEY_CODEC = 0x72 @@ -36,18 +37,8 @@ describe('pubsub routing', () => { args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] }) - // connect the two nodes - serial dial seems more stable? - // connect the two nodes over the KAD-DHT protocol, this should ensure - // both nodes have each other in their KAD buckets - let connected = false - for (const addr of kubo.peer.addresses) { - try { - await helia.libp2p.dialProtocol(addr, '/meshsub/1.1.0') - connected = true - break - } catch { } - } - expect(connected).to.be.true() + // connect the two nodes + await connect(helia, kubo, '/meshsub/1.1.0') name = ipns(helia, [ pubsub(helia) diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 16365cc..ea75858 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -72,6 +72,8 @@ import { CID } from 'multiformats/cid' import { resolveDnslink } from './utils/resolve-dns-link.js' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' +import type { ProgressEvent, ProgressOptions } from '@helia/interface' +import { CustomProgressEvent } from '@helia/interface' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Datastore } from 'interface-datastore' @@ -100,7 +102,16 @@ export interface ResolveOptions extends AbortOptions { nocache?: boolean } -export interface RepublishOptions extends AbortOptions { +export type RepublishProgressEvents = + ProgressEvent<'republish:start', unknown> | + ProgressEvent<'republish:success', IPNSEntry> | + ProgressEvent<'republish:error', { record: IPNSEntry, err: Error }> + +export type DHTProgressEvents = + ProgressEvent<'dht:query', Uint8Array> | + ProgressEvent<'dht:error', { record: Uint8Array, err: Error }> + +export interface RepublishOptions extends AbortOptions, ProgressOptions { /** * The republish interval in ms (default: 24hrs) */ @@ -200,12 +211,17 @@ class DefaultIPNS implements IPNS { throw new Error('Republish is already running') } + const progress = options.progress ?? (() => {}) + options.signal?.addEventListener('abort', () => { clearTimeout(this.timeout) }) async function republish (): Promise { const startTime = Date.now() + + progress(new CustomProgressEvent('republish:start')) + const finishType = Date.now() const timeTaken = finishType - startTime let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken From 1daaf119bf28a89f07036d86bc547226f04928a0 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 19:44:54 +0100 Subject: [PATCH 06/10] chore: turn of kubo dht for pubsub tests --- packages/interop/test/pubsub.spec.ts | 7 +++++++ packages/ipns/src/index.ts | 18 +----------------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/packages/interop/test/pubsub.spec.ts b/packages/interop/test/pubsub.spec.ts index 9cf4e09..0245383 100644 --- a/packages/interop/test/pubsub.spec.ts +++ b/packages/interop/test/pubsub.spec.ts @@ -34,6 +34,13 @@ describe('pubsub routing', () => { pubsub: gossipsub() }) kubo = await createKuboNode({ + ipfsOptions: { + config: { + Routing: { + Type: 'none' + } + } + }, args: ['--enable-pubsub-experiment', '--enable-namesys-pubsub'] }) diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index ea75858..16365cc 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -72,8 +72,6 @@ import { CID } from 'multiformats/cid' import { resolveDnslink } from './utils/resolve-dns-link.js' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' -import type { ProgressEvent, ProgressOptions } from '@helia/interface' -import { CustomProgressEvent } from '@helia/interface' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Datastore } from 'interface-datastore' @@ -102,16 +100,7 @@ export interface ResolveOptions extends AbortOptions { nocache?: boolean } -export type RepublishProgressEvents = - ProgressEvent<'republish:start', unknown> | - ProgressEvent<'republish:success', IPNSEntry> | - ProgressEvent<'republish:error', { record: IPNSEntry, err: Error }> - -export type DHTProgressEvents = - ProgressEvent<'dht:query', Uint8Array> | - ProgressEvent<'dht:error', { record: Uint8Array, err: Error }> - -export interface RepublishOptions extends AbortOptions, ProgressOptions { +export interface RepublishOptions extends AbortOptions { /** * The republish interval in ms (default: 24hrs) */ @@ -211,17 +200,12 @@ class DefaultIPNS implements IPNS { throw new Error('Republish is already running') } - const progress = options.progress ?? (() => {}) - options.signal?.addEventListener('abort', () => { clearTimeout(this.timeout) }) async function republish (): Promise { const startTime = Date.now() - - progress(new CustomProgressEvent('republish:start')) - const finishType = Date.now() const timeTaken = finishType - startTime let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken From dd221dae787e9b8266062c7a50127f8e3af533a7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 19:45:30 +0100 Subject: [PATCH 07/10] chore: simplify config --- packages/interop/test/fixtures/create-helia.browser.ts | 5 ----- 1 file changed, 5 deletions(-) diff --git a/packages/interop/test/fixtures/create-helia.browser.ts b/packages/interop/test/fixtures/create-helia.browser.ts index 6311c2a..a3684df 100644 --- a/packages/interop/test/fixtures/create-helia.browser.ts +++ b/packages/interop/test/fixtures/create-helia.browser.ts @@ -39,11 +39,6 @@ export async function createHeliaNode (): Promise { }), pubsub: gossipsub(), datastore, - identify: { - host: { - agentVersion: 'helia/0.0.0' - } - }, nat: { enabled: false }, From 23cf544714c9c22c2e983f845682627883b1c17e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Sun, 5 Feb 2023 20:41:40 +0100 Subject: [PATCH 08/10] chore: skip test on electron main --- packages/interop/test/dht.spec.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/interop/test/dht.spec.ts b/packages/interop/test/dht.spec.ts index c4fd590..053424d 100644 --- a/packages/interop/test/dht.spec.ts +++ b/packages/interop/test/dht.spec.ts @@ -22,6 +22,7 @@ import { concat as uint8ArrayConcat } from 'uint8arrays/concat' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { waitFor } from './fixtures/wait-for.js' import { connect } from './fixtures/connect.js' +import { isElectronMain } from 'wherearewe' describe('dht routing', () => { let helia: Helia @@ -146,7 +147,13 @@ describe('dht routing', () => { expect(resolved).to.equal(`/ipfs/${value.toString()}`) }) - it('should publish on kubo and resolve on helia', async () => { + it('should publish on kubo and resolve on helia', async function () { + if (isElectronMain) { + // electron main does not have fetch, FormData or Blob APIs + // can revisit when kubo-rpc-client supports the key.import API + return this.skip() + } + await createNodes('helia') const keyName = 'my-ipns-key' From 21be465c5d54ceb50d2f3b5b433343160e451521 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 7 Feb 2023 09:23:41 +0100 Subject: [PATCH 09/10] feat: add progress events --- packages/ipns/package.json | 4 +- packages/ipns/src/index.ts | 82 +++++++++++++++--------- packages/ipns/src/routing/dht.ts | 39 +++++++---- packages/ipns/src/routing/index.ts | 21 +++++- packages/ipns/src/routing/local-store.ts | 63 ++++++++++++++++++ packages/ipns/src/routing/pubsub.ts | 61 +++++++++++------- packages/ipns/src/utils/local-store.ts | 44 ------------- packages/ipns/test/publish.spec.ts | 11 ++++ packages/ipns/test/resolve.spec.ts | 13 ++++ 9 files changed, 229 insertions(+), 109 deletions(-) create mode 100644 packages/ipns/src/routing/local-store.ts delete mode 100644 packages/ipns/src/utils/local-store.ts diff --git a/packages/ipns/package.json b/packages/ipns/package.json index 0105a27..5b08770 100644 --- a/packages/ipns/package.json +++ b/packages/ipns/package.json @@ -173,12 +173,14 @@ "is-ipfs": "^8.0.1", "multiformats": "^11.0.1", "p-queue": "^7.3.0", + "progress-events": "^1.0.0", "uint8arrays": "^4.0.3" }, "devDependencies": { "@libp2p/peer-id-factory": "^2.0.1", "aegir": "^38.1.0", - "datastore-core": "^8.0.4" + "datastore-core": "^8.0.4", + "sinon": "^15.0.1" }, "browser": { "./dist/src/utils/resolve-dns-link.js": "./dist/src/utils/resolve-dns-link.browser.js" diff --git a/packages/ipns/src/index.ts b/packages/ipns/src/index.ts index 16365cc..2e13c94 100644 --- a/packages/ipns/src/index.ts +++ b/packages/ipns/src/index.ts @@ -66,17 +66,18 @@ import type { AbortOptions } from '@libp2p/interfaces' import { isPeerId, PeerId } from '@libp2p/interface-peer-id' import { create, marshal, peerIdToRoutingKey, unmarshal } from 'ipns' import type { IPNSEntry } from 'ipns' -import type { IPNSRouting } from './routing/index.js' +import type { IPNSRouting, IPNSRoutingEvents } from './routing/index.js' import { ipnsValidator } from 'ipns/validator' import { CID } from 'multiformats/cid' import { resolveDnslink } from './utils/resolve-dns-link.js' import { logger } from '@libp2p/logger' import { peerIdFromString } from '@libp2p/peer-id' +import type { ProgressEvent, ProgressOptions } from 'progress-events' +import { CustomProgressEvent } from 'progress-events' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import type { Datastore } from 'interface-datastore' -import { localStore } from './utils/local-store.js' -import type { LocalStore } from './utils/local-store.js' +import { localStore, LocalStore } from './routing/local-store.js' const log = logger('helia:ipns') @@ -86,21 +87,36 @@ const HOUR = 60 * MINUTE const DEFAULT_LIFETIME_MS = 24 * HOUR const DEFAULT_REPUBLISH_INTERVAL_MS = 23 * HOUR -export interface PublishOptions extends AbortOptions { +export type PublishProgressEvents = + ProgressEvent<'ipns:publish:start'> | + ProgressEvent<'ipns:publish:success', IPNSEntry> | + ProgressEvent<'ipns:publish:error', Error> + +export type ResolveProgressEvents = + ProgressEvent<'ipns:resolve:start', unknown> | + ProgressEvent<'ipns:resolve:success', IPNSEntry> | + ProgressEvent<'ipns:resolve:error', Error> + +export type RepublishProgressEvents = + ProgressEvent<'ipns:republish:start', unknown> | + ProgressEvent<'ipns:republish:success', IPNSEntry> | + ProgressEvent<'ipns:republish:error', { record: IPNSEntry, err: Error }> + +export interface PublishOptions extends AbortOptions, ProgressOptions { /** * Time duration of the record in ms */ lifetime?: number } -export interface ResolveOptions extends AbortOptions { +export interface ResolveOptions extends AbortOptions, ProgressOptions { /** * do not use cached entries */ nocache?: boolean } -export interface RepublishOptions extends AbortOptions { +export interface RepublishOptions extends AbortOptions, ProgressOptions { /** * The republish interval in ms (default: 24hrs) */ @@ -149,36 +165,41 @@ class DefaultIPNS implements IPNS { } async publish (key: PeerId, value: CID | PeerId, options: PublishOptions = {}): Promise { - let sequenceNumber = 1n - const routingKey = peerIdToRoutingKey(key) - - if (await this.localStore.has(routingKey, options)) { - // if we have published under this key before, increment the sequence number - const buf = await this.localStore.get(routingKey, options) - const existingRecord = unmarshal(buf) - sequenceNumber = existingRecord.sequence + 1n - } + try { + let sequenceNumber = 1n + const routingKey = peerIdToRoutingKey(key) + + if (await this.localStore.has(routingKey, options)) { + // if we have published under this key before, increment the sequence number + const buf = await this.localStore.get(routingKey, options) + const existingRecord = unmarshal(buf) + sequenceNumber = existingRecord.sequence + 1n + } - let str + let str - if (isPeerId(value)) { - str = `/ipns/${value.toString()}` - } else { - str = `/ipfs/${value.toString()}` - } + if (isPeerId(value)) { + str = `/ipns/${value.toString()}` + } else { + str = `/ipfs/${value.toString()}` + } - const bytes = uint8ArrayFromString(str) + const bytes = uint8ArrayFromString(str) - // create record - const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS) - const marshaledRecord = marshal(record) + // create record + const record = await create(key, bytes, sequenceNumber, options.lifetime ?? DEFAULT_LIFETIME_MS) + const marshaledRecord = marshal(record) - await this.localStore.put(routingKey, marshaledRecord, options) + await this.localStore.put(routingKey, marshaledRecord, options) - // publish record to routing - await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) + // publish record to routing + await Promise.all(this.routers.map(async r => { await r.put(routingKey, marshaledRecord, options) })) - return record + return record + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:publish:error', err)) + throw err + } } async resolve (key: PeerId, options: ResolveOptions = {}): Promise { @@ -206,6 +227,9 @@ class DefaultIPNS implements IPNS { async function republish (): Promise { const startTime = Date.now() + + options.onProgress?.(new CustomProgressEvent('ipns:republish:start')) + const finishType = Date.now() const timeTaken = finishType - startTime let nextInterval = DEFAULT_REPUBLISH_INTERVAL_MS - timeTaken diff --git a/packages/ipns/src/routing/dht.ts b/packages/ipns/src/routing/dht.ts index 87efdc2..a160cec 100644 --- a/packages/ipns/src/routing/dht.ts +++ b/packages/ipns/src/routing/dht.ts @@ -1,7 +1,8 @@ import { logger } from '@libp2p/logger' import type { IPNSRouting } from '../index.js' import type { DHT, QueryEvent } from '@libp2p/interface-dht' -import type { AbortOptions } from '@libp2p/interfaces' +import type { GetOptions, PutOptions } from './index.js' +import { CustomProgressEvent, ProgressEvent } from 'progress-events' const log = logger('helia:ipns:routing:dht') @@ -11,6 +12,10 @@ export interface DHTRoutingComponents { } } +export type DHTProgressEvents = + ProgressEvent<'ipns:routing:dht:query', QueryEvent> | + ProgressEvent<'ipns:routing:dht:error', Error> + export class DHTRouting implements IPNSRouting { private readonly dht: DHT @@ -18,15 +23,21 @@ export class DHTRouting implements IPNSRouting { this.dht = components.libp2p.dht } - async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: AbortOptions = {}): Promise { + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise { let putValue = false - for await (const event of this.dht.put(routingKey, marshaledRecord, options)) { - logEvent('DHT put event', event) + try { + for await (const event of this.dht.put(routingKey, marshaledRecord, options)) { + logEvent('DHT put event', event) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) - if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') { - putValue = true + if (event.name === 'PEER_RESPONSE' && event.messageName === 'PUT_VALUE') { + putValue = true + } } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) } if (!putValue) { @@ -34,13 +45,19 @@ export class DHTRouting implements IPNSRouting { } } - async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { - for await (const event of this.dht.get(routingKey, options)) { - logEvent('DHT get event', event) + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + for await (const event of this.dht.get(routingKey, options)) { + logEvent('DHT get event', event) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:query', event)) - if (event.name === 'VALUE') { - return event.value + if (event.name === 'VALUE') { + return event.value + } } + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:dht:error', err)) } throw new Error('Not found') diff --git a/packages/ipns/src/routing/index.ts b/packages/ipns/src/routing/index.ts index 556e4ae..3837487 100644 --- a/packages/ipns/src/routing/index.ts +++ b/packages/ipns/src/routing/index.ts @@ -1,9 +1,26 @@ +import type { ProgressOptions } from 'progress-events' import type { AbortOptions } from '@libp2p/interfaces' +import type { DHTProgressEvents } from './dht.js' +import type { DatastoreProgressEvents } from './local-store.js' +import type { PubSubProgressEvents } from './pubsub.js' + +export interface PutOptions extends AbortOptions, ProgressOptions { + +} + +export interface GetOptions extends AbortOptions, ProgressOptions { + +} export interface IPNSRouting { - put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: AbortOptions) => Promise - get: (routingKey: Uint8Array, options?: AbortOptions) => Promise + put: (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: PutOptions) => Promise + get: (routingKey: Uint8Array, options?: GetOptions) => Promise } +export type IPNSRoutingEvents = + DatastoreProgressEvents | + DHTProgressEvents | + PubSubProgressEvents + export { dht } from './dht.js' export { pubsub } from './pubsub.js' diff --git a/packages/ipns/src/routing/local-store.ts b/packages/ipns/src/routing/local-store.ts new file mode 100644 index 0000000..4ea0a37 --- /dev/null +++ b/packages/ipns/src/routing/local-store.ts @@ -0,0 +1,63 @@ +import { CustomProgressEvent, ProgressEvent } from 'progress-events' +import type { AbortOptions } from '@libp2p/interfaces' +import { Libp2pRecord } from '@libp2p/record' +import { Datastore, Key } from 'interface-datastore' +import { toString as uint8ArrayToString } from 'uint8arrays/to-string' +import type { GetOptions, IPNSRouting, PutOptions } from '../routing' + +function dhtRoutingKey (key: Uint8Array): Key { + return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) +} + +export type DatastoreProgressEvents = + ProgressEvent<'ipns:routing:datastore:put'> | + ProgressEvent<'ipns:routing:datastore:get'> | + ProgressEvent<'ipns:routing:datastore:error', Error> + +export interface LocalStore extends IPNSRouting { + has: (routingKey: Uint8Array, options?: AbortOptions) => Promise +} + +/** + * Returns an IPNSRouting implementation that reads/writes IPNS records to the + * datastore as DHT records. This lets us publish IPNS records offline then + * serve them to the network later in response to DHT queries. + */ +export function localStore (datastore: Datastore): LocalStore { + return { + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}) { + try { + const key = dhtRoutingKey(routingKey) + + // Marshal to libp2p record as the DHT does + const record = new Libp2pRecord(routingKey, marshaledRecord, new Date()) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:put')) + await datastore.put(key, record.serialize(), options) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + const key = dhtRoutingKey(routingKey) + + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:get')) + const buf = await datastore.get(key, options) + + // Unmarshal libp2p record as the DHT does + const record = Libp2pRecord.deserialize(buf) + + return record.value + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:routing:datastore:error', err)) + throw err + } + }, + async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { + const key = dhtRoutingKey(routingKey) + return await datastore.has(key, options) + } + } +} diff --git a/packages/ipns/src/routing/pubsub.ts b/packages/ipns/src/routing/pubsub.ts index dff9225..bb10e30 100644 --- a/packages/ipns/src/routing/pubsub.ts +++ b/packages/ipns/src/routing/pubsub.ts @@ -3,15 +3,15 @@ import { fromString as uint8ArrayFromString } from 'uint8arrays/from-string' import { toString as uint8ArrayToString } from 'uint8arrays/to-string' import { logger } from '@libp2p/logger' import type { PeerId } from '@libp2p/interface-peer-id' -import type { Message, PubSub } from '@libp2p/interface-pubsub' +import type { Message, PublishResult, PubSub } from '@libp2p/interface-pubsub' import type { Datastore } from 'interface-datastore' -import type { AbortOptions } from '@libp2p/interfaces' -import type { IPNSRouting } from './index.js' +import type { GetOptions, IPNSRouting, PutOptions } from './index.js' import { CodeError } from '@libp2p/interfaces/errors' -import { localStore, LocalStore } from '../utils/local-store.js' +import { localStore, LocalStore } from './local-store.js' import { ipnsValidator } from 'ipns/validator' import { ipnsSelector } from 'ipns/selector' import { equals as uint8ArrayEquals } from 'uint8arrays/equals' +import { CustomProgressEvent, ProgressEvent } from 'progress-events' const log = logger('helia:ipns:routing:pubsub') @@ -23,6 +23,11 @@ export interface PubsubRoutingComponents { } } +export type PubSubProgressEvents = + ProgressEvent<'ipns:pubsub:publish', { topic: string, result: PublishResult }> | + ProgressEvent<'ipns:pubsub:subscribe', { topic: string }> | + ProgressEvent<'ipns:pubsub:error', Error> + /** * This IPNS routing receives IPNS record updates via dedicated * pubsub topic. @@ -96,14 +101,19 @@ class PubSubRouting implements IPNSRouting { /** * Put a value to the pubsub datastore indexed by the received key properly encoded */ - async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options?: AbortOptions): Promise { - const topic = keyToTopic(routingKey) - - log('publish value for topic %s', topic) - - const result = await this.pubsub.publish(topic, marshaledRecord) - - log('published record on topic %s to %d recipients', topic, result.recipients) + async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: PutOptions = {}): Promise { + try { + const topic = keyToTopic(routingKey) + + log('publish value for topic %s', topic) + const result = await this.pubsub.publish(topic, marshaledRecord) + + log('published record on topic %s to %d recipients', topic, result.recipients) + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:publish', { topic, result })) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:error', err)) + throw err + } } /** @@ -111,18 +121,25 @@ class PubSubRouting implements IPNSRouting { * Also, the identifier topic is subscribed to and the pubsub datastore records will be * updated once new publishes occur */ - async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { - const topic = keyToTopic(routingKey) + async get (routingKey: Uint8Array, options: GetOptions = {}): Promise { + try { + const topic = keyToTopic(routingKey) - // ensure we are subscribed to topic - if (!this.pubsub.getTopics().includes(topic)) { - log('add subscription for topic', topic) - this.pubsub.subscribe(topic) - this.subscriptions.push(topic) - } + // ensure we are subscribed to topic + if (!this.pubsub.getTopics().includes(topic)) { + log('add subscription for topic', topic) + this.pubsub.subscribe(topic) + this.subscriptions.push(topic) - // chain through to local store - return await this.localStore.get(routingKey, options) + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:subscribe', { topic })) + } + + // chain through to local store + return await this.localStore.get(routingKey, options) + } catch (err: any) { + options.onProgress?.(new CustomProgressEvent('ipns:pubsub:error', err)) + throw err + } } /** diff --git a/packages/ipns/src/utils/local-store.ts b/packages/ipns/src/utils/local-store.ts deleted file mode 100644 index 3f4d922..0000000 --- a/packages/ipns/src/utils/local-store.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { AbortOptions } from '@libp2p/interfaces' -import { Libp2pRecord } from '@libp2p/record' -import { Datastore, Key } from 'interface-datastore' -import { toString as uint8ArrayToString } from 'uint8arrays/to-string' -import type { IPNSRouting } from '../routing' - -function dhtRoutingKey (key: Uint8Array): Key { - return new Key('/dht/record/' + uint8ArrayToString(key, 'base32'), false) -} - -export interface LocalStore extends IPNSRouting { - has: (routingKey: Uint8Array, options?: AbortOptions) => Promise -} - -/** - * Returns an IPNSRouting implementation that reads/writes IPNS records to the - * datastore as DHT records. This lets us publish IPNS records offline then - * serve them to the network later in response to DHT queries. - */ -export function localStore (datastore: Datastore): LocalStore { - return { - async put (routingKey: Uint8Array, marshaledRecord: Uint8Array, options: AbortOptions = {}) { - const key = dhtRoutingKey(routingKey) - - // Marshal to libp2p record as the DHT does - const record = new Libp2pRecord(routingKey, marshaledRecord, new Date()) - - await datastore.put(key, record.serialize(), options) - }, - async get (routingKey: Uint8Array, options: AbortOptions = {}): Promise { - const key = dhtRoutingKey(routingKey) - const buf = await datastore.get(key, options) - - // Unmarshal libp2p record as the DHT does - const record = Libp2pRecord.deserialize(buf) - - return record.value - }, - async has (routingKey: Uint8Array, options: AbortOptions = {}): Promise { - const key = dhtRoutingKey(routingKey) - return await datastore.has(key, options) - } - } -} diff --git a/packages/ipns/test/publish.spec.ts b/packages/ipns/test/publish.spec.ts index 6345a1d..2ff3ea4 100644 --- a/packages/ipns/test/publish.spec.ts +++ b/packages/ipns/test/publish.spec.ts @@ -6,6 +6,7 @@ import type { IPNS } from '../src/index.js' import { ipns } from '../src/index.js' import { CID } from 'multiformats/cid' import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import Sinon from 'sinon' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') @@ -36,4 +37,14 @@ describe('publish', () => { expect(ipnsEntry).to.have.property('sequence', 1n) expect(ipnsEntry).to.have.property('ttl', BigInt(lifetime) * 100000n) }) + + it('should emit progress events', async function () { + const key = await createEd25519PeerId() + const onProgress = Sinon.stub() + await name.publish(key, cid, { + onProgress + }) + + expect(onProgress).to.have.property('called', true) + }) }) diff --git a/packages/ipns/test/resolve.spec.ts b/packages/ipns/test/resolve.spec.ts index a62c9a4..e489127 100644 --- a/packages/ipns/test/resolve.spec.ts +++ b/packages/ipns/test/resolve.spec.ts @@ -6,6 +6,7 @@ import type { IPNS } from '../src/index.js' import { ipns } from '../src/index.js' import { CID } from 'multiformats/cid' import { createEd25519PeerId } from '@libp2p/peer-id-factory' +import Sinon from 'sinon' const cid = CID.parse('QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn') @@ -62,4 +63,16 @@ describe('resolve', () => { throw err } }) + + it('should emit progress events', async function () { + const onProgress = Sinon.stub() + const key = await createEd25519PeerId() + await name.publish(key, cid) + + await name.resolve(key, { + onProgress + }) + + expect(onProgress).to.have.property('called', true) + }) }) From 25a0506e5ad708ff7d7a3cb83d7ee1ae3b020bec Mon Sep 17 00:00:00 2001 From: achingbrain Date: Mon, 13 Feb 2023 16:57:20 +0100 Subject: [PATCH 10/10] chore: add logo --- README.md | 6 ++++++ packages/interop/README.md | 6 ++++++ packages/ipns/README.md | 6 ++++++ 3 files changed, 18 insertions(+) diff --git a/README.md b/README.md index 4345fd8..08516c5 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/ipns [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) diff --git a/packages/interop/README.md b/packages/interop/README.md index c99e295..e905e05 100644 --- a/packages/interop/README.md +++ b/packages/interop/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/ipns-interop [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech) diff --git a/packages/ipns/README.md b/packages/ipns/README.md index 8798097..cdca30f 100644 --- a/packages/ipns/README.md +++ b/packages/ipns/README.md @@ -1,3 +1,9 @@ +

+ + Helia logo + +

+ # @helia/ipns [![ipfs.tech](https://img.shields.io/badge/project-IPFS-blue.svg?style=flat-square)](https://ipfs.tech)