From b8bf044e562349b002b8d4b4f6cb644240f0d17e Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 20 May 2020 14:13:08 +0200 Subject: [PATCH 01/38] feat: rendezvous protocol full implementation --- .aegir.js | 57 +++++ .travis.yml | 61 +++-- LIBP2P.md | 47 ++++ README.md | 113 ++++++++- appveyor.yml | 28 --- package.json | 76 ++++-- src/constants.js | 5 + src/errors.js | 8 + src/index.js | 340 ++++++++++++++++++++++---- src/proto.js | 4 +- src/rpc.js | 155 ------------ src/server/index.js | 171 +++++++++---- src/server/queue.js | 33 --- src/server/rpc.js | 159 ------------ src/server/rpc/handlers/discover.js | 64 +++++ src/server/rpc/handlers/index.js | 21 ++ src/server/rpc/handlers/register.js | 76 ++++++ src/server/rpc/handlers/unregister.js | 42 ++++ src/server/rpc/index.js | 65 +++++ src/server/store/basic/index.js | 60 ----- test/client-mode.spec.js | 47 ++++ test/client.id.json | 5 - test/client2.id.json | 5 - test/connectivity.spec.js | 67 +++++ test/discovery.spec.js | 68 ------ test/fixtures/browser.js | 7 + test/fixtures/peers.js | 27 ++ test/flows.spec.js | 101 ++++++++ test/rendezvous.spec.js | 224 +++++++++++++++++ test/server.id.json | 5 - test/utils.js | 122 ++++----- 31 files changed, 1509 insertions(+), 754 deletions(-) create mode 100644 .aegir.js create mode 100644 LIBP2P.md delete mode 100644 appveyor.yml create mode 100644 src/constants.js create mode 100644 src/errors.js delete mode 100644 src/rpc.js delete mode 100644 src/server/queue.js delete mode 100644 src/server/rpc.js create mode 100644 src/server/rpc/handlers/discover.js create mode 100644 src/server/rpc/handlers/index.js create mode 100644 src/server/rpc/handlers/register.js create mode 100644 src/server/rpc/handlers/unregister.js create mode 100644 src/server/rpc/index.js delete mode 100644 src/server/store/basic/index.js create mode 100644 test/client-mode.spec.js delete mode 100644 test/client.id.json delete mode 100644 test/client2.id.json create mode 100644 test/connectivity.spec.js delete mode 100644 test/discovery.spec.js create mode 100644 test/fixtures/browser.js create mode 100644 test/fixtures/peers.js create mode 100644 test/flows.spec.js create mode 100644 test/rendezvous.spec.js delete mode 100644 test/server.id.json diff --git a/.aegir.js b/.aegir.js new file mode 100644 index 0000000..e78b224 --- /dev/null +++ b/.aegir.js @@ -0,0 +1,57 @@ +'use strict' + +const Libp2p = require('libp2p') +const { MULTIADDRS_WEBSOCKETS } = require('./test/fixtures/browser') +const Peers = require('./test/fixtures/peers') +const PeerId = require('peer-id') +const WebSockets = require('libp2p-websockets') +const Muxer = require('libp2p-mplex') +const { NOISE: Crypto } = require('libp2p-noise') + +const Rendezvous = require('.') + +let libp2p, rendezvous + +const before = async () => { + // Use the last peer + const peerId = await PeerId.createFromJSON(Peers[Peers.length - 1]) + + libp2p = new Libp2p({ + addresses: { + listen: [MULTIADDRS_WEBSOCKETS[0]] + }, + peerId, + modules: { + transport: [WebSockets], + streamMuxer: [Muxer], + connEncryption: [Crypto] + }, + config: { + relay: { + enabled: true, + hop: { + enabled: true, + active: false + } + } + } + }) + + await libp2p.start() + + // rendezvous = new Rendezvous({ libp2p }) + // await rendezvous.start() +} + +const after = async () => { + // await rendezvous.stop() + await libp2p.stop() +} + +module.exports = { + bundlesize: { maxSize: '100kB' }, + hooks: { + pre: before, + post: after + } +} diff --git a/.travis.yml b/.travis.yml index 74f58e8..fdab469 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,31 +1,42 @@ -sudo: false language: node_js +cache: npm +stages: + - check + - test + - cov -matrix: - include: - - node_js: 6 - env: CXX=g++-4.8 - - node_js: 8 - env: CXX=g++-4.8 - # - node_js: stable - # env: CXX=g++-4.8 +node_js: + - '10' + - '12' + +os: + - linux + - osx + - windows -script: - - npm run lint - - npm run test - - npm run coverage +script: npx nyc -s npm run test:node -- --bail +after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov + +jobs: + include: + - stage: check + script: + - npx aegir dep-check + - npm run lint -before_script: - - export DISPLAY=:99.0 - - sh -e /etc/init.d/xvfb start + - stage: test + name: chrome + addons: + chrome: stable + script: + - npx aegir test -t browser -t webworker -after_success: - - npm run coverage-publish + - stage: test + name: firefox + addons: + firefox: latest + script: + - npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless -addons: - firefox: 'latest' - apt: - sources: - - ubuntu-toolchain-r-test - packages: - - g++-4.8 +notifications: + email: false \ No newline at end of file diff --git a/LIBP2P.md b/LIBP2P.md new file mode 100644 index 0000000..f3cf41e --- /dev/null +++ b/LIBP2P.md @@ -0,0 +1,47 @@ +# Rendezvous Protocol in js-libp2p + +The rendezvous protocol can be used in different contexts across libp2p. For using it, the libp2p network needs to have well known libp2p nodes acting as rendezvous servers. These nodes will have an extra role in the network. They will collect and maintain a list of registrations per rendezvous namespace. Other peers in the network will act as rendezvous clients and will register themselves on given namespaces by messaging a rendezvous server node. Taking into account these registrations, a rendezvous client is able to discover other peers in a given namespace by querying a server. + +## Usage + +`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows to enable the rendezvous protocol, as well as its server mode, enable automatic peer discover and to specify the topics to register from startup. + +The rendezvous comes with a discovery service that enables libp2p to automatically discover other peers in the provided namespaces and eventually connect to them. +**TODO: it should be compliant with the peer-discovery interface and configured as any other discovery service instead!!** + +You can configure it through libp2p as follows: + +```js +const Libp2p = require('libp2p') + +const node = await Libp2p.create({ + // ... required configurations + rendezvous: { + enabled: true, + namespaces: ['/namespace/1', '/namespace/2'], + discovery: { + enabled: true, + interval: 1000 + }, + server: { + enabled: true + } + } +}) +``` + +While `js-libp2p` supports the rendezvous protocol out of the box, it also provides a rendezvous API that users can interact with. This API should allow users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover other peers. + +## Libp2p Flow + +When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server and ask for nodes in the given namespaces. The rendezvous server can be added to the bootstrap nodes or manually dialed. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. + +If the discovery service is disabled, the rendezvous API should allow users to discover peers registered on provided namespaces. + +When a libp2p node running the rendezvous protocol is going to stop, it should unregister from all the namespaces previously registered. + +In the event of a rendezvous client getting connected to a second rendezvous server, it should propagate its registrations to it. The rendezvous server should clean its registrations for a peer when it is not connected with it anymore. + +## Other notes: + +After a query is made, who is responsible for determining if we need more records? (cookie reuse) diff --git a/README.md b/README.md index 1443992..995cc57 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,117 @@ -# libp2p-rendezvous +# js-libp2p-rendezvous -A javascript implementation of the rendezvous protocol for libp2p +[![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai) +[![](https://img.shields.io/badge/project-libp2p-yellow.svg?style=flat-square)](http://libp2p.io/) +[![](https://img.shields.io/badge/freenode-%23libp2p-yellow.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23libp2p) +[![](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io) +> Javascript implementation of the rendezvous protocol for libp2p + +## Overview + +Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. Any node implementing the rendezvous protocol can act as a rendezvous point, allowing the discovery of relevant peers in a decentralized fashion. + +See https://github.com/libp2p/specs/tree/master/rendezvous for more details ## Lead Maintainer [Vasco Santos](https://github.com/vasco-santos). -See https://github.com/libp2p/specs/pull/44 for more details +## API + +### rendezvous.register + +Registers the peer in a given namespace. + +`rendezvous.register(namespace, [ttl])` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to register | +| ttl | `number` | registration ttl in ms (default: `7200e3` and minimum `120`) | + +#### Returns + +| Type | Description | +|------|-------------| +| `Promise` | Remaining ttl value | + +#### Example + +```js +// ... +const ttl = await rendezvous.register(namespace) +``` + +### rendezvous.unregister + +Unregisters the peer from a given namespace. + +`rendezvous.unregister(namespace)` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to unregister | + +#### Returns + +| Type | Description | +|------|-------------| +| `Promise` | Operation resolved | + +#### Example + +```js +// ... +await rendezvous.register(namespace) +await rendezvous.unregister(namespace) +``` + +### rendezvous.discover + +Discovers peers registered under a given namespace. + +`rendezvous.discover(namespace, [limit], [cookie])` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to discover | +| limit | `number` | limit of peers to discover | +| cookie | `Buffer` | | + +#### Returns + +| Type | Description | +|------|-------------| +| `AsyncIterable<{ id: PeerId, signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | + +#### Example + +```js +// ... +await rendezvous.register(namespace) + +for await (const reg of rendezvous.discover(namespace)) { + console.log(reg.id, reg.signedPeerRecord, reg.ns, reg.ttl) +} +``` + +## Contribute + +Feel free to join in. All welcome. Open an [issue](https://github.com/libp2p/js-libp2p-pubsub-peer-discovery/issues)! + +This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). + +[![](https://cdn.rawgit.com/jbenet/contribute-ipfs-gif/master/img/contribute.gif)](https://github.com/ipfs/community/blob/master/contributing.md) + +## License + +MIT - Protocol Labs 2020 + +[multiaddr]: https://github.com/multiformats/js-multiaddr diff --git a/appveyor.yml b/appveyor.yml deleted file mode 100644 index 58aef65..0000000 --- a/appveyor.yml +++ /dev/null @@ -1,28 +0,0 @@ -version: "{build}" - -environment: - matrix: - - nodejs_version: "6" - - nodejs_version: "8" - -matrix: - fast_finish: true - -install: - # Install Node.js - - ps: Install-Product node $env:nodejs_version - - # Upgrade npm - - npm install -g npm - - # Output our current versions for debugging - - node --version - - npm --version - - # Install our package dependencies - - npm install - -test_script: - - npm run test:node - -build: off diff --git a/package.json b/package.json index 8e3af76..6218213 100644 --- a/package.json +++ b/package.json @@ -1,11 +1,16 @@ { "name": "libp2p-rendezvous", "version": "0.0.0", - "description": "A javascript implementation of the rendezvous protocol for libp2p", - "leadMaintainer": "Vasco Santos ", - "main": "index.js", - "scripts": { - "test": "aegir test" + "description": "Javascript implementation of the rendezvous protocol for libp2p", + "leadMaintainer": "Vasco Santos ", + "main": "src/index.js", + "files": [ + "dist", + "src" + ], + "repository": { + "type": "git", + "url": "git+https://github.com/libp2p/js-libp2p-rendezvous.git" }, "keywords": [ "libp2p", @@ -13,28 +18,49 @@ "protocol", "discovery" ], - "author": "Maciej Krüger ", - "license": "MIT", - "dependencies": { - "chai": "^4.1.2", - "dirty-chai": "^2.0.1", - "protons": "^1.0.1", - "pull-protocol-buffers": "^0.1.2" + "bugs": { + "url": "https://github.com/libp2p/js-libp2p-rendezvous/issues" }, - "devDependencies": { - "aegir": "^13.1.0", - "libp2p": "^0.20.2", - "libp2p-mplex": "^0.7.0", - "libp2p-secio": "^0.10.0", - "libp2p-spdy": "^0.12.1", - "libp2p-tcp": "^0.12.0" + "homepage": "https://libp2p.io", + "license": "MIT", + "engines": { + "node": ">=10.0.0", + "npm": ">=6.0.0" }, - "repository": { - "type": "git", - "url": "git+https://github.com/mkg20001/libp2p-rendezvous.git" + "scripts": { + "lint": "aegir lint", + "build": "aegir build", + "test": "aegir test", + "test:node": "aegir test -t node", + "test:browser": "aegir test -t browser", + "release": "aegir release", + "release-minor": "aegir release --type minor", + "release-major": "aegir release --type major", + "coverage": "nyc --reporter=text --reporter=lcov npm test" }, - "bugs": { - "url": "https://github.com/mkg20001/libp2p-rendezvous/issues" + "dependencies": { + "debug": "^4.1.1", + "err-code": "^2.0.3", + "it-buffer": "^0.1.2", + "it-length-prefixed": "^3.0.1", + "it-pipe": "^1.1.0", + "libp2p-interfaces": "^0.3.0", + "multiaddr": "^7.5.0", + "peer-id": "^0.13.13", + "protons": "^1.2.0", + "streaming-iterables": "^4.1.2" }, - "homepage": "https://github.com/mkg20001/libp2p-rendezvous#readme" + "devDependencies": { + "aegir": "^23.0.0", + "chai": "^4.2.0", + "chai-as-promised": "^7.1.1", + "dirty-chai": "^2.0.1", + "libp2p": "^0.28.3", + "libp2p-mplex": "^0.9.5", + "libp2p-noise": "^1.1.2", + "libp2p-websockets": "^0.13.6", + "p-times": "^3.0.0", + "p-wait-for": "^3.1.0", + "sinon": "^9.0.2" + } } diff --git a/src/constants.js b/src/constants.js new file mode 100644 index 0000000..3f063b7 --- /dev/null +++ b/src/constants.js @@ -0,0 +1,5 @@ +'use strict' + +exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' +exports.MAX_NS_LENGTH = 255 // TODO: spec this +exports.MAX_LIMIT = 1000 // TODO: spec this diff --git a/src/errors.js b/src/errors.js new file mode 100644 index 0000000..d130f4b --- /dev/null +++ b/src/errors.js @@ -0,0 +1,8 @@ +'use strict' + +exports.codes = { + INVALID_NAMESPACE: 'ERR_INVALID_NAMESPACE', + INVALID_TTL: 'ERR_INVALID_TTL', + INVALID_MULTIADDRS: 'ERR_INVALID_MULTIADDRS', + NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS' +} diff --git a/src/index.js b/src/index.js index fcbf075..eea845d 100644 --- a/src/index.js +++ b/src/index.js @@ -1,78 +1,320 @@ 'use strict' -const RPC = require('./rpc') -const noop = () => {} +const debug = require('debug') +const log = debug('libp2p:redezvous') +log.error = debug('libp2p:redezvous:error') -class RendezvousDiscovery { - constructor (swarm) { - this.swarm = swarm - this.peers = [] +const errCode = require('err-code') +const pipe = require('it-pipe') +const lp = require('it-length-prefixed') +const { collect } = require('streaming-iterables') +const { toBuffer } = require('it-buffer') + +const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') + +const Server = require('./server') +const { codes: errCodes } = require('./errors') +const { PROTOCOL_MULTICODEC } = require('./constants') +const { Message } = require('./proto') +const MESSAGE_TYPE = Message.MessageType + +/** + * Libp2p Rendezvous. + * A lightweight mechanism for generalized peer discovery. + */ +class Rendezvous { + /** + * @constructor + * @param {object} params + * @param {Libp2p} params.libp2p + * @param {object} params.options + * @param {boolean} [params.options.isServer = true] + */ + constructor ({ libp2p, options = { isServer: true } }) { + this._libp2p = libp2p + this._peerId = libp2p.peerId + this._registrar = libp2p.registrar + this._options = options + this._server = undefined + + /** + * @type {Map} + */ + this._rendezvousConns = new Map() + + this._registrarId = undefined + this._onPeerConnected = this._onPeerConnected.bind(this) + this._onPeerDisconnected = this._onPeerDisconnected.bind(this) } - _dial (pi, cb) { - if (!cb) cb = noop - this.swarm.dialProtocol(pi, '/rendezvous/1.0.0', (err, conn) => { - if (err) return cb(err) - const rpc = new RPC() - rpc.setup(conn, err => { - if (err) return cb(err) - this.peers.push(rpc) - cb() - }) + /** + * Register the rendezvous protocol in the libp2p node. + * @returns {Promise} + */ + async start () { + if (this._registrarId) { + return + } + + log('starting') + + // Create Rendezvous point if enabled + if (this._options.isServer) { + this._server = new Server({ registrar: this._registrar }) + } + + // register protocol with topology + const topology = new MulticodecTopology({ + multicodecs: PROTOCOL_MULTICODEC, + handlers: { + onConnect: this._onPeerConnected, + onDisconnect: this._onPeerDisconnected + } }) + this._registrarId = await this._registrar.register(topology) + + log('started') + } + + /** + * Unregister the rendezvous protocol and the streams with other peers will be closed. + * @returns {Promise} + */ + async stop () { + if (!this._registrarId) { + return + } + + log('stopping') + + // unregister protocol and handlers + await this._registrar.unregister(this._registrarId) + + this._registrarId = undefined + log('stopped') } - _rpc (cmd, ...a) { // TODO: add. round-robin / multicast / anycast? - this.peers[0][cmd](...a) + /** + * Registrar notifies a connection successfully with rendezvous protocol. + * @private + * @param {PeerId} peerId remote peer-id + * @param {Connection} conn connection to the peer + */ + _onPeerConnected (peerId, conn) { + const idB58Str = peerId.toB58String() + log('connected', idB58Str) + + this._rendezvousConns.set(idB58Str, conn) } - register (ns, peer, cb) { - this._rpc('register', ns, peer, 0, cb) // TODO: interface does not expose ttl option?! + /** + * Registrar notifies a closing connection with rendezvous protocol. + * @private + * @param {PeerId} peerId peerId + */ + _onPeerDisconnected (peerId) { + const idB58Str = peerId.toB58String() + log('disconnected', idB58Str) + + this._rendezvousConns.delete(idB58Str) + + if (this._server) { + this._server.removePeerRegistrations(peerId) + } } - discover (ns, limit, cookie, cb) { - if (typeof cookie === 'function') { - cb = cookie - cookie = Buffer.from('') + /** + * Register the peer in a given namespace + * @param {string} ns + * @param {number} [ttl = 7200e3] registration ttl in ms (minimum 120) + * @returns {Promise} + */ + async register (ns, ttl = 7200e3) { + if (!ns) { + throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) } - if (typeof limit === 'function') { - cookie = Buffer.from('') - cb = limit - limit = 0 + + if (ttl < 120) { + throw errCode(new Error('a valid ttl must be provided (bigger than 120)'), errCodes.INVALID_TTL) } - if (typeof ns === 'function') { - cookie = Buffer.from('') - limit = 0 - cb = ns - ns = null + + const addrs = [] + for (const m of this._libp2p.multiaddrs) { + if (!multiaddr.isMultiaddr(m)) { + throw errCode(new Error('one or more of the provided multiaddrs is not valid'), errCodes.INVALID_MULTIADDRS) + } + + addrs.push(m.buffer) + } + + // Are there available rendezvous servers? + if (!this._rendezvousConns.size) { + throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } - this._rpc('discover', ns, limit, cookie, cb) + const message = Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + peer: { + id: this._peerId.toBytes(), + addrs + }, + ns, + ttl // TODO: convert to seconds + } + }) + + const registerTasks = [] + const taskFn = async (id) => { + const conn = this._rendezvousConns.get(id) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [message], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + + if (!recMessage.type === MESSAGE_TYPE.REGISTER_RESPONSE) { + throw new Error('unexpected message received') + } + + return recMessage.registerResponse.ttl + } + + for (const id of this._rendezvousConns.keys()) { + registerTasks.push(taskFn(id)) + } + + // Return first ttl + const [returnTtl] = await Promise.all(registerTasks) + return returnTtl } - unregister (ns, id) { + /** + * Unregister peer from the nampesapce. + * @param {string} ns + * @returns {Promise} + */ + async unregister (ns) { if (!ns) { - id = this.swarm.peerInfo.id.toBytes() - ns = null + throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) } - if (!id) { - id = this.swarm.peerInfo.id.toBytes() + + // Are there available rendezvous servers? + if (!this._rendezvousConns.size) { + throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + } + + const message = Message.encode({ + type: MESSAGE_TYPE.UNREGISTER, + unregister: { + id: this._peerId.toBytes(), + ns + } + }) + + const unregisterTasks = [] + const taskFn = async (id) => { + const conn = this._rendezvousConns.get(id) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + await pipe( + [message], + lp.encode(), + stream, + async (source) => { + for await (const _ of source) { } // eslint-disable-line + } + ) + } + + for (const id of this._rendezvousConns.keys()) { + unregisterTasks.push(taskFn(id)) } - this._rpc('unregister', ns, id) + await Promise.all(unregisterTasks) } - start (cb) { - this.swarm.on('peer:connect', peer => { - this._dial(peer) + /** + * Discover peers registered under a given namespace + * @param {string} ns + * @param {number} [limit] + * @param {Buffer} [cookie] + * @returns {AsyncIterable<{ id: PeerId, multiaddrs: Array, ns: string, ttl: number }>} + */ + async * discover (ns, limit, cookie) { + // Are there available rendezvous servers? + if (!this._rendezvousConns.size) { + throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + } + + const registrationTransformer = (r) => ({ + id: PeerId.createFromBytes(r.peer.id), + multiaddrs: r.peer.addrs && r.peer.addrs.map((a) => multiaddr(a)), + ns: r.ns, + ttl: r.ttl }) - cb() - } - stop (cb) { - // TODO: shutdown all conns - cb() + // Local search if Server + if (this._server) { + const localRegistrations = this._server.getRegistrations(ns, limit) + for (const r of localRegistrations) { + yield registrationTransformer(r) + + limit-- + if (limit === 0) { + return + } + } + } + + const message = Message.encode({ + type: MESSAGE_TYPE.DISCOVER, + discover: { + ns, + limit, + cookie + } + }) + + for (const id of this._rendezvousConns.keys()) { + const conn = this._rendezvousConns.get(id) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [message], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + + if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { + throw new Error('unexpected message received') + } + + for (const r of recMessage.discoverResponse.registrations) { + // track registrations and check if already provided + yield registrationTransformer(r) + + limit-- + if (limit === 0) { + return + } + } + } } } -module.exports = RendezvousDiscovery +module.exports = Rendezvous diff --git a/src/proto.js b/src/proto.js index b2c6d94..ed9574b 100644 --- a/src/proto.js +++ b/src/proto.js @@ -3,6 +3,7 @@ const protons = require('protons') module.exports = protons(` +message Message { enum MessageType { REGISTER = 0; REGISTER_RESPONSE = 1; @@ -19,6 +20,7 @@ module.exports = protons(` E_INVALID_COOKIE = 103; E_NOT_AUTHORIZED = 200; E_INTERNAL_ERROR = 300; + E_UNAVAILABLE = 400; } message PeerInfo { @@ -35,6 +37,7 @@ module.exports = protons(` message RegisterResponse { optional ResponseStatus status = 1; optional string statusText = 2; + optional int64 ttl = 3; // in seconds } message Unregister { @@ -55,7 +58,6 @@ module.exports = protons(` optional string statusText = 4; } -message Message { optional MessageType type = 1; optional Register register = 2; optional RegisterResponse registerResponse = 3; diff --git a/src/rpc.js b/src/rpc.js deleted file mode 100644 index 5d0781b..0000000 --- a/src/rpc.js +++ /dev/null @@ -1,155 +0,0 @@ -'use strict' - -const pull = require('pull-stream') -const ppb = require('pull-protocol-buffers') -const {Message, MessageType} = require('./proto') -const Pushable = require('pull-pushable') -const debug = require('debug') -const log = debug('libp2p-rendezvous:rpc') -const Peer = require('peer-info') -const Id = require('peer-id') -const once = require('once') - -const TIMEOUT = 1000 * 10 // TODO: spec this - -function wrap (f, t) { - let cb = once((...a) => { - clearTimeout(timeout) - f(...a) - }) - let timeout - timeout = setTimeout(() => cb(new Error('Timeout!')), t) - return cb -} - -class RPC { - constructor () { - this.source = Pushable() - this.cbs = { - discover: [], - register: [] - } - } - sink (read) { - const next = (end, msg, doend) => { - if (doend) { - log('crash@%s: %s', this.id, doend) - return read(doend, next) - } - if (end) { - this.online = false - log('end@%s: %s', this.id, end) - this.source.end() - return - } - let f - let pi - switch (msg.type) { - case MessageType.REGISTER_RESPONSE: - f = this.cbs.register.shift() - if (typeof f !== 'function') { - log('register@%s: response ignored, no cb found!', this.id) - return read(null, next) - } else { - let e - if (msg.registerResponse.status) { - e = new Error('Server returned error: ' + (msg.registerResponse.statusText || '(unknown code)')) - } - f(e) - } - break - case MessageType.DISCOVER_RESPONSE: - try { - f = this.cbs.discover.shift() - if (typeof f !== 'function') { - log('discover@%s: response ignored, no cb found!', this.id) - return read(null, next) - } else { - if (msg.discoverResponse.status) { - return setImmediate(() => f(new Error('Server returned error: ' + (msg.discoverResponse.statusText || '(unknown code)')))) - } - pi = msg.discoverResponse.registrations.map(p => { - try { - // TODO: use other values like ttl/ns in peer-info? - const pi = new Peer(new Id(p.peer.id)) - p.peer.addrs.forEach(a => pi.multiaddrs.add(a)) - return pi - } catch (e) { - log('discover@%s: invalid pi returned: %s', this.id, e) - } - }).filter(Boolean) - setImmediate(() => f(null, { - cookie: msg.discoverResponse.cookie, - peers: pi - })) - } - } catch (e) { - f(e) - return next(null, null, e) - } - break - default: // should that disconnect or just get ignored? - log('error@%s: sent wrong msg type %s', this.id, msg.type) - return next(null, null, true) - } - read(null, next) - } - read(null, next) - } - setup (conn, cb) { - conn.getPeerInfo((err, pi) => { - if (err) return cb(err) - this.pi = pi - this.id = pi.id.toB58String() - pull( - conn, - ppb.decode(Message), - this, - ppb.encode(Message), - conn - ) - - this.online = true - cb() - }) - } - - register (ns, peer, ttl, cb) { - this.source.push({ - type: MessageType.REGISTER, - register: { - ns, - peer: { - id: peer.id.toBytes(), - addrs: peer.multiaddrs.toArray().map(a => a.buffer) - }, - ttl - } - }) - this.cbs.register.push(wrap(cb, TIMEOUT)) - } - - discover (ns, limit, cookie, cb) { - this.source.push({ - type: MessageType.DISCOVER, - discover: { - ns, - limit, - cookie - } - }) - this.cbs.discover.push(wrap(cb, TIMEOUT)) - } - - unregister (ns, id) { - this.source.push({ - type: MessageType.UNREGISTER, - unregister: { - ns, - id - } - }) - } -} - -module.exports = RPC diff --git a/src/server/index.js b/src/server/index.js index e7d7ca7..0786ccb 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -1,73 +1,136 @@ 'use strict' -// const {waterfall} = require('async') -const RPC = require('./rpc') const debug = require('debug') -const log = debug('libp2p:rendezvous:server') -const AsyncQueue = require('./queue') -const BasicStore = require('./store/basic') - -class Server { - constructor (opt) { - if (!opt) opt = {} - this.node = opt.node - this.config = opt.config - this.que = new AsyncQueue() - this.table = { - NS: {}, - RPC: {} - } - const Store = opt.store || BasicStore - this.store = new Store(this) - this._stubNS = this.store.create(Buffer.alloc(256, '0').toString()) +const log = debug('libp2p:redezvous-server') +log.error = debug('libp2p:redezvous-server:error') + +const { PROTOCOL_MULTICODEC, MAX_LIMIT } = require('../constants') +const rpc = require('./rpc') + +/** +* Rendezvous registration. +* @typedef {Object} Registration +* @property {PeerId} peerId +* @property {Array} addrs +* @property {number} expiration +*/ + +/** + * Libp2p rendezvous server. + */ +class RendezvousServer { + /** + * @constructor + * @param {object} params + * @param {Registrar} params.registrar + */ + constructor ({ registrar }) { + this._registrar = registrar + + /** + * Registrations per namespace. + * @type {Map>} + */ + this.registrations = new Map() + + // Incoming streams handling + this._registrar.handle(PROTOCOL_MULTICODEC, rpc(this)) } - start () { - this.gcIntv = setInterval(this.gc.bind(this), 60 * 1000) - this.node.handle('/rendezvous/1.0.0', (proto, conn) => { - const rpc = new RPC(this) - rpc.setup(conn, err => { - if (err) return log(err) - this.storeRPC(rpc) - }) + // TODO: Should we have a start method to gv the expired registrations? + // I am removing them on discover, but it should be useful to have a gc too + + /** + * Add a peer registration to a namespace. + * @param {string} ns + * @param {PeerId} peerId + * @param {Array} addrs + * @param {number} ttl + * @returns {void} + */ + addRegistration (ns, peerId, addrs, ttl) { + const nsRegistrations = this.registrations.get(ns) || new Map() + + nsRegistrations.set(peerId.toB58String(), { + peerId, + addrs, + expiration: Date.now() + ttl }) - } - stop () { - clearInterval(this.gcIntv) - // TODO: clear vars, shutdown conns, etc. - this.node.unhandle('/rendezvous/1.0.0') + this.registrations.set(ns, nsRegistrations) } - storeRPC (rpc) { - // TODO: should a peer that's connected twice be overriden or rejected? - this.table.RPC[rpc.id] = rpc - // TODO: remove on disconnect + /** + * Remove rengistration of a given namespace to a peer + * @param {string} ns + * @param {PeerId} peerId + * @returns {void} + */ + removeRegistration (ns, peerId) { + const nsRegistrations = this.registrations.get(ns) + + if (nsRegistrations) { + nsRegistrations.delete(peerId.toB58String()) + + // Remove registrations map to namespace if empty + if (!nsRegistrations.size) { + this.registrations.delete(ns) + } + log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) + } } - getNS (name, create) { - if (!this.table.NS[name]) { - if (create) { - return (this.table.NS[name] = this.store.create(name)) - } else { - return this._stubNS + /** + * Remove registrations of a given peer + * @param {PeerId} peerId + * @returns {void} + */ + removePeerRegistrations (peerId) { + for (const [ns, reg] of this.registrations.entries()) { + reg.delete(peerId.toB58String()) + + // Remove registrations map to namespace if empty + if (!reg.size) { + this.registrations.delete(ns) } } - return this.table.NS[name] + + log('removed existing registrations for peer', peerId.toB58String()) } - gc () { - Object.keys(this.table.NS).forEach(ns => { - const n = this.table.NS[ns] - const removed = n.gc() - if (n.useless) { - log('drop NS %s because it is empty', n.name) - delete this.table.NS[ns] - } else { - if (removed) n.update() + /** + * Get registrations for a namespace + * @param {string} ns + * @param {number} limit + * @returns {Array} + */ + getRegistrations (ns, limit = MAX_LIMIT) { + const nsRegistrations = this.registrations.get(ns) || new Map() + const registrations = [] + + for (const [idStr, reg] of nsRegistrations.entries()) { + if (reg.expiration <= Date.now()) { + // Clean outdated registration + nsRegistrations.delete(idStr) + continue } - }) + + registrations.push({ + ns, + peer: { + id: reg.peerId.toBytes(), + addrs: reg.addrs + }, + ttl: reg.expiration - Date.now() + }) + + // Stop if reached limit + if (registrations.length === limit) { + break + } + } + return registrations } } -module.exports = Server +module.exports = RendezvousServer diff --git a/src/server/queue.js b/src/server/queue.js deleted file mode 100644 index f12b5c6..0000000 --- a/src/server/queue.js +++ /dev/null @@ -1,33 +0,0 @@ -'use strict' - -const debug = require('debug') -const log = debug('libp2p:rendezvous:queue') - -class AsyncQueue { - constructor () { - this.tasks = [] - this.taskIds = {} - this.triggered = false - } - add (name, fnc) { - if (this.taskIds[name]) return - log('queueing %s', name) - this.taskIds[name] = true - this.tasks.push(fnc) - this.trigger() - } - trigger () { - if (this.triggered) return - this.triggered = true - setTimeout(() => { - log('exec') - this.tasks.forEach(f => f()) - this.tasks = [] - this.taskIds = {} - this.triggered = false - log('exec done') - }, 100).unref() - } -} - -module.exports = AsyncQueue diff --git a/src/server/rpc.js b/src/server/rpc.js deleted file mode 100644 index fd49d2d..0000000 --- a/src/server/rpc.js +++ /dev/null @@ -1,159 +0,0 @@ -'use strict' - -const pull = require('pull-stream') -const ppb = require('pull-protocol-buffers') -const {Message, MessageType, ResponseStatus} = require('../proto') -const Pushable = require('pull-pushable') -const debug = require('debug') -const log = debug('libp2p-rendezvous:server:rpc') -const Peer = require('peer-info') -const Id = require('peer-id') - -const MAX_NS_LENGTH = 255 // TODO: spec this -const MAX_LIMIT = 1000 // TODO: spec this - -const registerErrors = { - 100: 'Invalid namespace provided', - 101: 'Invalid peer-info provided', - 102: 'Invalid TTL provided', - 103: 'Invalid cookie provided', - 200: 'Not authorized', - 300: 'Internal Server Error' -} - -const craftStatus = (status) => { - return { - status, - statusText: registerErrors[status] - } -} - -class RPC { - constructor (main) { - this.main = main - this.source = Pushable() - } - sink (read) { - const next = (end, msg, doend) => { - if (doend) { - log('crash@%s: %s', this.id, doend) - return read(doend, next) - } - if (end) { - this.online = false - log('end@%s: %s', this.id, end) - this.source.end() - return - } - switch (msg.type) { - case MessageType.REGISTER: - try { - log('register@%s: trying register on %s', this.id, msg.register.ns) - if (msg.register.peer.id && new Id(msg.register.peer.id).toB58String() !== this.id) { - log('register@%s: auth err (want %s)', this.id, new Id(msg.register.peer.id).toB58String()) - this.source.push({ - type: MessageType.REGISTER_RESPONSE, - registerResponse: craftStatus(ResponseStatus.E_NOT_AUTHORIZED) - }) - return read(null, next) - } else if (!msg.register.peer.id) { - msg.register.peer.id = this.pi.id.toBytes() - } - if (msg.register.ns > MAX_NS_LENGTH) { - log('register@%s: ns err', this.id) - this.source.push({ - type: MessageType.REGISTER_RESPONSE, - registerResponse: craftStatus(ResponseStatus.E_INVALID_NAMESPACE) - }) - return read(null, next) - } - const pi = new Peer(new Id(msg.register.peer.id)) - msg.register.peer.addrs.forEach(a => pi.multiaddrs.add(a)) - this.main.getNS(msg.register.ns, true).addPeer(pi, Date.now(), msg.register.ttl, () => this.online) - log('register@%s: ok', this.id) - this.source.push({ - type: MessageType.REGISTER_RESPONSE, - registerResponse: craftStatus(ResponseStatus.OK) - }) - } catch (e) { - log('register@%s: internal error', this.id) - log(e) - this.source.push({ - type: MessageType.REGISTER_RESPONSE, - registerResponse: craftStatus(ResponseStatus.E_INTERNAL_ERROR) - }) - return read(null, next) - } - break - case MessageType.UNREGISTER: - try { - log('unregister@%s: unregister from %s', this.id, msg.unregister.ns) - // TODO: currently ignores id since there is no ownership error. change? - this.main.getNS(msg.unregister.ns).removePeer(this.id) - } catch (e) { - return next(null, null, e) - } - break - case MessageType.DISCOVER: - try { - // TODO: add more errors - log('discover@%s: discover on %s', this.id, msg.discover.ns) - if (msg.discover.limit <= 0 || msg.discover.limit > MAX_LIMIT) msg.discover.limit = MAX_LIMIT - const {peers, cookie} = this.main.getNS(msg.discover.ns).getPeers(msg.discover.cookie || Buffer.from(''), msg.discover.limit, this.id) - log('discover@%s: got %s peers', this.id, peers.length) - this.source.push({ - type: MessageType.DISCOVER_RESPONSE, - discoverResponse: { - registrations: peers.map(p => { - return { - ns: msg.discover.ns, - peer: { - id: p.pi.id.toBytes(), - addrs: p.pi.multiaddrs.toArray().map(a => a.buffer) - }, - ttl: p.ttl - } - }), - cookie - } - }) - } catch (e) { - log('discover@%s: internal error', this.id) - log(e) - this.source.push({ - type: MessageType.DISCOVER_RESPONSE, - registerResponse: craftStatus(ResponseStatus.E_INTERNAL_ERROR) - }) - return read(null, next) - } - break - // case MessageType.REGISTER_RESPONSE: - // case MessageType.DISCOVER_RESPONSE: - default: // should that disconnect or just get ignored? - log('error@%s: sent wrong msg type %s', this.id, msg.type) - return next(null, null, true) - } - read(null, next) - } - read(null, next) - } - setup (conn, cb) { - conn.getPeerInfo((err, pi) => { - if (err) return cb(err) - this.pi = pi - this.id = pi.id.toB58String() - pull( - conn, - ppb.decode(Message), - this, - ppb.encode(Message), - conn - ) - - this.online = true - cb() - }) - } -} - -module.exports = RPC diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js new file mode 100644 index 0000000..a82ae7f --- /dev/null +++ b/src/server/rpc/handlers/discover.js @@ -0,0 +1,64 @@ + +'use strict' + +const debug = require('debug') +const log = debug('libp2p:redezvous:protocol:discover') +log.error = debug('libp2p:redezvous:protocol:discover:error') + +const { Message } = require('../../../proto') +const MESSAGE_TYPE = Message.MessageType +const RESPONSE_STATUS = Message.ResponseStatus + +const { MAX_NS_LENGTH, MAX_LIMIT } = require('../../../constants') + +module.exports = (rendezvousPoint) => { + /** + * Process `Discover` Rendezvous messages. + * + * @param {PeerId} peerId + * @param {Message} msg + * @returns {Message} + */ + return function discover (peerId, msg) { + try { + log(`discover ${peerId.toB58String()}: discover on ${msg.discover.ns}`) + + // Validate namespace + if (!msg.discover.ns || msg.discover.ns > MAX_NS_LENGTH) { + log.error(`invalid namespace received: ${msg.discover.ns}`) + + return { + type: MESSAGE_TYPE.DISCOVER_RESPONSE, + discoverResponse: { + status: RESPONSE_STATUS.E_INVALID_NAMESPACE + } + } + } + + if (!msg.discover.limit || msg.discover.limit <= 0 || msg.discover.limit > MAX_LIMIT) { + msg.discover.limit = MAX_LIMIT + } + + // Get registrations + const registrations = rendezvousPoint.getRegistrations(msg.discover.ns, msg.discover.limit) + + return { + type: MESSAGE_TYPE.DISCOVER_RESPONSE, + discoverResponse: { + cookie: undefined, // TODO + registrations, + status: RESPONSE_STATUS.OK + } + } + } catch (err) { + log.error(err) + } + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + discoverResponse: { + status: RESPONSE_STATUS.E_INTERNAL_ERROR + } + } + } +} diff --git a/src/server/rpc/handlers/index.js b/src/server/rpc/handlers/index.js new file mode 100644 index 0000000..89248ab --- /dev/null +++ b/src/server/rpc/handlers/index.js @@ -0,0 +1,21 @@ +'use strict' + +const { Message } = require('../../../proto') +const MESSAGE_TYPE = Message.MessageType + +module.exports = (server) => { + const handlers = { + [MESSAGE_TYPE.REGISTER]: require('./register')(server), + [MESSAGE_TYPE.UNREGISTER]: require('./unregister')(server), + [MESSAGE_TYPE.DISCOVER]: require('./discover')(server) + } + + /** + * Get the message handler matching the passed in type. + * @param {number} type + * @returns {function(PeerId, Message, function(Error, Message))} + */ + return function getMessageHandler (type) { + return handlers[type] + } +} diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js new file mode 100644 index 0000000..e6ed495 --- /dev/null +++ b/src/server/rpc/handlers/register.js @@ -0,0 +1,76 @@ + +'use strict' + +const debug = require('debug') +const log = debug('libp2p:redezvous:protocol:register') +log.error = debug('libp2p:redezvous:protocol:register:error') + +const { Message } = require('../../../proto') +const MESSAGE_TYPE = Message.MessageType +const RESPONSE_STATUS = Message.ResponseStatus + +const { MAX_NS_LENGTH } = require('../../../constants') + +module.exports = (rendezvousPoint) => { + /** + * Process `Register` Rendezvous messages. + * + * @param {PeerId} peerId + * @param {Message} msg + * @returns {Message} + */ + return function register (peerId, msg) { + try { + log(`register ${peerId.toB58String()}: trying register on ${msg.register.ns}`) + + // Validate auth + if (!msg.register.peer.id.equals(peerId.toBytes())) { + log.error('unauthorized peer id to register') + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.E_NOT_AUTHORIZED + } + } + } + + // Validate namespace + if (!msg.register.ns || msg.register.ns > MAX_NS_LENGTH) { + log.error(`invalid namespace received: ${msg.register.ns}`) + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.E_INVALID_NAMESPACE + } + } + } + + // Add registration + rendezvousPoint.addRegistration( + msg.register.ns, + peerId, + msg.register.peer.addrs, + msg.register.ttl + ) + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.OK, + ttt: msg.register.ttl + } + } + } catch (err) { + log.error(err) + } + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.E_INTERNAL_ERROR + } + } + } +} diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js new file mode 100644 index 0000000..936dc77 --- /dev/null +++ b/src/server/rpc/handlers/unregister.js @@ -0,0 +1,42 @@ + +'use strict' + +const debug = require('debug') +const log = debug('libp2p:redezvous:protocol:unregister') +log.error = debug('libp2p:redezvous:protocol:unregister:error') + +module.exports = (rendezvousPoint) => { + /** + * Process `Unregister` Rendezvous messages. + * + * @param {PeerId} peerId + * @param {Message} msg + */ + return function unregister (peerId, msg) { + try { + log(`unregister ${peerId.toB58String()}: trying unregister from ${msg.unregister.ns}`) + + if (!msg.unregister.id && !msg.unregister.ns) { + throw new Error('no peerId or namespace provided') + } + + // Validate auth + if (!msg.unregister.id.equals(peerId.toBytes())) { + log.error('unauthorized peer id to unregister') + + // TODO: auth validation of peerId? -- there is no answer + return + } + + // Remove registration + if (!msg.unregister.ns) { + rendezvousPoint.removePeerRegistrations(peerId) + } else { + rendezvousPoint.removeRegistration(msg.unregister.ns, peerId) + } + } catch (err) { + log.error(err) + } + // TODO: internal error? -- there is no answer + } +} diff --git a/src/server/rpc/index.js b/src/server/rpc/index.js new file mode 100644 index 0000000..d4f423e --- /dev/null +++ b/src/server/rpc/index.js @@ -0,0 +1,65 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:redezvous-point:rpc') +log.error = debug('libp2p:redezvous-point:rpc:error') + +const pipe = require('it-pipe') +const lp = require('it-length-prefixed') +const { toBuffer } = require('it-buffer') + +const handlers = require('./handlers') +const { Message } = require('../../proto') + +module.exports = (rendezvous) => { + const getMessageHandler = handlers(rendezvous) + + /** + * Process incoming Rendezvous messages. + * @param {PeerId} peerId + * @param {Message} msg + * @returns {Promise} + */ + function handleMessage (peerId, msg) { + const handler = getMessageHandler(msg.type) + + if (!handler) { + log.error(`no handler found for message type: ${msg.type}`) + return + } + + return handler(peerId, msg) + } + + /** + * Handle incoming streams on the rendezvous protocol. + * @param {Object} props + * @param {DuplexStream} props.stream + * @param {Connection} props.connection connection + * @returns {Promise} + */ + return async function onIncomingStream ({ stream, connection }) { + const peerId = connection.remotePeer + + log('incoming stream from: %s', peerId.toB58String()) + + await pipe( + stream.source, + lp.decode(), + toBuffer, + source => (async function * () { + for await (const msg of source) { + // handle the message + const desMessage = Message.decode(msg) + const res = await handleMessage(peerId, desMessage) + + if (res) { + yield Message.encode(res) + } + } + })(), + lp.encode(), + stream.sink + ) + } +} diff --git a/src/server/store/basic/index.js b/src/server/store/basic/index.js deleted file mode 100644 index 0095090..0000000 --- a/src/server/store/basic/index.js +++ /dev/null @@ -1,60 +0,0 @@ -'use strict' - -class NS { - constructor (name, que) { // name is a utf8 string - this.name = name - this.hexName = Buffer.from(name).toString('hex') // needed to prevent queue-dos attacks - this.que = que - this.id = {} - this.sorted = [] - } - addPeer (pi, ts, ttl, isOnline) { // isOnline returns a bool if the rpc connection still exists - const id = pi.id.toB58String() - this.id[id] = {pi, ts, ttl} - if (ttl) { - let expireAt = ts + ttl * 1000 - this.id[id].online = () => Date.now() >= expireAt - } else { - this.id[id].online = isOnline - } - this.update() - } - removePeer (pid) { - delete this.id[pid] - this.update() - } - update () { - this.que.add('sort@' + this.hexName, () => { - this.sorted = Object.keys(this.id).map(id => { return {id, ts: this.id[id].ts} }).sort((a, b) => a.ts - b.ts) - }) - } - getPeers (cookie, limit, ownId) { - cookie = cookie.length ? parseInt(cookie.toString(), 10) : 0 - let p = this.sorted.filter(p => p.ts > cookie && p.id !== ownId).slice(0, limit).map(p => this.id[p.id]) - let newCookie - if (p.length) { - newCookie = Buffer.from(p[p.length - 1].ts.toString()) - } else { - newCookie = Buffer.from('') - } - return {cookie: newCookie, peers: p} - } - gc () { - return Object.keys(this.id).filter(k => !this.id[k].online()).map(k => delete this.id[k]).length - } - get useless () { - return !Object.keys(this.id).length - } -} - -class BasicStore { - constructor (main) { - this.main = main - } - create (name) { - return new NS(name, this.main.que) - } -} - -module.exports = BasicStore -module.exports.NS = NS diff --git a/test/client-mode.spec.js b/test/client-mode.spec.js new file mode 100644 index 0000000..1dcb441 --- /dev/null +++ b/test/client-mode.spec.js @@ -0,0 +1,47 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai +const sinon = require('sinon') + +const Rendezvous = require('../src') + +const { createPeer } = require('./utils') + +describe('client mode', () => { + let peer, rendezvous + + afterEach(async () => { + peer && await peer.stop() + rendezvous && await rendezvous.stop() + }) + + it('registers a rendezvous handler by default', async () => { + [peer] = await createPeer() + rendezvous = new Rendezvous({ libp2p: peer }) + + const spyHandle = sinon.spy(peer.registrar, '_handle') + + await rendezvous.start() + + expect(spyHandle).to.have.property('callCount', 1) + }) + + it('can be started only in client mode', async () => { + [peer] = await createPeer() + rendezvous = new Rendezvous({ + libp2p: peer, + options: { + isRendezvousPoint: false + } + }) + + const spyHandle = sinon.spy(peer.registrar, '_handle') + + await rendezvous.start() + expect(spyHandle).to.have.property('callCount', 0) + }) +}) diff --git a/test/client.id.json b/test/client.id.json deleted file mode 100644 index 97b1858..0000000 --- a/test/client.id.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "QmVMx9YqSRYB75sGcYiTVtCpygxcfaxcuSJMBYoBadgJ7r", - "privKey": "CAASqQkwggSlAgEAAoIBAQCZPiah1KCGIIsMDXvxxk3djZfgCpckUDOAsG83FNwLx+3Z8Lg1LLAoArtula6/4LOaTaRZA9LKiSBw4yEgTMlinw77hxg6SLoHeMHi1AS/0MxCQuKxWZaeM5dtFkiUU/qJVwhTksIjmHtm/gWcBjmObAnRzeHIOLdlBL+6tcELYKH4OKcxD/VWMxFBbo5bjnPTddeQpSEtTVzsqX4kC4sBIHO3otEY8z9nefRXP8zIZ3TpfWcXMAhzbF7aJWHlUkDSblDCH41JlDXenvcerTlPN0Oqdj+8e5914qSTdSPAHbFyiGKeDc55thYZI2jDpNksSOZ2/HhDOjmNAE970VrRAgMBAAECggEBAJVnQdTvb521JruWfevHkezaemMFEDxoMP5bheKm5K5buuqLxZyaOBiaKVD0gE40bgaXgg8DKkUqkkVdO9O46XLMbpgOKzHP7AcS1b0nRoYYtLw5Z7jPBoiw9gZ1/kcW5SF3h/erEroPlOhh6ugmLYFMlfpGBsXlfe/wRFltkItcohvQKm45tDowmRNX7Gw/qE2m2Bcu8nF34OWNqSlhi/LccZWMHYXemy/MyYPeM2pMsebyEOKQb7cTKPUS7UqOYotvq8p8b/IRRi0vJZNtnON3qQDZhP3uPzZYFErWcrL3BIEgwMBNcJb2yNIvrc7LnH8xO0l7fLuYmJW1su8mQf0CgYEA5BlzcW24UPTvZdPjWHAGZBwK8RLHKxh615jfTjO4SI9ggdNVIOAWFQSKAyJT4hLyvg+4yR21WpZMj2pR3loO+Nn7djviB8a8SlWeqPSjpN/QT9nSl6JVcdoNp4vEyu+9O2MbNJp5oCZ/q6k9DovvZuxMc/2cNWo/YR4K6xcvehcCgYEAq/ywujkJ9fptpkJ5HzO7zt5sy+dodT2EDPTCeMQeLNXTC+P31th2FIl4/FzGrLJzVtWmP3kUS0es8hrRuLBy+Zg51czMliX8eJa36Vk305EAynkfIcGEzgmdbDVx6QyIWk1Y18WzVyuiVnLMbxT6ZfNXahwLNnV7umk0OH1bK1cCgYEAo9GTk7dVVO9UsDFJak6qiGOLiDAQUuc18nmchzGl/JbcnOEGlqHZuiaUaEPTMt6g79eiwu5PPUwMmEOnoKXVcuw7KWNApo0Y1dpAJN/uV49WsMKj+Lth2m7ct6QuJgGgSnKXK2R2TYrYzpSxgS0HN0gmcHeIJOS1uC43cTgppOkCgYAFTGSZaBZxeISWQaf/mRVpGxsY8Qkby4hc6dFv7QLM+M1mqWBCQyroGRAcHjOUsG6zNyPHAtDoPM4MK11Ypj70h4cImiWXXpY3lNUXoEMDBo2Sr0aRQKf5vPwXkFHxDwzIU2ewRgvvXI3EwgagSXIpX+TKhRCnXdkw9frA3sPHQwKBgQC4mCqbD3DhjiQFPOhhlZtHgruKs6b6L7npyfc4ZrDVwGFrOcbBQbuLD2f92wbB0FKWozx4FP0nlawigVHimMWG4qyJycDMBptfsUrqvxzRagmci+ZRY+cjyz2sA1Tox4nE4vVEz5ZxBGdmkRLY+hWRZMWDJrYLoZSBEKj0B336yw==", - "pubKey": "CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCZPiah1KCGIIsMDXvxxk3djZfgCpckUDOAsG83FNwLx+3Z8Lg1LLAoArtula6/4LOaTaRZA9LKiSBw4yEgTMlinw77hxg6SLoHeMHi1AS/0MxCQuKxWZaeM5dtFkiUU/qJVwhTksIjmHtm/gWcBjmObAnRzeHIOLdlBL+6tcELYKH4OKcxD/VWMxFBbo5bjnPTddeQpSEtTVzsqX4kC4sBIHO3otEY8z9nefRXP8zIZ3TpfWcXMAhzbF7aJWHlUkDSblDCH41JlDXenvcerTlPN0Oqdj+8e5914qSTdSPAHbFyiGKeDc55thYZI2jDpNksSOZ2/HhDOjmNAE970VrRAgMBAAE=" -} diff --git a/test/client2.id.json b/test/client2.id.json deleted file mode 100644 index 57c78fa..0000000 --- a/test/client2.id.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "QmcpjE4Mgs1kc31gN6DSRDPwQqY8C2x6iGuFo3gzTxJdSQ", - "privKey": "CAASpgkwggSiAgEAAoIBAQCZ2FkxB7wjzf1H8fqWGh843E5ZtmSQpQ4DtP4HYZAgNPNc66GmP9itAR44WziPgS3BC3gfXuWE7OTtZ0dhZj2e2BTpKZwXVoAwBaV9IZrbTntFsG50rHzoyCulGvJu2RPmG1PyE9+WLXM+oiaRqY2YshLoMkS88BsRw9+PBaA2jA75Bay/wm8AsmXEc/PloAGE6PiyM6nD/66WGpxScZ2Z1BO0MUy0MUBBCvA+D8fxTpzLqBGHrIJxIK+MebAkWNFWrWyvdnapABCxvL5pSjjjGb1SyUkeTr8Bn/IsFfnjjuZaszKTqYsNGZCxlDJdrL2rlEoP4MIEP2wM+WHYqFIBAgMBAAECggEAZV4mNqYwEy9xCeyo/iosFF0kyvvg+2Wl/E9PajGgs3fwOnOPyWkcLbIk5WFFvViSezZBafovJQyqMrrwT378byNVc+RU0xPN1taBmheAX6wwkVSVEw9sJj1udJVy1BL4h4/OGh16HwvHeaeB3kxn3grHZnNo000pqOT08tn0HLvZ1396SUXjyNahMco/dTOtUz8PQAKtu+MW0ekkczGi6mBPaiR/rzlzQZRresQ368azwjTTwe1T4YwQ7buPxrW2GRfnVfU5Uvp99acKnj5bKixQtyZDiKbHWK5PX3diMfYuPdgdz0R4EOIokE0akR+lxpk4Um7+gDHbI6ufvLQSAQKBgQDX0WGxpJg6nwHspxdS75syelyRuGWoS6PPxqhQws37RLSV6aoAZoAzvhxDPT3aDaR1VVJuxnDpVjidCTArk3vHuzOMriwKLDz2NifkWnSCiBdCjpY2aqbF+sQgwLAf+5RV8xQOkc2+4p0xXYV+21XXWLN8R2gezB7exXyrvH4CkQKBgQC2fSHOcvCM9ZmyLc99bfGJ0NbTC0s9G0EkFWni9V61vfQYxME4Mi1XPRwOrCP/dguATL/QRMD24xvZHYR1+vwY+A3MPZdOtrgeVdjia9VuZ8VLEdgFTFWOY9d3XlbLiiRSj0G8IjC+WEN5k5mWJn7XA+HdkQhuY+NyJ1nI5q4wcQKBgEUxqHTgJL6GxIMvf1bj44pnmM5PpKg0uCyhsM1T596rxIpcBFlkg64TQdR9ChujTBsiY++ISCNHtZcDnyIZgxIifwCXxx7r2A/IhTm9lqVTJMH+HUMNJrNLFx65KL7YVlLIQKH7NVACMAvnxClMAVWt5r3t1wAoyaz6/GHDaVNBAoGAJVtoWELfS3vbgsYt+5dOItBFqd5eAJxbsW9Qxc1FHh9MoOVmSIK9FWbFH5vNorYflJwhiBkLB39mbAPG4gAHK3VcHbteBhcRieQ5CeDZSEil8sAsYKlHumZl7WG6kuAsn1oEMucs40peRb0Za8tlm86HpjvSZga8wNmdX6sZbYECgYArG+pWYWjjXi1ithkCaLiL1816KPBqwXOjPHjXFuUSpbU+O7lBlEhMD6Fcj8IuxQzvbwn8L7TZRyZtN5xYBlPAblAU5PTgMKU1mQiqt0IyfifnyG0As+fbTcmGMAmd089UGBp/OLblgzoXfDhhMSo/ymrfjFYOQr88mPnbRagGWA==", - "pubKey": "CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCZ2FkxB7wjzf1H8fqWGh843E5ZtmSQpQ4DtP4HYZAgNPNc66GmP9itAR44WziPgS3BC3gfXuWE7OTtZ0dhZj2e2BTpKZwXVoAwBaV9IZrbTntFsG50rHzoyCulGvJu2RPmG1PyE9+WLXM+oiaRqY2YshLoMkS88BsRw9+PBaA2jA75Bay/wm8AsmXEc/PloAGE6PiyM6nD/66WGpxScZ2Z1BO0MUy0MUBBCvA+D8fxTpzLqBGHrIJxIK+MebAkWNFWrWyvdnapABCxvL5pSjjjGb1SyUkeTr8Bn/IsFfnjjuZaszKTqYsNGZCxlDJdrL2rlEoP4MIEP2wM+WHYqFIBAgMBAAE=" -} diff --git a/test/connectivity.spec.js b/test/connectivity.spec.js new file mode 100644 index 0000000..0314f79 --- /dev/null +++ b/test/connectivity.spec.js @@ -0,0 +1,67 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +const { expect } = chai +const pWaitFor = require('p-wait-for') + +const multiaddr = require('multiaddr') + +const Rendezvous = require('../src') + +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] +const { createPeer } = require('./utils') + +describe('connectivity', () => { + let peers + + beforeEach(async () => { + // Create libp2p nodes + peers = await createPeer({ + number: 2 + }) + + // Create && start rendezvous + peers.map((libp2p) => { + const rendezvous = new Rendezvous({ libp2p }) + rendezvous.start() + libp2p.rendezvous = rendezvous + }) + + // Connect to testing relay node + await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) + }) + + afterEach(() => peers.map(async (libp2p) => { + await libp2p.rendezvous.stop() + await libp2p.stop() + })) + + it('updates known rendezvous points', async () => { + expect(peers[0].rendezvous._rendezvousConns.size).to.equal(0) + expect(peers[1].rendezvous._rendezvousConns.size).to.equal(0) + + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peers[1].peerId.toB58String()}`) + const connection = await peers[0].dial(m) + + expect(peers[0].peerStore.peers.size).to.equal(2) + expect(peers[1].peerStore.peers.size).to.equal(2) + + // Wait event propagation + // Relay peer is not with rendezvous enabled + await pWaitFor(() => + peers[0].rendezvous._rendezvousConns.size === 1 && + peers[1].rendezvous._rendezvousConns.size === 1) + + expect(peers[0].rendezvous._rendezvousConns.get(peers[1].peerId.toB58String())).to.exist() + expect(peers[1].rendezvous._rendezvousConns.get(peers[0].peerId.toB58String())).to.exist() + + await connection.close() + + // Wait event propagation + await pWaitFor(() => peers[0].rendezvous._rendezvousConns.size === 0) + }) +}) diff --git a/test/discovery.spec.js b/test/discovery.spec.js deleted file mode 100644 index 687b40f..0000000 --- a/test/discovery.spec.js +++ /dev/null @@ -1,68 +0,0 @@ -'use strict' - -/* eslint-env mocha */ - -const {parallel} = require('async') -const Utils = require('./utils') - -const chai = require('chai') -const dirtyChai = require('dirty-chai') -const expect = chai.expect -chai.use(dirtyChai) - -describe('discovery', () => { - let client - let client2 - let server - - before(done => { - Utils.default((err, _client, _server, _client2) => { - if (err) return done(err) - client = _client - client2 = _client2 - server = _server - parallel([client, client2].map(c => cb => c._dial(server.node.peerInfo, cb)), done) - }) - }) - - it('register', done => { - parallel( - [client, client2].map(c => cb => c.register('hello', c.swarm.peerInfo, cb)), - (...a) => setTimeout(() => done(...a), 100) // Queue is being processed every 100ms - ) - }) - - it('discover', done => { - client.discover('hello', (err, res) => { - if (err) return done(err) - expect(err).to.not.exist() - expect(res.peers).to.have.lengthOf(1) - expect(res.peers[0].id.toB58String()).to.equal(client2.swarm.peerInfo.id.toB58String()) - done() - }) - }) - - it('unregister', done => { - client2.unregister('hello') - setTimeout(() => done(), 100) // Queue is being processed every 100ms - }) - - it('discover (after unregister)', done => { - client.discover('hello', (err, res) => { - if (err) return done(err) - expect(err).to.not.exist() - expect(res.peers).to.have.lengthOf(0) - done() - }) - }) - - it('unregister other client', done => { - client.unregister('hello') - setTimeout(() => done(), 100) // Queue is being processed every 100ms - }) - - it('gc', () => { - server.gc() - expect(Object.keys(server.table.NS)).to.have.lengthOf(0) - }) -}) diff --git a/test/fixtures/browser.js b/test/fixtures/browser.js new file mode 100644 index 0000000..901c32e --- /dev/null +++ b/test/fixtures/browser.js @@ -0,0 +1,7 @@ +'use strict' + +const multiaddr = require('multiaddr') + +module.exports.MULTIADDRS_WEBSOCKETS = [ + multiaddr('/ip4/127.0.0.1/tcp/15001/ws/p2p/QmckxVrJw1Yo8LqvmDJNUmdAsKtSbiKWmrXJFyKmUraBoN') +] diff --git a/test/fixtures/peers.js b/test/fixtures/peers.js new file mode 100644 index 0000000..fad0d23 --- /dev/null +++ b/test/fixtures/peers.js @@ -0,0 +1,27 @@ +'use strict' + +module.exports = [{ + id: 'QmNMMAqSxPetRS1cVMmutW5BCN1qQQyEr4u98kUvZjcfEw', + privKey: 'CAASpQkwggShAgEAAoIBAQDPek2aeHMa0blL42RTKd6xgtkk4Zkldvq4LHxzcag5uXepiQzWANEUvoD3KcUTmMRmx14PvsxdLCNst7S2JSa0R2n5wSRs14zGy6892lx4H4tLBD1KSpQlJ6vabYM1CJhIQRG90BtzDPrJ/X1iJ2HA0PPDz0Mflam2QUMDDrU0IuV2m7gSCJ5r4EmMs3U0xnH/1gShkVx4ir0WUdoWf5KQUJOmLn1clTRHYPv4KL9A/E38+imNAXfkH3c2T7DrCcYRkZSpK+WecjMsH1dCX15hhhggNqfp3iulO1tGPxHjm7PDGTPUjpCWKpD5e50sLqsUwexac1ja6ktMfszIR+FPAgMBAAECggEAB2H2uPRoRCAKU+T3gO4QeoiJaYKNjIO7UCplE0aMEeHDnEjAKC1HQ1G0DRdzZ8sb0fxuIGlNpFMZv5iZ2ZFg2zFfV//DaAwTek9tIOpQOAYHUtgHxkj5FIlg2BjlflGb+ZY3J2XsVB+2HNHkUEXOeKn2wpTxcoJE07NmywkO8Zfr1OL5oPxOPlRN1gI4ffYH2LbfaQVtRhwONR2+fs5ISfubk5iKso6BX4moMYkxubYwZbpucvKKi/rIjUA3SK86wdCUnno1KbDfdXSgCiUlvxt/IbRFXFURQoTV6BOi3sP5crBLw8OiVubMr9/8WE6KzJ0R7hPd5+eeWvYiYnWj4QKBgQD6jRlAFo/MgPO5NZ/HRAk6LUG+fdEWexA+GGV7CwJI61W/Dpbn9ZswPDhRJKo3rquyDFVZPdd7+RlXYg1wpmp1k54z++L1srsgj72vlg4I8wkZ4YLBg0+zVgHlQ0kxnp16DvQdOgiRFvMUUMEgetsoIx1CQWTd67hTExGsW+WAZQKBgQDT/WaHWvwyq9oaZ8G7F/tfeuXvNTk3HIJdfbWGgRXB7lJ7Gf6FsX4x7PeERfL5a67JLV6JdiLLVuYC2CBhipqLqC2DB962aKMvxobQpSljBBZvZyqP1IGPoKskrSo+2mqpYkeCLbDMuJ1nujgMP7gqVjabs2zj6ACKmmpYH/oNowJ/T0ZVtvFsjkg+1VsiMupUARRQuPUWMwa9HOibM1NIZcoQV2NGXB5Z++kR6JqxQO0DZlKArrviclderUdY+UuuY4VRiSEprpPeoW7ZlbTku/Ap8QZpWNEzZorQDro7bnfBW91fX9/81ets/gCPGrfEn+58U3pdb9oleCOQc/ifpQKBgBTYGbi9bYbd9vgZs6bd2M2um+VFanbMytS+g5bSIn2LHXkVOT2UEkB+eGf9KML1n54QY/dIMmukA8HL1oNAyalpw+/aWj+9Ui5kauUhGEywHjSeBEVYM9UXizxz+m9rsoktLLLUI0o97NxCJzitG0Kub3gn0FEogsUeIc7AdinZAoGBANnM1vcteSQDs7x94TDEnvvqwSkA2UWyLidD2jXgE0PG4V6tTkK//QPBmC9eq6TIqXkzYlsErSw4XeKO91knFofmdBzzVh/ddgx/NufJV4tXF+a2iTpqYBUJiz9wpIKgf43/Ob+P1EA99GAhSdxz1ess9O2aTqf3ANzn6v6g62Pv', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDPek2aeHMa0blL42RTKd6xgtkk4Zkldvq4LHxzcag5uXepiQzWANEUvoD3KcUTmMRmx14PvsxdLCNst7S2JSa0R2n5wSRs14zGy6892lx4H4tLBD1KSpQlJ6vabYM1CJhIQRG90BtzDPrJ/X1iJ2HA0PPDz0Mflam2QUMDDrU0IuV2m7gSCJ5r4EmMs3U0xnH/1gShkVx4ir0WUdoWf5KQUJOmLn1clTRHYPv4KL9A/E38+imNAXfkH3c2T7DrCcYRkZSpK+WecjMsH1dCX15hhhggNqfp3iulO1tGPxHjm7PDGTPUjpCWKpD5e50sLqsUwexac1ja6ktMfszIR+FPAgMBAAE=' +}, { + id: 'QmW8rAgaaA6sRydK1k6vonShQME47aDxaFidbtMevWs73t', + privKey: 'CAASpwkwggSjAgEAAoIBAQCTU3gVDv3SRXLOsFln9GEf1nJ/uCEDhOG10eC0H9l9IPpVxjuPT1ep+ykFUdvefq3D3q+W3hbmiHm81o8dYv26RxZIEioToUWp7Ec5M2B/niYoE93za9/ZDwJdl7eh2hNKwAdxTmdbXUPjkIU4vLyHKRFbJIn9X8w9djldz8hoUvC1BK4L1XrT6F2l0ruJXErH2ZwI1youfSzo87TdXIoFKdrQLuW6hOtDCGKTiS+ab/DkMODc6zl8N47Oczv7vjzoWOJMUJs1Pg0ZsD1zmISY38P0y/QyEhatZn0B8BmSWxlLQuukatzOepQI6k+HtfyAAjn4UEqnMaXTP1uwLldVAgMBAAECggEAHq2f8MqpYjLiAFZKl9IUs3uFZkEiZsgx9BmbMAb91Aec+WWJG4OLHrNVTG1KWp+IcaQablEa9bBvoToQnS7y5OpOon1d066egg7Ymfmv24NEMM5KRpktCNcOSA0CySpPIB6yrg6EiUr3ixiaFUGABKkxmwgVz/Q15IqM0ZMmCUsC174PMAz1COFZxD0ZX0zgHblOJQW3dc0X3XSzhht8vU02SMoVObQHQfeXEHv3K/RiVj/Ax0bTc5JVkT8dm8xksTtsFCNOzRBqFS6MYqX6U/u0Onz3Jm5Jt7fLWb5n97gZR4SleyGrqxYNb46d9X7mP0ie7E6bzFW0DsWBIeAqVQKBgQDW0We2L1n44yOvJaMs3evpj0nps13jWidt2I3RlZXjWzWHiYQfvhWUWqps/xZBnAYgnN/38xbKzHZeRNhrqOo+VB0WK1IYl0lZVE4l6TNKCsLsUfQzsb1pePkd1eRZA+TSqsi+I/IOQlQU7HA0bMrah/5FYyUBP0jYvCOvYTlZuwKBgQCvkcVRydVlzjUgv7lY5lYvT8IHV5iYO4Qkk2q6Wjv9VUKAJZauurMdiy05PboWfs5kbETdwFybXMBcknIvZO4ihxmwL8mcoNwDVZHI4bXapIKMTCyHgUKvJ9SeTcKGC7ZuQJ8mslRmYox/HloTOXEJgQgPRxXcwa3amzvdZI+6LwKBgQCLsnQqgxKUi0m6bdR2qf7vzTH4258z6X34rjpT0F5AEyF1edVFOz0XU/q+lQhpNEi7zqjLuvbYfSyA026WXKuwSsz7jMJ/oWqev/duKgAjp2npesY/E9gkjfobD+zGgoS9BzkyhXe1FCdP0A6L2S/1+zg88WOwMvJxl6/xLl24XwKBgCm60xSajX8yIQyUpWBM9yUtpueJ2Xotgz4ST+bVNbcEAddll8gWFiaqgug9FLLuFu5lkYTHiPtgc1RNdphvO+62/9MRuLDixwh/2TPO+iNqwKDKJjda8Nei9vVddCPaOtU/xNQ0xLzFJbG9LBmvqH9izOCcu8SJwGHaTcNUeJj/AoGADCJ26cY30c13F/8awAAmFYpZWCuTP5ppTsRmjd63ixlrqgkeLGpJ7kYb5fXkcTycRGYgP0e1kssBGcmE7DuG955fx3ZJESX3GQZ+XfMHvYGONwF1EiK1f0p6+GReC2VlQ7PIkoD9o0hojM6SnWvv9EXNjCPALEbfPFFvcniKVsE=', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCTU3gVDv3SRXLOsFln9GEf1nJ/uCEDhOG10eC0H9l9IPpVxjuPT1ep+ykFUdvefq3D3q+W3hbmiHm81o8dYv26RxZIEioToUWp7Ec5M2B/niYoE93za9/ZDwJdl7eh2hNKwAdxTmdbXUPjkIU4vLyHKRFbJIn9X8w9djldz8hoUvC1BK4L1XrT6F2l0ruJXErH2ZwI1youfSzo87TdXIoFKdrQLuW6hOtDCGKTiS+ab/DkMODc6zl8N47Oczv7vjzoWOJMUJs1Pg0ZsD1zmISY38P0y/QyEhatZn0B8BmSWxlLQuukatzOepQI6k+HtfyAAjn4UEqnMaXTP1uwLldVAgMBAAE=' +}, { + id: 'QmZqCdSzgpsmB3Qweb9s4fojAoqELWzqku21UVrqtVSKi4', + privKey: 'CAASpgkwggSiAgEAAoIBAQCdbSEsTmw7lp5HagRcx57DaLiSUEkh4iBcKc7Y+jHICEIA8NIVi9FlfGEZj9G21FpiTR4Cy+BLVEuf8Nm90bym4iV+cSumeS21fvD8xGTEbeKGljs6OYHy3M45JhWF85gqHQJOqZufI2NRDuRgMZEO2+qGEXmSlv9mMXba/+9ecze8nSpB7bG2Z2pnKDeYwhF9Cz+ElMyn7TBWDjJERGVgFbTpdM3rBnbhB/TGpvs732QqZmIBlxnDb/Jn0l1gNZCgkEDcJ/0NDMBJTQ8vbvcdmaw3eaMPLkn1ix4wdu9QWCA0IBtuY1R7vSUtf4irnLJG7DnAw2GfM5QrF3xF1GLXAgMBAAECggEAQ1N0qHoxl5pmvqv8iaFlqLSUmx5y6GbI6CGJMQpvV9kQQU68yjItr3VuIXx8d/CBZyEMAK4oko7OeOyMcr3MLKLy3gyQWnXgsopDjhZ/8fH8uwps8g2+IZuFJrO+6LaxEPGvFu06fOiphPUVfn40R2KN/iBjGeox+AaXijmCqaV2vEdNJJPpMfz6VKZBDLTrbiqvo/3GN1U99PUqfPWpOWR29oAhh/Au6blSqvqTUPXB2+D/X6e1JXv31mxMPK68atDHSUjZWKB9lE4FMK1bkSKJRbyXmNIlbZ9V8X4/0r8/6T7JnW7ZT8ugRkquohmwgG7KkDXB1YsOCKXYUqzVYQKBgQDtnopFXWYl7XUyePJ/2MA5i7eoko9jmF44L31irqmHc5unNf6JlNBjlxTNx3WyfzhUzrn3c18psnGkqtow0tkBj5hmqn8/WaPbc5UA/5R1FNaNf8W5khn7MDm6KtYRPjN9djqTDiVHyC6ljONYd+5S+MqyKVWZ3t/xvG60sw85qwKBgQCpmpDtL+2JBwkfeUr3LyDcQxvbfzcv8lXj2otopWxWiLiZF1HzcqgAa2CIwu9kCGEt9Zr+9E4uINbe1To0b01/FhvR6xKO/ukceGA/mBB3vsKDcRmvpBUp+3SmnhY0nOk+ArQl4DhJ34k8pDM3EDPrixPf8SfVdU/8IM32lsdHhQKBgHLgpvCKCwxjFLnmBzcPzz8C8TOqR3BbBZIcQ34l+wflOGdKj1hsfaLoM8KYn6pAHzfBCd88A9Hg11hI0VuxVACRL5jS7NnvuGwsIOluppNEE8Ys86aXn7/0vLPoab3EWJhbRE48FIHzobmft3nZ4XpzlWs02JGfUp1IAC2UM9QpAoGAeWy3pZhSr2/iEC5+hUmwdQF2yEbj8+fDpkWo2VrVnX506uXPPkQwE1zM2Bz31t5I9OaJ+U5fSpcoPpDaAwBMs1fYwwlRWB8YNdHY1q6/23svN3uZsC4BGPV2JnO34iMUudilsRg+NGVdk5TbNejbwx7nM8Urh59djFzQGGMKeSECgYA0QMCARPpdMY50Mf2xQaCP7HfMJhESSPaBq9V3xY6ToEOEnXgAR5pNjnU85wnspHp+82r5XrKfEQlFxGpj2YA4DRRmn239sjDa29qP42UNAFg1+C3OvXTht1d5oOabaGhU0udwKmkEKUbb0bG5xPQJ5qeSJ5T1gLzLk3SIP0GlSw==', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCdbSEsTmw7lp5HagRcx57DaLiSUEkh4iBcKc7Y+jHICEIA8NIVi9FlfGEZj9G21FpiTR4Cy+BLVEuf8Nm90bym4iV+cSumeS21fvD8xGTEbeKGljs6OYHy3M45JhWF85gqHQJOqZufI2NRDuRgMZEO2+qGEXmSlv9mMXba/+9ecze8nSpB7bG2Z2pnKDeYwhF9Cz+ElMyn7TBWDjJERGVgFbTpdM3rBnbhB/TGpvs732QqZmIBlxnDb/Jn0l1gNZCgkEDcJ/0NDMBJTQ8vbvcdmaw3eaMPLkn1ix4wdu9QWCA0IBtuY1R7vSUtf4irnLJG7DnAw2GfM5QrF3xF1GLXAgMBAAE=' +}, { + id: 'QmR5VwgsL7jyfZHAGyp66tguVrQhCRQuRc3NokocsCZ3fA', + privKey: 'CAASpwkwggSjAgEAAoIBAQCGXYU+uc2nn1zuJhfdFOl34upztnrD1gpHu58ousgHdGlGgYgbqLBAvIAauXdEL0+e30HofjA634SQxE+9nV+0FQBam1DDzHQlXsuwHV+2SKvSDkk4bVllMFpu2SJtts6VH+OXC/2ANJOm+eTALykQPYXgLIBxrhp/eD+Jz5r6wW2nq3k6OmYyK/4pgGzFjo5UyX+fa/171AJ68UPboFpDy6BZCcUjS0ondxPvD7cv5jMNqqMKIB/7rpi8n+Q3oeccRqVL56wH+FE3/QLjwYHwY6ILNRyvNXRqHjwBEXB2R5moXN0AFUWTw9rt3KhFiEjR1U81BTw5/xS7W2Iu0FgZAgMBAAECggEAS64HK8JZfE09eYGJNWPe8ECmD1C7quw21BpwVe+GVPSTizvQHswPohbKDMNj0srXDMPxCnNw1OgqcaOwyjsGuZaOoXoTroTM8nOHRIX27+PUqzaStS6aCG2IsiCozKUHjGTuupftS7XRaF4eIsUtWtFcQ1ytZ9pJYHypRQTi5NMSrTze5ThjnWxtHilK7gnBXik+aR0mYEVfSn13czQEC4rMOs+b9RAc/iibDNoLopfIdvmCCvfxzmySnR7Cu1iSUAONkir7PB+2Mt/qRFCH6P+jMamtCgQ8AmifXgVmDUlun+4MnKg3KrPd6ZjOEKhVe9mCHtGozk65RDREShfDdQKBgQDi+x2MuRa9peEMOHnOyXTS+v+MFcfmG0InsO08rFNBKZChLB+c9UHBdIvexpfBHigSyERfuDye4z6lxi8ZnierWMYJP30nxmrnxwTGTk1MQquhfs1A0kpmDnPsjlOS/drEIEIssNx2WbfJ7YtMxLWBtp+BJzGpQmr0LKC+NHRSrwKBgQCXiy2kJESIUkIs2ihV55hhT6/bZo1B1O5DPA2nkjOBXqXF6fvijzMDX82JjLd07lQZlI0n1Q/Hw0p4iYi9YVd2bLkLXF5UIb2qOeHj76enVFOrPHUSkC9Y2g/0Xs+60Ths2xRd8RrrfQU3kl5iVpBywkCIrb2M5+wRnNTk1W3TtwKBgQCvplyrteAfSurpJhs9JzE8w/hWU9SqAZYkWQp91W1oE95Um2yrbjBAoQxMjaqKS+f/APPIjy56Vqj4aHGyhW11b/Fw3qzfxvCcBKtxOs8eoMlo5FO6QgJJEA4tlcafDcvp0nzjUMqK28safLU7503+33B35fjMXxWdd5u9FaKfCQKBgC4W6j6tuRosymuRvgrCcRnHfpify/5loEFallyMnpWOD6Tt0OnK25z/GifnYDRz96gAAh5HMpFy18dpLOlMHamqz2yhHx8/U8vd5tHIJZlCkF/X91M5/uxrBccwvsT2tM6Got8fYSyVzWxlW8dUxIHiinYHQUsFjkqdBDLEpq5pAoGASoTw5RBEWFM0GuAZdXsyNyxU+4S+grkTS7WdW/Ymkukh+bJZbnvF9a6MkSehqXnknthmufonds2AFNS//63gixENsoOhzT5+2cdfc6tJECvJ9xXVXkf85AoQ6T/RrXF0W4m9yQyCngNJUrKUOIH3oDIfdZITlYzOC3u1ojj7VuQ=', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCGXYU+uc2nn1zuJhfdFOl34upztnrD1gpHu58ousgHdGlGgYgbqLBAvIAauXdEL0+e30HofjA634SQxE+9nV+0FQBam1DDzHQlXsuwHV+2SKvSDkk4bVllMFpu2SJtts6VH+OXC/2ANJOm+eTALykQPYXgLIBxrhp/eD+Jz5r6wW2nq3k6OmYyK/4pgGzFjo5UyX+fa/171AJ68UPboFpDy6BZCcUjS0ondxPvD7cv5jMNqqMKIB/7rpi8n+Q3oeccRqVL56wH+FE3/QLjwYHwY6ILNRyvNXRqHjwBEXB2R5moXN0AFUWTw9rt3KhFiEjR1U81BTw5/xS7W2Iu0FgZAgMBAAE=' +}, { + id: 'QmScLDqRg7H6ipCYxm9fVk152UWavQFKscTdoT4YNHxgqp', + privKey: 'CAASpwkwggSjAgEAAoIBAQCWEHaTZ6LBLFP5OPrUqjDM/cF4b2zrfh1Zm3kd02ZtgQB3iYtZqRPJT5ctT3A7WdVF/7dCxPGOCkJlLekTx4Y4gD8JtjA+EfN9fR/2RBKbti2N3CD4vkGp9ss4hbBFcXIhl8zuD/ELHutbV6b8b4QXJGnxfp/B+1kNPnyd7SJznS0QyvI8OLI1nAkVKdYLDRW8kPKeHyx1xhdNDuTQVTFyAjRGQ4e3UYFB7bYIHW3E6kCtCoJDlj+JPC02Yt1LHzIzZVLvPvNFnYY2mag6OiGFuh/oMBIqvnPc1zRZ3eLUqeGZjQVaoR0kdgZUKz7Q2TBeNldxK/s6XO0DnkQTlelNAgMBAAECggEAdmt1dyswR2p4tdIeNpY7Pnj9JNIhTNDPznefI0dArCdBvBMhkVaYk6MoNIxcj6l7YOrDroAF8sXr0TZimMY6B/pERKCt/z1hPWTxRQBBAvnHhwvwRPq2jK6BfhAZoyM8IoBNKowP9mum5QUNdGV4Al8s73KyFX0IsCfgZSvNpRdlt+DzPh+hu/CyoZaMpRchJc1UmK8Fyk3KfO+m0DZNfHP5P08lXNfM6MZLgTJVVgERHyG+vBOzTd2RElMe19nVCzHwb3dPPRZSQ7Fnz3rA+GeLqsM2Zi4HNhfbD1OcD9C4wDj5tYL6hWTkdz4IlfVcjCeUHxgIOhdDV2K+OwbuAQKBgQD0FjUZ09UW2FQ/fitbvIB5f1SkXWPxTF9l6mAeuXhoGv2EtQUO4vq/PK6N08RjrZdWQy6UsqHgffi7lVQ8o3hvCKdbtf4sP+cM92OrY0WZV89os79ndj4tyvmnP8WojwRjt/2XEfgdoWcgWxW9DiYINTOQVimZX+X/3on4s8hEgQKBgQCdY3kOMbyQeLTRkqHXjVTY4ddO+v4S4wOUa1l4rTqAbq1W3JYWwoDQgFuIu3limIHmjnSJpCD4EioXFsM7p6csenoc20sHxsaHnJ6Mn5Te41UYmY9EW0otkQ0C3KbXM0hwQkjyplnEmZawGKmjEHW8DJ3vRYTv9TUCgYKxDHgOzQKBgB4A/NYH7BG61eBYKgxEx6YnuMfbkwV+Vdu5S8d7FQn3B2LgvZZu4FPRqcNVXLbEB+5ao8czjiKCWaj1Wj15+rvrXGcxn+Tglg5J+r5+nXeUC7LbJZQaPNp0MOwWMr3dlrSLUWjYlJ9Pz9VyXOG4c4Rexc/gR4zK9QLW4C7qKpwBAoGAZzyUb0cYlPtYQA+asTU3bnvVKy1f8yuNcZFowst+EDiI4u0WVh+HNzy6zdmLKa03p+/RaWeLaK0hhrubnEnAUmCUMNF3ScaM+u804LDcicc8TkKLwx7ObU0z56isl4RAA8K27tNHFrpYKXJD834cfBkaj5ReOrfw6Y/iFhhDuBECgYEA8gbC76uz7LSHhW30DSRTcqOzTyoe2oYKQaxuxYNp7vSSOkcdRen+mrdflDvud2q/zN2QdL4pgqdldHlR35M/lJ0f0B6zp74jlzbO9700wzsOqreezGc5eWiroDL100U9uIZ50BKb8CKtixIHpinUSPIUcVDkSAZ2y7mbfCxQwqQ=', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCWEHaTZ6LBLFP5OPrUqjDM/cF4b2zrfh1Zm3kd02ZtgQB3iYtZqRPJT5ctT3A7WdVF/7dCxPGOCkJlLekTx4Y4gD8JtjA+EfN9fR/2RBKbti2N3CD4vkGp9ss4hbBFcXIhl8zuD/ELHutbV6b8b4QXJGnxfp/B+1kNPnyd7SJznS0QyvI8OLI1nAkVKdYLDRW8kPKeHyx1xhdNDuTQVTFyAjRGQ4e3UYFB7bYIHW3E6kCtCoJDlj+JPC02Yt1LHzIzZVLvPvNFnYY2mag6OiGFuh/oMBIqvnPc1zRZ3eLUqeGZjQVaoR0kdgZUKz7Q2TBeNldxK/s6XO0DnkQTlelNAgMBAAE=' +}, { + id: 'QmckxVrJw1Yo8LqvmDJNUmdAsKtSbiKWmrXJFyKmUraBoN', + privKey: 'CAASpwkwggSjAgEAAoIBAQC1/GFud/7xutux7qRfMj1sIdMRh99/chR6HqVj6LQqrgk4jil0mdN/LCk/tqPqmDtObHdmEhCoybzuhLbCKgUqryKDwO6yBJHSKWY9QqrKZtLJ37SgKwGjE3+NUD4r1dJHhtQrICFdOdSCBzs/v8gi+J+KZLHo7+Nms4z09ysy7qZh94Pd7cW4gmSMergqUeANLD9C0ERw1NXolswOW7Bi7UGr7yuBxejICLO3nkxe0OtpQBrYrqdCD9vs3t/HQZbPWVoiRj4VO7fxkAPKLl30HzcIfxj/ayg8NHcH59d08D+N2v5Sdh28gsiYKIPE9CXvuw//HUY2WVRY5fDC5JglAgMBAAECggEBAKb5aN/1w3pBqz/HqRMbQpYLNuD33M3PexBNPAy+P0iFpDo63bh5Rz+A4lvuFNmzUX70MFz7qENlzi6+n/zolxMB29YtWBUH8k904rTEjXXl//NviQgITZk106tx+4k2x5gPEm57LYGfBOdFAUzNhzDnE2LkXwRNzkS161f7zKwOEsaGWRscj6UvhO4MIFxjb32CVwt5eK4yOVqtyMs9u30K4Og+AZYTlhtm+bHg6ndCCBO6CQurCQ3jD6YOkT+L3MotKqt1kORpvzIB0ujZRf49Um8wlcjC5G9aexBeGriXaVdPF62zm7GA7RMsbQM/6aRbA1fEQXvJhHUNF9UFeaECgYEA8wCjKqQA7UQnHjRwTsktdwG6szfxd7z+5MTqHHTWhWzgcQLgdh5/dO/zanEoOThadMk5C1Bqjq96gH2xim8dg5XQofSVtV3Ui0dDa+XRB3E3fyY4D3RF5hHv85O0GcvQc6DIb+Ja1oOhvHowFB1C+CT3yEgwzX/EK9xpe+KtYAkCgYEAv7hCnj/DcZFU3fAfS+unBLuVoVJT/drxv66P686s7J8UM6tW+39yDBZ1IcwY9vHFepBvxY2fFfEeLI02QFM+lZXVhNGzFkP90agNHK01psGgrmIufl9zAo8WOKgkLgbYbSHzkkDeqyjEPU+B0QSsZOCE+qLCHSdsnTmo/TjQhj0CgYAz1+j3yfGgrS+jVBC53lXi0+2fGspbf2jqKdDArXSvFqFzuudki/EpY6AND4NDYfB6hguzjD6PnoSGMUrVfAtR7X6LbwEZpqEX7eZGeMt1yQPMDr1bHrVi9mS5FMQR1NfuM1lP9Xzn00GIUpE7WVrWUhzDEBPJY/7YVLf0hFH08QKBgDWBRQZJIVBmkNrHktRrVddaSq4U/d/Q5LrsCrpymYwH8WliHgpeTQPWmKXwAd+ZJdXIzYjCt202N4eTeVqGYOb6Q/anV2WVYBbM4avpIxoA28kPGY6nML+8EyWIt2ApBOmgGgvtEreNzwaVU9NzjHEyv6n7FlVwlT1jxCe3XWq5AoGASYPKQoPeDlW+NmRG7z9EJXJRPVtmLL40fmGgtju9QIjLnjuK8XaczjAWT+ySI93Whu+Eujf2Uj7Q+NfUjvAEzJgwzuOd3jlQvoALq11kuaxlNQTn7rx0A1QhBgUJE8AkvShPC9FEnA4j/CLJU0re9H/8VvyN6qE0Mho0+YbjpP8=', + pubKey: 'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC1/GFud/7xutux7qRfMj1sIdMRh99/chR6HqVj6LQqrgk4jil0mdN/LCk/tqPqmDtObHdmEhCoybzuhLbCKgUqryKDwO6yBJHSKWY9QqrKZtLJ37SgKwGjE3+NUD4r1dJHhtQrICFdOdSCBzs/v8gi+J+KZLHo7+Nms4z09ysy7qZh94Pd7cW4gmSMergqUeANLD9C0ERw1NXolswOW7Bi7UGr7yuBxejICLO3nkxe0OtpQBrYrqdCD9vs3t/HQZbPWVoiRj4VO7fxkAPKLl30HzcIfxj/ayg8NHcH59d08D+N2v5Sdh28gsiYKIPE9CXvuw//HUY2WVRY5fDC5JglAgMBAAE=' +}] diff --git a/test/flows.spec.js b/test/flows.spec.js new file mode 100644 index 0000000..3a32042 --- /dev/null +++ b/test/flows.spec.js @@ -0,0 +1,101 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +const { expect } = chai + +const pWaitFor = require('p-wait-for') +const multiaddr = require('multiaddr') + +const Rendezvous = require('../src') + +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] +const { createPeer } = require('./utils') + +const namespace = 'ns' + +describe('flows', () => { + describe('3 rendezvous all acting as rendezvous point', () => { + let peers + + const connectPeers = async (peer, otherPeer) => { + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) + await peer.dial(m) + + // Wait event propagation + await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) + } + + beforeEach(async () => { + // Create libp2p nodes + peers = await createPeer({ + number: 3 + }) + + // Create 3 rendezvous peers + peers.forEach((peer) => { + const rendezvous = new Rendezvous({ + libp2p: peer + }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + + // Connect to testing relay node + await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) + }) + + afterEach(() => peers.map(async (libp2p) => { + await libp2p.rendezvous.stop() + await libp2p.stop() + })) + + it.skip('should not discover replicated peers?', () => { + // TODO + }) + + it('discover find registered peer for namespace only when registered', async () => { + await connectPeers(peers[0], peers[1]) + await connectPeers(peers[2], peers[1]) + + const registers = [] + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + // Peer0 register itself on namespace (connected to Peer1) + await peers[0].rendezvous.register(namespace) + + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of peers[2].rendezvous.discover(namespace)) { + registers.push(reg) + } + expect(registers).to.have.lengthOf(1) + expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) + expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].ns).to.eql(namespace) + expect(registers[0].ttl).to.exist() + + // Peer0 unregister itself on namespace (connected to Peer1) + await peers[0].rendezvous.unregister(namespace) + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + }) + + it('discovers locally first, and if limit achieved, not go to the network', async () => { + + }) + }) + + describe('3 rendezvous, one acting as rendezvous point', () => { + + }) +}) diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js new file mode 100644 index 0000000..d9608fb --- /dev/null +++ b/test/rendezvous.spec.js @@ -0,0 +1,224 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai +const sinon = require('sinon') +const pWaitFor = require('p-wait-for') + +const multiaddr = require('multiaddr') + +const Rendezvous = require('../src') +const { codes: errCodes } = require('../src/errors') + +const { createPeer } = require('./utils') +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + +const namespace = 'ns' + +describe('rendezvous', () => { + describe('start and stop', () => { + let peer, rendezvous + + beforeEach(async () => { + [peer] = await createPeer() + rendezvous = new Rendezvous({ libp2p: peer }) + }) + + afterEach(async () => { + await peer.stop() + await rendezvous.stop() + }) + + it('can be started and stopped', async () => { + const spyRegister = sinon.spy(peer.registrar, 'register') + const spyUnregister = sinon.spy(peer.registrar, 'unregister') + + await rendezvous.start() + await rendezvous.stop() + + expect(spyRegister).to.have.property('callCount', 1) + expect(spyUnregister).to.have.property('callCount', 1) + }) + + it('registers the protocol once, if multiple starts', async () => { + const spyRegister = sinon.spy(peer.registrar, 'register') + + await rendezvous.start() + await rendezvous.start() + + expect(spyRegister).to.have.property('callCount', 1) + + await rendezvous.stop() + }) + + it('only unregisters on stop if already started', async () => { + const spyUnregister = sinon.spy(peer.registrar, 'unregister') + + await rendezvous.stop() + + expect(spyUnregister).to.have.property('callCount', 0) + }) + }) + + describe('api', () => { + let peers + + const connectPeers = async (peer, otherPeer) => { + // Connect to testing relay node + await peer.dial(relayAddr) + await otherPeer.dial(relayAddr) + + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) + await peer.dial(m) + + // Wait event propagation + await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) + } + + beforeEach(async () => { + peers = await createPeer({ number: 3 }) + + // Create 3 rendezvous peers + // Peer0 will not be a server + peers.forEach((peer, index) => { + const rendezvous = new Rendezvous({ + libp2p: peer, + options: { + isServer: index !== 0 + } + }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + }) + + afterEach(async () => { + for (const peer of peers) { + await peer.rendezvous.stop() + await peer.stop() + } + }) + + it('register throws error if a namespace is not provided', async () => { + await expect(peers[0].rendezvous.register()) + .to.eventually.rejected() + .and.have.property('code', errCodes.INVALID_NAMESPACE) + }) + + it('register throws error if ttl is too small', async () => { + await expect(peers[0].rendezvous.register(namespace, 10)) + .to.eventually.rejected() + .and.have.property('code', errCodes.INVALID_TTL) + }) + + it('register throws error if no connected rendezvous servers', async () => { + await expect(peers[0].rendezvous.register(namespace)) + .to.eventually.rejected() + .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + }) + + it('register to a connected rendezvous server node', async () => { + await connectPeers(peers[0], peers[1]) + + // Register + expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + await peers[0].rendezvous.register(namespace) + + expect(peers[1].rendezvous._server.registrations.size).to.eql(1) + expect(peers[1].rendezvous._server.registrations.get(namespace)).to.exist() + + await peers[1].rendezvous.stop() + await peers[1].stop() + }) + + it('unregister throws if a namespace is not provided', async () => { + await expect(peers[0].rendezvous.unregister()) + .to.eventually.rejected() + .and.have.property('code', errCodes.INVALID_NAMESPACE) + }) + + it('register throws error if no connected rendezvous servers', async () => { + await expect(peers[0].rendezvous.unregister(namespace)) + .to.eventually.rejected() + .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + }) + + it('unregister to a connected rendezvous server node', async () => { + await connectPeers(peers[0], peers[1]) + + // Register + expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + await peers[0].rendezvous.register(namespace) + + expect(peers[1].rendezvous._server.registrations.size).to.eql(1) + expect(peers[1].rendezvous._server.registrations.get(namespace)).to.exist() + + // Unregister + await peers[0].rendezvous.unregister(namespace) + expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + + await peers[1].rendezvous.stop() + await peers[1].stop() + }) + + it('unregister to a connected rendezvous server node not fails if not registered', async () => { + await connectPeers(peers[0], peers[1]) + + // Unregister + await peers[0].rendezvous.unregister(namespace) + + await peers[1].rendezvous.stop() + }) + + it('discover throws error if a namespace is not provided', async () => { + try { + for await (const _ of peers[0].rendezvous.discover()) {} // eslint-disable-line + } catch (err) { + expect(err).to.exist() + expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + return + } + throw new Error('discover should throw error if a namespace is not provided') + }) + + it('discover does not find any register if there is none', async () => { + await connectPeers(peers[0], peers[1]) + + for await (const reg of peers[0].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + await peers[1].rendezvous.stop() + }) + + it('discover find registered peer for namespace', async () => { + await connectPeers(peers[0], peers[1]) + await connectPeers(peers[2], peers[1]) + + const registers = [] + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + // Peer0 register itself on namespace (connected to Peer1) + await peers[0].rendezvous.register(namespace) + + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of peers[2].rendezvous.discover(namespace)) { + registers.push(reg) + } + expect(registers).to.have.lengthOf(1) + expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) + expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].ns).to.eql(namespace) + expect(registers[0].ttl).to.exist() + }) + }) +}) diff --git a/test/server.id.json b/test/server.id.json deleted file mode 100644 index 51bad08..0000000 --- a/test/server.id.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "id": "QmQ4eanHt2D1ye44ebGH9RB5XTrzMcZENsjdR4Zd2ELTig", - "privKey": "CAASqAkwggSkAgEAAoIBAQC9fPF9cVj8qtRJa57bmRfkb77ViZRG926fDQTAfzX9tICD3hiHYZLD/tb/0cr7z3Y2amZWrSyuCG3pkhEASk6bb6eND134EWH5wUPxGvWKaw1SmndlGUL8xy/EokH17ieoV1s2fGZ0V6GIeh9/z5REQ6rVNhNy3UOnEm2HcDQn9tmT5tALLbuIgvNA6otFq67/tB6PSiC4mP40kgUzO22E+n20f5HntSkWAS1WCsqL4nhRhKGToRIojpgzlEn6EkE45VkyGQvsgTxNAPGaSMHfS8+J4L8LK2nqTgQkrkUcpOAlg3gATaDHTJJyY2hiVAkAuJb8f+hi73f+cSmpbb9BAgMBAAECggEASR8P6YJ1/nrFlNeM49z+FU7x62E98OzGqWXSsZ3lbdPbzAdGm+eRRUTwHqQMmoOCcJk6iLQnC7mBAKM3IE+Mafr6Qzrs3i+HCWQFHeNzYUjSSVAGRuMqsHUE//JFVevjLdkX/7ydpMO0OAA4a4/k/TrHj6NgefDcjHpV/e/UkJ7MOsjN2QWpx2a4rvOHBDK12eNM9T99zw3MUfNDbw5BGPno9mqGN5uVP1csAZnzVLP3G9utr18veBxf06PZrdLnIakD9oMoXaNwrrQj0v1CQuVQO7tuYcm1r/SzkqGEslX8AmaOb1tiugod5n9dsShpfLrZiS91k8lfWFMgVIBw4QKBgQD4d8PF61VnbZaK4PUJXYrvgXY37tP8NkSrSZoJdpL4Y94FSdd7DHTR/CdNOyqqCXAhSgU6PC21+CqUMAxRfrvoe80vhkMNsBBQRIXUD1EFB20XOLlu+VNngMNWweDsRzwa/zkYOZViG2h8db0XYY2ST4G0CTJNniQO/LD6sa9hVQKBgQDDO3dF81njsXtME08IJ7iBVZ5MvOGSrX78nYPSqALZwWwivF8J0TY1gUJTqBDOoav61aqXWjxTRJ3/mab5DUyJzqU8Ho3KPQzoXgdT+HuppjaDBWt6IHW5eOyRQH7skGL5/EvdIpgTDIhlbDU5nVR6FPV4w6LPB3PHFtI3HC1WPQKBgQCv5zINY38R+w6SAZLYb4YV64SDMqyXKOBSl4fa3TxNZ35eJhnMPlRR+P7l+VZKDOZ6Wsn6oXIHGssiICYIZ/2mKEdqNtYv0Y6rFOfd6n4EXm6H+xukihTW+NzSBe4zuHa/8iI8mT+9tgOx4TTeYaz1gR4lFEGtm6CRj6nHwZWVBQKBgQCc/EIqU0XimyJDx/ry2c244fnKRs8zvKKxyo7nYwX3x1qGi+X35OysFWYaEriBDutVZV4pGfwMEM7jatAiz5jN7wZa0068Yl7wsjs+QD5f6jFHJaKIr3U6UIwZOD1XR7ruvPrbtCeImblLpLkfvOzixduk4dsWki1811Lt0ZB7GQKBgA+JFfb6aF+dnBub/sbogB0CGE7h1gHkZ01QlSuS2r7c9KKzyNahQhcv3BOXJaUEV3Atpkwca+P9cRtE72YgrjLeEbmKtlLVeMwO79bmLDsDm3oQlJlTowGnPmTU8QpMrtqxP41/y6t8VJvAmJiFd3Efq2Ojww7/u3IhecNd0hBo", - "pubKey": "CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC9fPF9cVj8qtRJa57bmRfkb77ViZRG926fDQTAfzX9tICD3hiHYZLD/tb/0cr7z3Y2amZWrSyuCG3pkhEASk6bb6eND134EWH5wUPxGvWKaw1SmndlGUL8xy/EokH17ieoV1s2fGZ0V6GIeh9/z5REQ6rVNhNy3UOnEm2HcDQn9tmT5tALLbuIgvNA6otFq67/tB6PSiC4mP40kgUzO22E+n20f5HntSkWAS1WCsqL4nhRhKGToRIojpgzlEn6EkE45VkyGQvsgTxNAPGaSMHfS8+J4L8LK2nqTgQkrkUcpOAlg3gATaDHTJJyY2hiVAkAuJb8f+hi73f+cSmpbb9BAgMBAAE=" -} diff --git a/test/utils.js b/test/utils.js index 177aef9..ffb5e38 100644 --- a/test/utils.js +++ b/test/utils.js @@ -1,87 +1,51 @@ 'use strict' -const Libp2p = require('libp2p') -const TCP = require('libp2p-tcp') -const MPLEX = require('libp2p-mplex') -const SPDY = require('libp2p-spdy') -const SECIO = require('libp2p-secio') - -const Id = require('peer-id') -const Peer = require('peer-info') - -const Server = require('../src/server') -const Client = require('../src') +const Transport = require('libp2p-websockets') +const Muxer = require('libp2p-mplex') +const { NOISE: Crypto } = require('libp2p-noise') +const PeerId = require('peer-id') -const Utils = module.exports = (id, addrs, cb) => { - Id.createFromJSON(require(id), (err, id) => { - if (err) return cb(err) - const peer = new Peer(id) - addrs.forEach(a => peer.multiaddrs.add(a)) +const pTimes = require('p-times') - const swarm = new Libp2p({ - transport: [ - new TCP() - ], - connection: { - muxer: [ - MPLEX, - SPDY - ], - crypto: [SECIO] - } - }, peer, null, { - relay: { - enabled: true, - hop: { - enabled: true, - active: false - } - } - }) - - swarm.start(err => { - if (err) return cb(err) - cb(null, swarm) - }) - }) -} - -Utils.id = (id, addrs, cb) => { - Id.createFromJSON(require(id), (err, id) => { - if (err) return cb(err) - const peer = new Peer(id) - addrs.forEach(a => peer.multiaddrs.add(a)) - cb(null, peer) - }) -} - -Utils.createServer = (id, addrs, opt, cb) => { - Utils(id, addrs, (err, swarm) => { - if (err) return cb(err) - const server = new Server(Object.assign(opt || {}, {node: swarm})) - server.start() - return cb(null, server, swarm) - }) +const Libp2p = require('libp2p') +const multiaddr = require('multiaddr') + +const Peers = require('./fixtures/peers') +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + +const defaultConfig = { + modules: { + transport: [Transport], + streamMuxer: [Muxer], + connEncryption: [Crypto] + } } -Utils.createClient = (id, addrs, cb) => { - Utils(id, addrs, (err, swarm) => { - if (err) return cb(err) - const client = new Client(swarm) - client.start(err => { - if (err) return cb(err) - return cb(null, client, swarm) - }) - }) +/** + * Create libp2p nodes. + * @param {Object} [properties] + * @param {Object} [properties.config] + * @param {number} [properties.number] number of peers (default: 1). + * @param {boolean} [properties.started] nodes should start (default: true) + * @return {Promise>} + */ +async function createPeer ({ number = 1, started = true, config = {} } = {}) { + const peerIds = await pTimes(number, (i) => PeerId.createFromJSON(Peers[i])) + const peers = await pTimes(number, (i) => Libp2p.create({ + peerId: peerIds[i], + addresses: { + listen: [multiaddr(`${relayAddr}/p2p-circuit`)] + }, + ...defaultConfig, + ...config + })) + + if (started) { + await Promise.all(peers.map((p) => p.start())) + } + + return peers } -Utils.default = cb => Utils.createServer('./server.id.json', ['/ip4/0.0.0.0/tcp/0'], {}, (err, server) => { - if (err) return cb(err) - Utils.createClient('./client.id.json', ['/ip4/0.0.0.0/tcp/0'], (err, client) => { - if (err) return cb(err) - Utils.createClient('./client2.id.json', ['/ip4/0.0.0.0/tcp/0'], (err, client2) => { - if (err) return cb(err) - return cb(null, client, server, client2) - }) - }) -}) +module.exports.createPeer = createPeer From d7290df03437d1a446da54c057d5ebca490f2dfd Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 10 Jul 2020 18:13:45 +0200 Subject: [PATCH 02/38] chore: interface-peer-discovery compliance --- LIBP2P.md | 1 - package.json | 2 + src/discovery.js | 71 ++++++++++++++++++++ src/index.js | 50 ++++++++++++-- test/client-mode.spec.js | 4 +- test/discovery.spec.js | 138 +++++++++++++++++++++++++++++++++++++++ test/rendezvous.spec.js | 26 ++------ test/utils.js | 16 +++++ 8 files changed, 280 insertions(+), 28 deletions(-) create mode 100644 src/discovery.js create mode 100644 test/discovery.spec.js diff --git a/LIBP2P.md b/LIBP2P.md index f3cf41e..a034d88 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -7,7 +7,6 @@ The rendezvous protocol can be used in different contexts across libp2p. For usi `js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows to enable the rendezvous protocol, as well as its server mode, enable automatic peer discover and to specify the topics to register from startup. The rendezvous comes with a discovery service that enables libp2p to automatically discover other peers in the provided namespaces and eventually connect to them. -**TODO: it should be compliant with the peer-discovery interface and configured as any other discovery service instead!!** You can configure it through libp2p as follows: diff --git a/package.json b/package.json index 6218213..8e8ac35 100644 --- a/package.json +++ b/package.json @@ -54,11 +54,13 @@ "aegir": "^23.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", + "delay": "^4.3.0", "dirty-chai": "^2.0.1", "libp2p": "^0.28.3", "libp2p-mplex": "^0.9.5", "libp2p-noise": "^1.1.2", "libp2p-websockets": "^0.13.6", + "p-defer": "^3.0.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", "sinon": "^9.0.2" diff --git a/src/discovery.js b/src/discovery.js new file mode 100644 index 0000000..2d2100d --- /dev/null +++ b/src/discovery.js @@ -0,0 +1,71 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:redezvous:discovery') +log.error = debug('libp2p:redezvous:discovery:error') + +const { EventEmitter } = require('events') + +const defaultOptions = { + interval: 5000 +} + +/** + * Libp2p Rendezvous discovery service. + */ +class Discovery extends EventEmitter { + /** + * @constructor + * @param {Rendezvous} rendezvous + * @param {Object} [options] + * @param {number} [options.interval = 5000] + */ + constructor (rendezvous, options = {}) { + super() + this._rendezvous = rendezvous + this._options = { + ...defaultOptions, + ...options + } + this._interval = undefined + } + + /** + * Start discovery service. + * @returns {void} + */ + start () { + if (this._interval) { + return + } + + this._interval = setInterval(() => this._discover(), this._options.interval) + } + + /** + * Stop discovery service. + * @returns {void} + */ + stop () { + clearInterval(this._interval) + this._interval = null + } + + /** + * Iterates over the registered namespaces and tries to discover new peers + * @returns {void} + */ + _discover () { + this._rendezvous._namespaces.forEach(async (ns) => { + for await (const reg of this._rendezvous.discover(ns)) { + // TODO: interface-peer-discovery with signedPeerRecord + this.emit('peer', { + id: reg.id, + multiaddrs: reg.multiaddrs + }) + } + }) + } +} + +module.exports = Discovery diff --git a/src/index.js b/src/index.js index eea845d..b6e1017 100644 --- a/src/index.js +++ b/src/index.js @@ -14,12 +14,17 @@ const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-to const multiaddr = require('multiaddr') const PeerId = require('peer-id') +const Discovery = require('./discovery') const Server = require('./server') const { codes: errCodes } = require('./errors') const { PROTOCOL_MULTICODEC } = require('./constants') const { Message } = require('./proto') const MESSAGE_TYPE = Message.MessageType +const defaultServerOptions = { + enabled: true +} + /** * Libp2p Rendezvous. * A lightweight mechanism for generalized peer discovery. @@ -30,20 +35,32 @@ class Rendezvous { * @param {object} params * @param {Libp2p} params.libp2p * @param {object} params.options - * @param {boolean} [params.options.isServer = true] + * @param {Array} [params.namespaces = []] + * @param {object} [params.discovery] + * @param {number} [params.discovery.interval = 5000] + * @param {object} [params.server] + * @param {boolean} [params.server.enabled = true] */ - constructor ({ libp2p, options = { isServer: true } }) { + constructor ({ libp2p, options = {} }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._registrar = libp2p.registrar - this._options = options - this._server = undefined + + this._namespaces = options.namespaces || [] + this.discovery = new Discovery(this, options.discovery) + + this._serverOptions = { + ...defaultServerOptions, + ...options.server || {} + } /** * @type {Map} */ this._rendezvousConns = new Map() + this._server = undefined + this._registrarId = undefined this._onPeerConnected = this._onPeerConnected.bind(this) this._onPeerDisconnected = this._onPeerDisconnected.bind(this) @@ -61,7 +78,7 @@ class Rendezvous { log('starting') // Create Rendezvous point if enabled - if (this._options.isServer) { + if (this._serverOptions.enabled) { this._server = new Server({ registrar: this._registrar }) } @@ -76,6 +93,8 @@ class Rendezvous { this._registrarId = await this._registrar.register(topology) log('started') + + this._keepRegistrations() } /** @@ -89,6 +108,7 @@ class Rendezvous { log('stopping') + clearInterval(this._interval) // unregister protocol and handlers await this._registrar.unregister(this._registrarId) @@ -96,6 +116,25 @@ class Rendezvous { log('stopped') } + _keepRegistrations () { + const register = () => { + if (!this._rendezvousConns.size) { + return + } + + const promises = [] + + this._namespaces.forEach((ns) => { + promises.push(this.register(ns)) + }) + + return Promise.all(promises) + } + + register() + this._interval = setInterval(register, 1000) + } + /** * Registrar notifies a connection successfully with rendezvous protocol. * @private @@ -317,4 +356,5 @@ class Rendezvous { } } +Rendezvous.tag = 'rendezvous' module.exports = Rendezvous diff --git a/test/client-mode.spec.js b/test/client-mode.spec.js index 1dcb441..4403fc3 100644 --- a/test/client-mode.spec.js +++ b/test/client-mode.spec.js @@ -35,7 +35,9 @@ describe('client mode', () => { rendezvous = new Rendezvous({ libp2p: peer, options: { - isRendezvousPoint: false + server: { + enabled: false + } } }) diff --git a/test/discovery.spec.js b/test/discovery.spec.js new file mode 100644 index 0000000..12e354b --- /dev/null +++ b/test/discovery.spec.js @@ -0,0 +1,138 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai + +const delay = require('delay') +const pDefer = require('p-defer') +const testsDiscovery = require('libp2p-interfaces/src/peer-discovery/tests') + +const Rendezvous = require('../src') + +const { createPeer, connectPeers } = require('./utils') + +describe('rendezvous discovery', () => { + let peers + + // Create 3 rendezvous peers + // Peer0 will be a server + beforeEach(async () => { + peers = await createPeer({ number: 3 }) + + peers.forEach((peer, index) => { + const rendezvous = new Rendezvous({ + libp2p: peer, + options: { + discovery: { + interval: 1000 + }, + server: { + enabled: index === 0 + } + } + }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + }) + + // Connect rendezvous clients to server + beforeEach(async () => { + await connectPeers(peers[1], peers[0]) + await connectPeers(peers[2], peers[0]) + + expect(peers[0].rendezvous._rendezvousConns.size).to.eql(0) + expect(peers[1].rendezvous._rendezvousConns.size).to.eql(1) + expect(peers[2].rendezvous._rendezvousConns.size).to.eql(1) + }) + + afterEach(async () => { + for (const peer of peers) { + peer.rendezvous.discovery.stop() + await peer.rendezvous.stop() + await peer.stop() + } + }) + + it('peer1 should discover peer2 once it registers to the same namespace', async () => { + const defer = pDefer() + const namespace = 'test-namespace' + peers[1].rendezvous._namespaces = [namespace] + + // Start discovery + peers[1].rendezvous.discovery.once('peer', (peer) => { + expect(peer.id.equals(peers[2].peerId)).to.be.true() + expect(peer.multiaddrs).to.eql(peers[2].multiaddrs) + defer.resolve() + }) + peers[1].rendezvous.discovery.start() + + // Register + expect(peers[0].rendezvous._server.registrations.size).to.eql(0) + await peers[2].rendezvous.register(namespace) + expect(peers[0].rendezvous._server.registrations.size).to.eql(1) + + await defer.promise + }) + + it.skip('peer1 should not discover peer2 if it registers in a different namespace', async () => { + const namespace1 = 'test-namespace1' + const namespace2 = 'test-namespace2' + await peers[1].rendezvous.register(namespace1) + + // Start discovery + peers[1].rendezvous.discovery.once('peer', () => { + throw new Error('no peer should be discovered') + }) + peers[1].rendezvous.discovery.start() + + // Register + expect(peers[0].rendezvous._server.registrations.size).to.eql(0) + await peers[2].rendezvous.register(namespace2) + expect(peers[0].rendezvous._server.registrations.size).to.eql(1) + + await delay(1500) + }) +}) + +describe('interface-discovery', () => { + let peers + + beforeEach(async () => { + peers = await createPeer({ number: 2 }) + + peers.forEach((peer, index) => { + const rendezvous = new Rendezvous({ + libp2p: peer, + options: { + discovery: { + interval: 1000 + }, + namespaces: ['test-namespace'], + server: { + enabled: index === 0 + } + } + }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + + await connectPeers(peers[1], peers[0]) + }) + + testsDiscovery({ + setup () { + return peers[1].rendezvous.discovery + }, + teardown () { + return Promise.all(peers.map(async (libp2p) => { + await libp2p.rendezvous.stop() + await libp2p.stop() + })) + } + }) +}) diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index d9608fb..37c1c1a 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -6,16 +6,11 @@ chai.use(require('dirty-chai')) chai.use(require('chai-as-promised')) const { expect } = chai const sinon = require('sinon') -const pWaitFor = require('p-wait-for') - -const multiaddr = require('multiaddr') const Rendezvous = require('../src') const { codes: errCodes } = require('../src/errors') -const { createPeer } = require('./utils') -const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] +const { createPeer, connectPeers } = require('./utils') const namespace = 'ns' @@ -29,8 +24,8 @@ describe('rendezvous', () => { }) afterEach(async () => { - await peer.stop() await rendezvous.stop() + await peer.stop() }) it('can be started and stopped', async () => { @@ -67,19 +62,6 @@ describe('rendezvous', () => { describe('api', () => { let peers - const connectPeers = async (peer, otherPeer) => { - // Connect to testing relay node - await peer.dial(relayAddr) - await otherPeer.dial(relayAddr) - - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) - await peer.dial(m) - - // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) - } - beforeEach(async () => { peers = await createPeer({ number: 3 }) @@ -89,7 +71,9 @@ describe('rendezvous', () => { const rendezvous = new Rendezvous({ libp2p: peer, options: { - isServer: index !== 0 + server: { + enabled: index !== 0 + } } }) rendezvous.start() diff --git a/test/utils.js b/test/utils.js index ffb5e38..1504cd5 100644 --- a/test/utils.js +++ b/test/utils.js @@ -6,6 +6,7 @@ const { NOISE: Crypto } = require('libp2p-noise') const PeerId = require('peer-id') const pTimes = require('p-times') +const pWaitFor = require('p-wait-for') const Libp2p = require('libp2p') const multiaddr = require('multiaddr') @@ -49,3 +50,18 @@ async function createPeer ({ number = 1, started = true, config = {} } = {}) { } module.exports.createPeer = createPeer + +async function connectPeers (peer, otherPeer) { + // Connect to testing relay node + await peer.dial(relayAddr) + await otherPeer.dial(relayAddr) + + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) + await peer.dial(m) + + // Wait event propagation + await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) +} + +module.exports.connectPeers = connectPeers From b0806705ae69e4bd3865b94f9f78f32032190e75 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 13 Jul 2020 13:11:41 +0200 Subject: [PATCH 03/38] feat: garbage collector --- src/constants.js | 2 +- src/index.js | 11 ++- src/server/index.js | 55 +++++++++++++-- test/server.spec.js | 167 ++++++++++++++++++++++++++++++++++++++++++++ test/utils.js | 14 ++++ 5 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 test/server.spec.js diff --git a/src/constants.js b/src/constants.js index 3f063b7..7d509d8 100644 --- a/src/constants.js +++ b/src/constants.js @@ -2,4 +2,4 @@ exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' exports.MAX_NS_LENGTH = 255 // TODO: spec this -exports.MAX_LIMIT = 1000 // TODO: spec this +exports.MAX_LIMIT = 1000 diff --git a/src/index.js b/src/index.js index b6e1017..12ae4c1 100644 --- a/src/index.js +++ b/src/index.js @@ -22,7 +22,8 @@ const { Message } = require('./proto') const MESSAGE_TYPE = Message.MessageType const defaultServerOptions = { - enabled: true + enabled: true, + gcInterval: 3e5 } /** @@ -40,6 +41,7 @@ class Rendezvous { * @param {number} [params.discovery.interval = 5000] * @param {object} [params.server] * @param {boolean} [params.server.enabled = true] + * @param {number} [params.server.gcInterval = 3e5] */ constructor ({ libp2p, options = {} }) { this._libp2p = libp2p @@ -79,7 +81,8 @@ class Rendezvous { // Create Rendezvous point if enabled if (this._serverOptions.enabled) { - this._server = new Server({ registrar: this._registrar }) + this._server = new Server(this._registrar, this._serverOptions) + this._server.start() } // register protocol with topology @@ -109,8 +112,12 @@ class Rendezvous { log('stopping') clearInterval(this._interval) + // unregister protocol and handlers await this._registrar.unregister(this._registrarId) + if (this._serverOptions.enabled) { + this._server.stop() + } this._registrarId = undefined log('stopped') diff --git a/src/server/index.js b/src/server/index.js index 0786ccb..0d4f88e 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -21,24 +21,69 @@ const rpc = require('./rpc') class RendezvousServer { /** * @constructor - * @param {object} params - * @param {Registrar} params.registrar + * @param {Registrar} registrar + * @param {object} options + * @param {number} options.gcInterval */ - constructor ({ registrar }) { + constructor (registrar, { gcInterval = 3e5 } = {}) { this._registrar = registrar + this._gcInterval = gcInterval /** * Registrations per namespace. * @type {Map>} */ this.registrations = new Map() + } + + /** + * Start rendezvous server for handling rendezvous streams and gc. + * @returns {void} + */ + start () { + if (this._interval) { + return + } + + log('starting') + + // Garbage collection + this._interval = setInterval(this._gc, this._gcInterval) // Incoming streams handling this._registrar.handle(PROTOCOL_MULTICODEC, rpc(this)) + + log('started') + } + + /** + * Stops rendezvous server gc and clears registrations + */ + stop () { + clearInterval(this._interval) + this._interval = undefined + this.registrations.clear() + + log('stopped') } - // TODO: Should we have a start method to gv the expired registrations? - // I am removing them on discover, but it should be useful to have a gc too + /** + * Garbage collector to removed outdated registrations. + * @returns {void} + */ + _gc () { + const now = Date.now() + + // Iterate namespaces + this.registrations.forEach((nsRegistrations) => { + // Iterate registrations for namespaces + nsRegistrations.forEach((reg, idStr) => { + if (now >= reg.expiration) { + nsRegistrations.delete(idStr) + } + }) + }) + } /** * Add a peer registration to a namespace. diff --git a/test/server.spec.js b/test/server.spec.js new file mode 100644 index 0000000..da3128a --- /dev/null +++ b/test/server.spec.js @@ -0,0 +1,167 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai + +const delay = require('delay') +const sinon = require('sinon') +const multiaddr = require('multiaddr') + +const RendezvousServer = require('../src/server') + +const { createPeerId } = require('./utils') + +const registrar = { + handle: () => { } +} +const testNamespace = 'test-namespace' +const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')].map((m) => m.buffer) + +describe('rendezvous server', () => { + let rServer + let peerIds + + before(async () => { + peerIds = await createPeerId({ number: 3 }) + }) + + afterEach(() => { + rServer && rServer.stop() + }) + + it('calls registrar handle on start once', () => { + rServer = new RendezvousServer(registrar) + + // Spy for handle + const spyHandle = sinon.spy(registrar, 'handle') + + rServer.start() + expect(spyHandle).to.have.property('callCount', 1) + + rServer.start() + expect(spyHandle).to.have.property('callCount', 1) + }) + + it('can add registrations to multiple namespaces', () => { + const otherNamespace = 'other-namespace' + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + // Add registration for peer 1 in a different namespace + rServer.addRegistration(otherNamespace, peerIds[0], multiaddrs, 1000) + + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + + const testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(2) + + const otherNsRegistrations = rServer.getRegistrations(otherNamespace) + expect(otherNsRegistrations).to.have.lengthOf(1) + }) + + it('should be able to limit registrations to get', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + + let testNsRegistrations = rServer.getRegistrations(testNamespace, 1) + expect(testNsRegistrations).to.have.lengthOf(1) + + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(2) + }) + + it('can remove registrations from a peer in a given namespace', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + + let testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(2) + + // Remove registration for peer0 + rServer.removeRegistration(testNamespace, peerIds[0]) + + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(1) + }) + + it('can remove all registrations from a peer', () => { + const otherNamespace = 'other-namespace' + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + // Add registration for peer 1 in a different namespace + rServer.addRegistration(otherNamespace, peerIds[0], multiaddrs, 1000) + + let testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(1) + + let otherNsRegistrations = rServer.getRegistrations(otherNamespace) + expect(otherNsRegistrations).to.have.lengthOf(1) + + // Remove all registrations for peer0 + rServer.removePeerRegistrations(peerIds[0]) + + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(0) + + otherNsRegistrations = rServer.getRegistrations(otherNamespace) + expect(otherNsRegistrations).to.have.lengthOf(0) + }) + + it('can attempt to remove a registration for a non existent namespace', () => { + const otherNamespace = 'other-namespace' + rServer = new RendezvousServer(registrar) + + rServer.removeRegistration(otherNamespace, peerIds[0]) + }) + + it('can attempt to remove a registration for a non existent peer', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + + let testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(1) + + // Remove registration for peer0 + rServer.removeRegistration(testNamespace, peerIds[1]) + + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(1) + }) + + it('gc expired records', async () => { + rServer = new RendezvousServer(registrar, { gcInterval: 300 }) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 500) + rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + + let testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(2) + + // wait for firt record to be removed + await delay(650) + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(1) + + await delay(400) + testNsRegistrations = rServer.getRegistrations(testNamespace) + expect(testNsRegistrations).to.have.lengthOf(0) + }) +}) diff --git a/test/utils.js b/test/utils.js index 1504cd5..d6db54f 100644 --- a/test/utils.js +++ b/test/utils.js @@ -23,6 +23,20 @@ const defaultConfig = { } } +/** + * Create Perr Id. + * @param {Object} [properties] + * @param {number} [properties.number] number of peers (default: 1). + * @return {Promise>} + */ +async function createPeerId ({ number = 1 }) { + const peerIds = await pTimes(number, (i) => PeerId.createFromJSON(Peers[i])) + + return peerIds +} + +module.exports.createPeerId = createPeerId + /** * Create libp2p nodes. * @param {Object} [properties] From ebb22d196650b111dfedbf500a1d6b2a94dcbda2 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 13 Jul 2020 17:43:39 +0200 Subject: [PATCH 04/38] feat: cookie for discovery --- src/index.js | 97 +++++++++++------ src/server/index.js | 98 +++++++++++------ src/server/rpc/handlers/discover.js | 17 ++- test/connectivity.spec.js | 14 +-- test/discovery.spec.js | 14 +-- test/flows.spec.js | 6 +- test/rendezvous.spec.js | 46 ++++++-- test/server.spec.js | 157 +++++++++++++++++++++++----- test/utils.js | 2 +- 9 files changed, 332 insertions(+), 119 deletions(-) diff --git a/src/index.js b/src/index.js index 12ae4c1..4078a70 100644 --- a/src/index.js +++ b/src/index.js @@ -26,6 +26,14 @@ const defaultServerOptions = { gcInterval: 3e5 } +/** +* Rendezvous point contains the connection to a rendezvous server, as well as, +* the cookies per namespace that the client received. +* @typedef {Object} RendezvousPoint +* @property {Connection} connection +* @property {Map} cookies +*/ + /** * Libp2p Rendezvous. * A lightweight mechanism for generalized peer discovery. @@ -57,9 +65,15 @@ class Rendezvous { } /** - * @type {Map} + * @type {Map} */ - this._rendezvousConns = new Map() + this._rendezvousPoints = new Map() + + /** + * Client cookies per namespace for own server + * @type {Map} + */ + this._cookiesSelf = new Map() this._server = undefined @@ -120,12 +134,19 @@ class Rendezvous { } this._registrarId = undefined + this._rendezvousPoints.clear() + this._cookiesSelf.clear() + log('stopped') } + /** + * Keep registrations updated on servers. + * @returns {void} + */ _keepRegistrations () { const register = () => { - if (!this._rendezvousConns.size) { + if (!this._rendezvousPoints.size) { return } @@ -152,7 +173,7 @@ class Rendezvous { const idB58Str = peerId.toB58String() log('connected', idB58Str) - this._rendezvousConns.set(idB58Str, conn) + this._rendezvousPoints.set(idB58Str, { connection: conn }) } /** @@ -164,7 +185,7 @@ class Rendezvous { const idB58Str = peerId.toB58String() log('disconnected', idB58Str) - this._rendezvousConns.delete(idB58Str) + this._rendezvousPoints.delete(idB58Str) if (this._server) { this._server.removePeerRegistrations(peerId) @@ -196,7 +217,7 @@ class Rendezvous { } // Are there available rendezvous servers? - if (!this._rendezvousConns.size) { + if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } @@ -214,8 +235,8 @@ class Rendezvous { const registerTasks = [] const taskFn = async (id) => { - const conn = this._rendezvousConns.get(id) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + const { connection } = this._rendezvousPoints.get(id) + const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( [message], @@ -235,7 +256,7 @@ class Rendezvous { return recMessage.registerResponse.ttl } - for (const id of this._rendezvousConns.keys()) { + for (const id of this._rendezvousPoints.keys()) { registerTasks.push(taskFn(id)) } @@ -255,7 +276,7 @@ class Rendezvous { } // Are there available rendezvous servers? - if (!this._rendezvousConns.size) { + if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } @@ -269,8 +290,8 @@ class Rendezvous { const unregisterTasks = [] const taskFn = async (id) => { - const conn = this._rendezvousConns.get(id) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + const { connection } = this._rendezvousPoints.get(id) + const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) await pipe( [message], @@ -282,7 +303,7 @@ class Rendezvous { ) } - for (const id of this._rendezvousConns.keys()) { + for (const id of this._rendezvousPoints.keys()) { unregisterTasks.push(taskFn(id)) } @@ -293,12 +314,11 @@ class Rendezvous { * Discover peers registered under a given namespace * @param {string} ns * @param {number} [limit] - * @param {Buffer} [cookie] * @returns {AsyncIterable<{ id: PeerId, multiaddrs: Array, ns: string, ttl: number }>} */ - async * discover (ns, limit, cookie) { + async * discover (ns, limit) { // Are there available rendezvous servers? - if (!this._rendezvousConns.size) { + if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } @@ -311,7 +331,9 @@ class Rendezvous { // Local search if Server if (this._server) { - const localRegistrations = this._server.getRegistrations(ns, limit) + const cookieSelf = this._cookiesSelf.get(ns) + const { cookie: cookieS, registrations: localRegistrations } = this._server.getRegistrations(ns, { limit, cookie: cookieSelf }) + for (const r of localRegistrations) { yield registrationTransformer(r) @@ -320,21 +342,28 @@ class Rendezvous { return } } - } - const message = Message.encode({ - type: MESSAGE_TYPE.DISCOVER, - discover: { - ns, - limit, - cookie - } - }) + // Store cookie self + this._cookiesSelf.set(ns, cookieS) + } - for (const id of this._rendezvousConns.keys()) { - const conn = this._rendezvousConns.get(id) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + // Iterate over all rendezvous points + for (const [id, rp] of this._rendezvousPoints.entries()) { + const rpCookies = rp.cookies || new Map() + + // Check if we have a cookie and encode discover message + const cookie = rpCookies.get(ns) + const message = Message.encode({ + type: MESSAGE_TYPE.DISCOVER, + discover: { + ns, + limit, + cookie: cookie ? Buffer.from(cookie) : undefined + } + }) + // Send discover message and wait for response + const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( [message], lp.encode(), @@ -350,10 +379,18 @@ class Rendezvous { throw new Error('unexpected message received') } + // Iterate over registrations response for (const r of recMessage.discoverResponse.registrations) { - // track registrations and check if already provided + // track registrations yield registrationTransformer(r) + // Store cookie + rpCookies.set(ns, recMessage.discoverResponse.cookie.toString()) + this._rendezvousPoints.set(id, { + connection: rp.connection, + cookies: rpCookies + }) + limit-- if (limit === 0) { return diff --git a/src/server/index.js b/src/server/index.js index 0d4f88e..909a7de 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -10,6 +10,7 @@ const rpc = require('./rpc') /** * Rendezvous registration. * @typedef {Object} Registration +* @property {string} id * @property {PeerId} peerId * @property {Array} addrs * @property {number} expiration @@ -33,7 +34,15 @@ class RendezvousServer { * Registrations per namespace. * @type {Map>} */ - this.registrations = new Map() + this.nsRegistrations = new Map() + + /** + * Registration ids per cookie. + * @type {Map>} + */ + this.cookieRegistrations = new Map() + + this._gc = this._gc.bind(this) } /** @@ -62,7 +71,9 @@ class RendezvousServer { stop () { clearInterval(this._interval) this._interval = undefined - this.registrations.clear() + + this.nsRegistrations.clear() + this.cookieRegistrations.clear() log('stopped') } @@ -73,16 +84,30 @@ class RendezvousServer { */ _gc () { const now = Date.now() + const removedIds = [] // Iterate namespaces - this.registrations.forEach((nsRegistrations) => { + this.nsRegistrations.forEach((nsEntry) => { // Iterate registrations for namespaces - nsRegistrations.forEach((reg, idStr) => { + nsEntry.forEach((reg, idStr) => { if (now >= reg.expiration) { - nsRegistrations.delete(idStr) + nsEntry.delete(idStr) + removedIds.push(reg.id) } }) }) + + // Remove outdated records references from cookies + for (const [key, idSet] of this.cookieRegistrations.entries()) { + const filteredIds = Array.from(idSet).filter((id) => !removedIds.includes(id)) + + if (filteredIds && filteredIds.length) { + this.cookieRegistrations.set(key, filteredIds) + } else { + // Empty + this.cookieRegistrations.delete(key) + } + } } /** @@ -94,15 +119,16 @@ class RendezvousServer { * @returns {void} */ addRegistration (ns, peerId, addrs, ttl) { - const nsRegistrations = this.registrations.get(ns) || new Map() + const nsEntry = this.nsRegistrations.get(ns) || new Map() - nsRegistrations.set(peerId.toB58String(), { + nsEntry.set(peerId.toB58String(), { + id: String(Math.random() + Date.now()), peerId, addrs, expiration: Date.now() + ttl }) - this.registrations.set(ns, nsRegistrations) + this.nsRegistrations.set(ns, nsEntry) } /** @@ -112,14 +138,14 @@ class RendezvousServer { * @returns {void} */ removeRegistration (ns, peerId) { - const nsRegistrations = this.registrations.get(ns) + const nsEntry = this.nsRegistrations.get(ns) - if (nsRegistrations) { - nsRegistrations.delete(peerId.toB58String()) + if (nsEntry) { + nsEntry.delete(peerId.toB58String()) // Remove registrations map to namespace if empty - if (!nsRegistrations.size) { - this.registrations.delete(ns) + if (!nsEntry.size) { + this.nsRegistrations.delete(ns) } log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) } @@ -131,12 +157,12 @@ class RendezvousServer { * @returns {void} */ removePeerRegistrations (peerId) { - for (const [ns, reg] of this.registrations.entries()) { + for (const [ns, reg] of this.nsRegistrations.entries()) { reg.delete(peerId.toB58String()) // Remove registrations map to namespace if empty if (!reg.size) { - this.registrations.delete(ns) + this.nsRegistrations.delete(ns) } } @@ -146,35 +172,45 @@ class RendezvousServer { /** * Get registrations for a namespace * @param {string} ns - * @param {number} limit - * @returns {Array} + * @param {object} [options] + * @param {number} [options.limit] + * @param {string} [options.cookie] + * @returns {{ registrations: Array, cookie: string }} */ - getRegistrations (ns, limit = MAX_LIMIT) { - const nsRegistrations = this.registrations.get(ns) || new Map() + getRegistrations (ns, { limit = MAX_LIMIT, cookie = String(Math.random() + Date.now()) } = {}) { + const nsEntry = this.nsRegistrations.get(ns) || new Map() const registrations = [] + const cRegistrations = this.cookieRegistrations.get(cookie) || new Set() - for (const [idStr, reg] of nsRegistrations.entries()) { + for (const [idStr, reg] of nsEntry.entries()) { if (reg.expiration <= Date.now()) { - // Clean outdated registration - nsRegistrations.delete(idStr) + // Clean outdated registration from registrations and cookie record + nsEntry.delete(idStr) + cRegistrations.delete(reg.id) continue } - registrations.push({ - ns, - peer: { - id: reg.peerId.toBytes(), - addrs: reg.addrs - }, - ttl: reg.expiration - Date.now() - }) + // If this record was already sent, continue + if (cRegistrations.has(reg.id)) { + continue + } + + cRegistrations.add(reg.id) + registrations.push(reg) // Stop if reached limit if (registrations.length === limit) { break } } - return registrations + + // Save cookie registrations + this.cookieRegistrations.set(cookie, cRegistrations) + + return { + registrations, + cookie + } } } diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index a82ae7f..eb20cf9 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -40,13 +40,24 @@ module.exports = (rendezvousPoint) => { } // Get registrations - const registrations = rendezvousPoint.getRegistrations(msg.discover.ns, msg.discover.limit) + const options = { + cookie: msg.discover.cookie ? msg.discover.cookie.toString() : undefined, + limit: msg.discover.limit + } + const { registrations, cookie } = rendezvousPoint.getRegistrations(msg.discover.ns, options) return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, discoverResponse: { - cookie: undefined, // TODO - registrations, + cookie: Buffer.from(cookie), + registrations: registrations.map((r) => ({ + ns: msg.discover.ns, + peer: { + id: r.peerId.toBytes(), + addrs: r.addrs + }, + ttl: r.expiration - Date.now() + })), status: RESPONSE_STATUS.OK } } diff --git a/test/connectivity.spec.js b/test/connectivity.spec.js index 0314f79..9f34576 100644 --- a/test/connectivity.spec.js +++ b/test/connectivity.spec.js @@ -40,8 +40,8 @@ describe('connectivity', () => { })) it('updates known rendezvous points', async () => { - expect(peers[0].rendezvous._rendezvousConns.size).to.equal(0) - expect(peers[1].rendezvous._rendezvousConns.size).to.equal(0) + expect(peers[0].rendezvous._rendezvousPoints.size).to.equal(0) + expect(peers[1].rendezvous._rendezvousPoints.size).to.equal(0) // Connect each other via relay node const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peers[1].peerId.toB58String()}`) @@ -53,15 +53,15 @@ describe('connectivity', () => { // Wait event propagation // Relay peer is not with rendezvous enabled await pWaitFor(() => - peers[0].rendezvous._rendezvousConns.size === 1 && - peers[1].rendezvous._rendezvousConns.size === 1) + peers[0].rendezvous._rendezvousPoints.size === 1 && + peers[1].rendezvous._rendezvousPoints.size === 1) - expect(peers[0].rendezvous._rendezvousConns.get(peers[1].peerId.toB58String())).to.exist() - expect(peers[1].rendezvous._rendezvousConns.get(peers[0].peerId.toB58String())).to.exist() + expect(peers[0].rendezvous._rendezvousPoints.get(peers[1].peerId.toB58String())).to.exist() + expect(peers[1].rendezvous._rendezvousPoints.get(peers[0].peerId.toB58String())).to.exist() await connection.close() // Wait event propagation - await pWaitFor(() => peers[0].rendezvous._rendezvousConns.size === 0) + await pWaitFor(() => peers[0].rendezvous._rendezvousPoints.size === 0) }) }) diff --git a/test/discovery.spec.js b/test/discovery.spec.js index 12e354b..0789773 100644 --- a/test/discovery.spec.js +++ b/test/discovery.spec.js @@ -44,9 +44,9 @@ describe('rendezvous discovery', () => { await connectPeers(peers[1], peers[0]) await connectPeers(peers[2], peers[0]) - expect(peers[0].rendezvous._rendezvousConns.size).to.eql(0) - expect(peers[1].rendezvous._rendezvousConns.size).to.eql(1) - expect(peers[2].rendezvous._rendezvousConns.size).to.eql(1) + expect(peers[0].rendezvous._rendezvousPoints.size).to.eql(0) + expect(peers[1].rendezvous._rendezvousPoints.size).to.eql(1) + expect(peers[2].rendezvous._rendezvousPoints.size).to.eql(1) }) afterEach(async () => { @@ -71,9 +71,9 @@ describe('rendezvous discovery', () => { peers[1].rendezvous.discovery.start() // Register - expect(peers[0].rendezvous._server.registrations.size).to.eql(0) + expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[2].rendezvous.register(namespace) - expect(peers[0].rendezvous._server.registrations.size).to.eql(1) + expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(1) await defer.promise }) @@ -90,9 +90,9 @@ describe('rendezvous discovery', () => { peers[1].rendezvous.discovery.start() // Register - expect(peers[0].rendezvous._server.registrations.size).to.eql(0) + expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[2].rendezvous.register(namespace2) - expect(peers[0].rendezvous._server.registrations.size).to.eql(1) + expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(1) await delay(1500) }) diff --git a/test/flows.spec.js b/test/flows.spec.js index 3a32042..cbacc4d 100644 --- a/test/flows.spec.js +++ b/test/flows.spec.js @@ -26,7 +26,7 @@ describe('flows', () => { await peer.dial(m) // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) + await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) } beforeEach(async () => { @@ -53,10 +53,6 @@ describe('flows', () => { await libp2p.stop() })) - it.skip('should not discover replicated peers?', () => { - // TODO - }) - it('discover find registered peer for namespace only when registered', async () => { await connectPeers(peers[0], peers[1]) await connectPeers(peers[2], peers[1]) diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index 37c1c1a..da511f6 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -110,11 +110,11 @@ describe('rendezvous', () => { await connectPeers(peers[0], peers[1]) // Register - expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[0].rendezvous.register(namespace) - expect(peers[1].rendezvous._server.registrations.size).to.eql(1) - expect(peers[1].rendezvous._server.registrations.get(namespace)).to.exist() + expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(1) + expect(peers[1].rendezvous._server.nsRegistrations.get(namespace)).to.exist() await peers[1].rendezvous.stop() await peers[1].stop() @@ -136,15 +136,15 @@ describe('rendezvous', () => { await connectPeers(peers[0], peers[1]) // Register - expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[0].rendezvous.register(namespace) - expect(peers[1].rendezvous._server.registrations.size).to.eql(1) - expect(peers[1].rendezvous._server.registrations.get(namespace)).to.exist() + expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(1) + expect(peers[1].rendezvous._server.nsRegistrations.get(namespace)).to.exist() // Unregister await peers[0].rendezvous.unregister(namespace) - expect(peers[1].rendezvous._server.registrations.size).to.eql(0) + expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[1].rendezvous.stop() await peers[1].stop() @@ -204,5 +204,37 @@ describe('rendezvous', () => { expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() }) + + it('discover find registered peer for namespace once (cookie usage)', async () => { + await connectPeers(peers[0], peers[1]) + await connectPeers(peers[2], peers[1]) + + const registers = [] + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + // Peer0 register itself on namespace (connected to Peer1) + await peers[0].rendezvous.register(namespace) + + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of peers[2].rendezvous.discover(namespace)) { + registers.push(reg) + } + + expect(registers).to.have.lengthOf(1) + expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) + expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].ns).to.eql(namespace) + expect(registers[0].ttl).to.exist() + + for await (const reg of peers[2].rendezvous.discover(namespace)) { + registers.push(reg) + } + + expect(registers).to.have.lengthOf(1) + }) }) }) diff --git a/test/server.spec.js b/test/server.spec.js index da3128a..6fa4a82 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -57,10 +57,10 @@ describe('rendezvous server', () => { // Add registration for peer 2 in test namespace rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) - const testNsRegistrations = rServer.getRegistrations(testNamespace) + const { registrations: testNsRegistrations } = rServer.getRegistrations(testNamespace) expect(testNsRegistrations).to.have.lengthOf(2) - const otherNsRegistrations = rServer.getRegistrations(otherNamespace) + const { registrations: otherNsRegistrations } = rServer.getRegistrations(otherNamespace) expect(otherNsRegistrations).to.have.lengthOf(1) }) @@ -72,11 +72,13 @@ describe('rendezvous server', () => { // Add registration for peer 2 in test namespace rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) - let testNsRegistrations = rServer.getRegistrations(testNamespace, 1) - expect(testNsRegistrations).to.have.lengthOf(1) + let r = rServer.getRegistrations(testNamespace, { limit: 1 }) + expect(r.registrations).to.have.lengthOf(1) + expect(r.cookie).to.exist() - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(2) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(2) + expect(r.cookie).to.exist() }) it('can remove registrations from a peer in a given namespace', () => { @@ -87,14 +89,16 @@ describe('rendezvous server', () => { // Add registration for peer 2 in test namespace rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) - let testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(2) + let r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(2) + expect(r.cookie).to.exist() // Remove registration for peer0 rServer.removeRegistration(testNamespace, peerIds[0]) - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(1) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) + expect(r.cookie).to.exist() }) it('can remove all registrations from a peer', () => { @@ -106,20 +110,20 @@ describe('rendezvous server', () => { // Add registration for peer 1 in a different namespace rServer.addRegistration(otherNamespace, peerIds[0], multiaddrs, 1000) - let testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(1) + let r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) - let otherNsRegistrations = rServer.getRegistrations(otherNamespace) - expect(otherNsRegistrations).to.have.lengthOf(1) + let otherR = rServer.getRegistrations(otherNamespace) + expect(otherR.registrations).to.have.lengthOf(1) // Remove all registrations for peer0 rServer.removePeerRegistrations(peerIds[0]) - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(0) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(0) - otherNsRegistrations = rServer.getRegistrations(otherNamespace) - expect(otherNsRegistrations).to.have.lengthOf(0) + otherR = rServer.getRegistrations(otherNamespace) + expect(otherR.registrations).to.have.lengthOf(0) }) it('can attempt to remove a registration for a non existent namespace', () => { @@ -135,14 +139,14 @@ describe('rendezvous server', () => { // Add registration for peer 1 in test namespace rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) - let testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(1) + let r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) // Remove registration for peer0 rServer.removeRegistration(testNamespace, peerIds[1]) - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(1) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) }) it('gc expired records', async () => { @@ -152,16 +156,113 @@ describe('rendezvous server', () => { rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 500) rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) - let testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(2) + let r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(2) // wait for firt record to be removed await delay(650) - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(1) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) await delay(400) - testNsRegistrations = rServer.getRegistrations(testNamespace) - expect(testNsRegistrations).to.have.lengthOf(0) + r = rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(0) + }) + + it('only new peers should be returned if cookie given', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + + // Get current registrations + const { cookie, registrations } = rServer.getRegistrations(testNamespace) + expect(cookie).to.exist() + expect(registrations).to.exist() + expect(registrations).to.have.lengthOf(1) + expect(registrations[0].peerId.toString()).to.eql(peerIds[0].toString()) + + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + + // Get second registration by using the cookie + const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + expect(cookie2).to.exist() + expect(cookie2).to.eql(cookie) + expect(registrations2).to.exist() + expect(registrations2).to.have.lengthOf(1) + expect(registrations2[0].peerId.toString()).to.eql(peerIds[1].toString()) + + // If no cookie provided, all registrations are given + const { registrations: registrations3 } = rServer.getRegistrations(testNamespace) + expect(registrations3).to.exist() + expect(registrations3).to.have.lengthOf(2) + }) + + it('no new peers should be returned if there are not new peers since latest query', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + + // Get current registrations + const { cookie, registrations } = rServer.getRegistrations(testNamespace) + expect(cookie).to.exist() + expect(registrations).to.exist() + expect(registrations).to.have.lengthOf(1) + + // Get registrations with same cookie and no new registration + const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + expect(cookie2).to.exist() + expect(cookie2).to.eql(cookie) + expect(registrations2).to.exist() + expect(registrations2).to.have.lengthOf(0) + }) + + it('new data for a peer should be returned if registration updated', () => { + rServer = new RendezvousServer(registrar) + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + + // Get current registrations + const { cookie, registrations } = rServer.getRegistrations(testNamespace) + expect(cookie).to.exist() + expect(registrations).to.exist() + expect(registrations).to.have.lengthOf(1) + expect(registrations[0].peerId.toString()).to.eql(peerIds[0].toString()) + + // Add new registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + + // Get registrations with same cookie and no new registration + const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + expect(cookie2).to.exist() + expect(cookie2).to.eql(cookie) + expect(registrations2).to.exist() + expect(registrations2).to.have.lengthOf(1) + expect(registrations2[0].peerId.toString()).to.eql(peerIds[0].toString()) + }) + + it('garbage collector should remove cookies of discarded records', async () => { + rServer = new RendezvousServer(registrar, { gcInterval: 300 }) + rServer.start() + + // Add registration for peer 1 in test namespace + rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 500) + + // Get current registrations + const { cookie, registrations } = rServer.getRegistrations(testNamespace) + expect(registrations).to.exist() + expect(registrations).to.have.lengthOf(1) + + // Verify internal state + expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(1) + expect(rServer.cookieRegistrations.get(cookie)).to.exist() + + await delay(800) + + expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(0) + expect(rServer.cookieRegistrations.get(cookie)).to.not.exist() }) }) diff --git a/test/utils.js b/test/utils.js index d6db54f..91b0d77 100644 --- a/test/utils.js +++ b/test/utils.js @@ -75,7 +75,7 @@ async function connectPeers (peer, otherPeer) { await peer.dial(m) // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousConns.size === 1) + await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) } module.exports.connectPeers = connectPeers From 9765a957d6f3ad2d38186e41c3fc3383db9a3f10 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 15 Jul 2020 09:55:28 +0200 Subject: [PATCH 05/38] chore: cleanup --- LIBP2P.md | 20 ++++++-------- README.md | 56 +++++++++++++++++++++++++++++++++++++--- src/discovery.js | 23 ++++++++++++----- src/index.js | 43 ++++++++++++++++-------------- test/client-mode.spec.js | 12 ++++----- test/discovery.spec.js | 26 ++++++++----------- test/rendezvous.spec.js | 8 +++--- 7 files changed, 119 insertions(+), 69 deletions(-) diff --git a/LIBP2P.md b/LIBP2P.md index a034d88..6359b5c 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -1,12 +1,12 @@ # Rendezvous Protocol in js-libp2p -The rendezvous protocol can be used in different contexts across libp2p. For using it, the libp2p network needs to have well known libp2p nodes acting as rendezvous servers. These nodes will have an extra role in the network. They will collect and maintain a list of registrations per rendezvous namespace. Other peers in the network will act as rendezvous clients and will register themselves on given namespaces by messaging a rendezvous server node. Taking into account these registrations, a rendezvous client is able to discover other peers in a given namespace by querying a server. +The rendezvous protocol can be used in different contexts across libp2p. For using it, the libp2p network needs to have well known libp2p nodes acting as rendezvous servers. These nodes will have an extra role in the network. They will collect and maintain a list of registrations per rendezvous namespace. Other peers in the network will act as rendezvous clients and will register themselves on given namespaces by messaging a rendezvous server node. Taking into account these registrations, a rendezvous client is able to discover other peers in a given namespace by querying a server. A registration should have a `ttl`, in order to avoid having invalid registrations. ## Usage -`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows to enable the rendezvous protocol, as well as its server mode, enable automatic peer discover and to specify the topics to register from startup. +`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled, as well as its server mode. In addition, automatic peer discovery can be enabled and namespaces to register can be specified from startup through the config. -The rendezvous comes with a discovery service that enables libp2p to automatically discover other peers in the provided namespaces and eventually connect to them. +The rendezvous implementation also brings a discovery service that enables libp2p to automatically discover other peers in the provided namespaces and eventually connect to them. You can configure it through libp2p as follows: @@ -29,18 +29,14 @@ const node = await Libp2p.create({ }) ``` -While `js-libp2p` supports the rendezvous protocol out of the box, it also provides a rendezvous API that users can interact with. This API should allow users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover other peers. +While `js-libp2p` supports the rendezvous protocol out of the box, it also provides a rendezvous API that users can interact with. This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover peers. ## Libp2p Flow -When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server and ask for nodes in the given namespaces. The rendezvous server can be added to the bootstrap nodes or manually dialed. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. +When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server and ask for nodes in given namespaces (namespaces provided for register). The rendezvous server can be added to the bootstrap nodes or manually dialed. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. -If the discovery service is disabled, the rendezvous API should allow users to discover peers registered on provided namespaces. +If the discovery service is disabled, the rendezvous API also allows users to discover peers registered on provided namespaces. -When a libp2p node running the rendezvous protocol is going to stop, it should unregister from all the namespaces previously registered. +When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. -In the event of a rendezvous client getting connected to a second rendezvous server, it should propagate its registrations to it. The rendezvous server should clean its registrations for a peer when it is not connected with it anymore. - -## Other notes: - -After a query is made, who is responsible for determining if we need more records? (cookie reuse) +In the event of a rendezvous client getting connected to a second rendezvous server, it will propagate its registrations to it. The rendezvous server will aso clean its registrations for a peer when it is not connected with it anymore. diff --git a/README.md b/README.md index 995cc57..541bdf4 100644 --- a/README.md +++ b/README.md @@ -19,18 +19,67 @@ See https://github.com/libp2p/specs/tree/master/rendezvous for more details ## API +### constructor + +Creating an instance of Rendezvous. + +`const rendezvous = new Rendezvous({ libp2p })` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| params | `object` | rendezvous parameters | +| params.libp2p | `Libp2p` | a libp2p node instance | +| params.namespaces | `Array` | namespaces to keep registering and discovering over time (default: `[]`) | +| params.server | `object` | rendezvous server options | +| params.server.enabled | `boolean` | rendezvous server enabled (default: `true`) | +| params.server.gcInterval | `number` | rendezvous garbage collector interval (default: `3e5`) | +| params.discovery | `object` | rendezvous peer discovery options | +| params.discovery.interval | `number` | automatic rendezvous peer discovery interval (default: `5e3`) | + +### rendezvous.start + +Register the rendezvous protocol topology into libp2p and starts its internal services. The rendezvous server will be started if enabled, as well as the service to keep self registrations available. + +`rendezvous.start()` + +When registering to new namespaces from the API, the new namespace will be added to the registrations to keep by default. + +### rendezvous.stop + +Unregister the rendezvous protocol and the streams with other peers will be closed. + +`rendezvous.stop()` + +### rendezvous.discovery.start + +Starts the rendezvous automatic discovery service. + +`rendezvous.discovery.start()` + +Like other libp2p discovery protocols, it will emit `peer` events when new peers are discovered. + +### rendezvous.discovery.stop + +Stops the rendezvous automatic discovery service. + +`rendezvous.discovery.stop()` + ### rendezvous.register Registers the peer in a given namespace. -`rendezvous.register(namespace, [ttl])` +`rendezvous.register(namespace, [options])` #### Parameters | Name | Type | Description | |------|------|-------------| | namespace | `string` | namespace to register | -| ttl | `number` | registration ttl in ms (default: `7200e3` and minimum `120`) | +| options | `object` | rendezvous registrations options | +| options.ttl | `number` | registration ttl in ms (default: `7200e3` and minimum `120`) | +| options.keep | `boolean` | register over time to guarantee availability (default: `true`) | #### Returns @@ -75,7 +124,7 @@ await rendezvous.unregister(namespace) Discovers peers registered under a given namespace. -`rendezvous.discover(namespace, [limit], [cookie])` +`rendezvous.discover(namespace, [limit])` #### Parameters @@ -83,7 +132,6 @@ Discovers peers registered under a given namespace. |------|------|-------------| | namespace | `string` | namespace to discover | | limit | `number` | limit of peers to discover | -| cookie | `Buffer` | | #### Returns diff --git a/src/discovery.js b/src/discovery.js index 2d2100d..b0d1c3a 100644 --- a/src/discovery.js +++ b/src/discovery.js @@ -6,8 +6,10 @@ log.error = debug('libp2p:redezvous:discovery:error') const { EventEmitter } = require('events') +const { codes: errCodes } = require('./errors') + const defaultOptions = { - interval: 5000 + interval: 5e3 } /** @@ -57,12 +59,19 @@ class Discovery extends EventEmitter { */ _discover () { this._rendezvous._namespaces.forEach(async (ns) => { - for await (const reg of this._rendezvous.discover(ns)) { - // TODO: interface-peer-discovery with signedPeerRecord - this.emit('peer', { - id: reg.id, - multiaddrs: reg.multiaddrs - }) + try { + for await (const reg of this._rendezvous.discover(ns)) { + // TODO: interface-peer-discovery with signedPeerRecord + this.emit('peer', { + id: reg.id, + multiaddrs: reg.multiaddrs + }) + } + } catch (err) { + // It will fail while there are no connected rendezvous servers + if (err.code !== errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) { + throw err + } } }) } diff --git a/src/index.js b/src/index.js index 4078a70..0f01512 100644 --- a/src/index.js +++ b/src/index.js @@ -43,25 +43,24 @@ class Rendezvous { * @constructor * @param {object} params * @param {Libp2p} params.libp2p - * @param {object} params.options * @param {Array} [params.namespaces = []] * @param {object} [params.discovery] - * @param {number} [params.discovery.interval = 5000] + * @param {number} [params.discovery.interval = 5e3] * @param {object} [params.server] * @param {boolean} [params.server.enabled = true] * @param {number} [params.server.gcInterval = 3e5] */ - constructor ({ libp2p, options = {} }) { + constructor ({ libp2p, namespaces = [], discovery = {}, server = {} }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._registrar = libp2p.registrar - this._namespaces = options.namespaces || [] - this.discovery = new Discovery(this, options.discovery) + this._namespaces = namespaces + this.discovery = new Discovery(this, discovery) this._serverOptions = { ...defaultServerOptions, - ...options.server || {} + ...server } /** @@ -84,9 +83,9 @@ class Rendezvous { /** * Register the rendezvous protocol in the libp2p node. - * @returns {Promise} + * @returns {void} */ - async start () { + start () { if (this._registrarId) { return } @@ -107,18 +106,17 @@ class Rendezvous { onDisconnect: this._onPeerDisconnected } }) - this._registrarId = await this._registrar.register(topology) - - log('started') + this._registrarId = this._registrar.register(topology) this._keepRegistrations() + log('started') } /** * Unregister the rendezvous protocol and the streams with other peers will be closed. - * @returns {Promise} + * @returns {void} */ - async stop () { + stop () { if (!this._registrarId) { return } @@ -128,7 +126,7 @@ class Rendezvous { clearInterval(this._interval) // unregister protocol and handlers - await this._registrar.unregister(this._registrarId) + this._registrar.unregister(this._registrarId) if (this._serverOptions.enabled) { this._server.stop() } @@ -153,7 +151,7 @@ class Rendezvous { const promises = [] this._namespaces.forEach((ns) => { - promises.push(this.register(ns)) + promises.push(this.register(ns, { keep: false })) }) return Promise.all(promises) @@ -195,10 +193,12 @@ class Rendezvous { /** * Register the peer in a given namespace * @param {string} ns - * @param {number} [ttl = 7200e3] registration ttl in ms (minimum 120) - * @returns {Promise} + * @param {object} [options] + * @param {number} [options.ttl = 7200e3] registration ttl in ms (minimum 120) + * @param {number} [options.keep = true] register over time to guarantee availability. + * @returns {Promise} rendezvous register ttl. */ - async register (ns, ttl = 7200e3) { + async register (ns, { ttl = 7200e3, keep = true } = {}) { if (!ns) { throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) } @@ -253,7 +253,7 @@ class Rendezvous { throw new Error('unexpected message received') } - return recMessage.registerResponse.ttl + return recMessage.registerResponse.ttl // TODO: convert to ms } for (const id of this._rendezvousPoints.keys()) { @@ -262,6 +262,10 @@ class Rendezvous { // Return first ttl const [returnTtl] = await Promise.all(registerTasks) + + // Keep registering if enabled + keep && this._namespaces.push(ns) + return returnTtl } @@ -307,6 +311,7 @@ class Rendezvous { unregisterTasks.push(taskFn(id)) } + this._namespaces.filter((keeptNampesace) => keeptNampesace !== ns) await Promise.all(unregisterTasks) } diff --git a/test/client-mode.spec.js b/test/client-mode.spec.js index 4403fc3..569ea52 100644 --- a/test/client-mode.spec.js +++ b/test/client-mode.spec.js @@ -16,7 +16,7 @@ describe('client mode', () => { afterEach(async () => { peer && await peer.stop() - rendezvous && await rendezvous.stop() + rendezvous && rendezvous.stop() }) it('registers a rendezvous handler by default', async () => { @@ -25,7 +25,7 @@ describe('client mode', () => { const spyHandle = sinon.spy(peer.registrar, '_handle') - await rendezvous.start() + rendezvous.start() expect(spyHandle).to.have.property('callCount', 1) }) @@ -34,16 +34,14 @@ describe('client mode', () => { [peer] = await createPeer() rendezvous = new Rendezvous({ libp2p: peer, - options: { - server: { - enabled: false - } + server: { + enabled: false } }) const spyHandle = sinon.spy(peer.registrar, '_handle') - await rendezvous.start() + rendezvous.start() expect(spyHandle).to.have.property('callCount', 0) }) }) diff --git a/test/discovery.spec.js b/test/discovery.spec.js index 0789773..b2ff1c8 100644 --- a/test/discovery.spec.js +++ b/test/discovery.spec.js @@ -25,13 +25,11 @@ describe('rendezvous discovery', () => { peers.forEach((peer, index) => { const rendezvous = new Rendezvous({ libp2p: peer, - options: { - discovery: { - interval: 1000 - }, - server: { - enabled: index === 0 - } + discovery: { + interval: 1000 + }, + server: { + enabled: index === 0 } }) rendezvous.start() @@ -107,14 +105,12 @@ describe('interface-discovery', () => { peers.forEach((peer, index) => { const rendezvous = new Rendezvous({ libp2p: peer, - options: { - discovery: { - interval: 1000 - }, - namespaces: ['test-namespace'], - server: { - enabled: index === 0 - } + discovery: { + interval: 1000 + }, + namespaces: ['test-namespace'], + server: { + enabled: index === 0 } }) rendezvous.start() diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index da511f6..4d23c5a 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -70,10 +70,8 @@ describe('rendezvous', () => { peers.forEach((peer, index) => { const rendezvous = new Rendezvous({ libp2p: peer, - options: { - server: { - enabled: index !== 0 - } + server: { + enabled: index !== 0 } }) rendezvous.start() @@ -95,7 +93,7 @@ describe('rendezvous', () => { }) it('register throws error if ttl is too small', async () => { - await expect(peers[0].rendezvous.register(namespace, 10)) + await expect(peers[0].rendezvous.register(namespace, { ttl: 10 })) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_TTL) }) From b6edaf336c8369905feac8bdc229e210bd688755 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 17 Jul 2020 18:11:51 +0200 Subject: [PATCH 06/38] chore: update aegir --- .aegir.js | 6 ++++++ package.json | 4 ++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/.aegir.js b/.aegir.js index e78b224..3b43827 100644 --- a/.aegir.js +++ b/.aegir.js @@ -53,5 +53,11 @@ module.exports = { hooks: { pre: before, post: after + }, + webpack: { + node: { + // this is needed until bcrypto stops using node buffers in browser code + Buffer: true + } } } diff --git a/package.json b/package.json index 8e8ac35..9cef77b 100644 --- a/package.json +++ b/package.json @@ -51,12 +51,12 @@ "streaming-iterables": "^4.1.2" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.3.0", "dirty-chai": "^2.0.1", - "libp2p": "^0.28.3", + "libp2p": "libp2p/js-libp2p#feat/certified-addressbook", "libp2p-mplex": "^0.9.5", "libp2p-noise": "^1.1.2", "libp2p-websockets": "^0.13.6", From 0e304f962688dfa322b019ee1facfb500e0ed009 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 17 Jul 2020 18:44:27 +0200 Subject: [PATCH 07/38] chore: convert to seconds in the wire --- src/index.js | 6 +++--- src/server/rpc/handlers/discover.js | 2 +- src/server/rpc/handlers/register.js | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/index.js b/src/index.js index 0f01512..070a4ac 100644 --- a/src/index.js +++ b/src/index.js @@ -229,7 +229,7 @@ class Rendezvous { addrs }, ns, - ttl // TODO: convert to seconds + ttl: ttl * 1e-3 // Convert to seconds } }) @@ -253,7 +253,7 @@ class Rendezvous { throw new Error('unexpected message received') } - return recMessage.registerResponse.ttl // TODO: convert to ms + return recMessage.registerResponse.ttl * 1e3 // convert to ms } for (const id of this._rendezvousPoints.keys()) { @@ -331,7 +331,7 @@ class Rendezvous { id: PeerId.createFromBytes(r.peer.id), multiaddrs: r.peer.addrs && r.peer.addrs.map((a) => multiaddr(a)), ns: r.ns, - ttl: r.ttl + ttl: r.ttl * 1e3 // convert to ms }) // Local search if Server diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index eb20cf9..caf8026 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -56,7 +56,7 @@ module.exports = (rendezvousPoint) => { id: r.peerId.toBytes(), addrs: r.addrs }, - ttl: r.expiration - Date.now() + ttl: (r.expiration - Date.now()) * 1e-3 // convert to seconds })), status: RESPONSE_STATUS.OK } diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index e6ed495..d950626 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -52,7 +52,7 @@ module.exports = (rendezvousPoint) => { msg.register.ns, peerId, msg.register.peer.addrs, - msg.register.ttl + msg.register.ttl * 1e3 // convert to ms ) return { From b2489245f31df9c1a719b15e36e26abb3e9f9efa Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 20 Jul 2020 12:00:11 +0200 Subject: [PATCH 08/38] chore: remove unregister comments for response --- src/server/rpc/handlers/unregister.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js index 936dc77..940b4d5 100644 --- a/src/server/rpc/handlers/unregister.js +++ b/src/server/rpc/handlers/unregister.js @@ -24,7 +24,6 @@ module.exports = (rendezvousPoint) => { if (!msg.unregister.id.equals(peerId.toBytes())) { log.error('unauthorized peer id to unregister') - // TODO: auth validation of peerId? -- there is no answer return } @@ -37,6 +36,5 @@ module.exports = (rendezvousPoint) => { } catch (err) { log.error(err) } - // TODO: internal error? -- there is no answer } } From 47641f7a9b0b6debc5bfae1cdf624c5d6ec339b2 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 22 Jul 2020 16:15:26 +0200 Subject: [PATCH 09/38] feat: use signed peer records to exchange multiaddrs --- README.md | 4 +- package.json | 3 + src/constants.js | 2 +- src/discovery.js | 12 ++- src/errors.js | 1 - src/index.js | 27 ++---- src/proto.js | 10 +-- src/server/index.js | 87 ++++++++++++-------- src/server/rpc/handlers/discover.js | 9 +- src/server/rpc/handlers/register.js | 28 ++++--- test/flows.spec.js | 3 +- test/rendezvous.spec.js | 16 +++- test/server.spec.js | 123 ++++++++++++++++------------ test/utils.js | 15 ++++ 14 files changed, 196 insertions(+), 144 deletions(-) diff --git a/README.md b/README.md index 541bdf4..949444a 100644 --- a/README.md +++ b/README.md @@ -137,7 +137,7 @@ Discovers peers registered under a given namespace. | Type | Description | |------|-------------| -| `AsyncIterable<{ id: PeerId, signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | +| `AsyncIterable<{ signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | #### Example @@ -146,7 +146,7 @@ Discovers peers registered under a given namespace. await rendezvous.register(namespace) for await (const reg of rendezvous.discover(namespace)) { - console.log(reg.id, reg.signedPeerRecord, reg.ns, reg.ttl) + console.log(reg.signedPeerRecord, reg.ns, reg.ttl) } ``` diff --git a/package.json b/package.json index 9cef77b..538fedc 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,9 @@ "protons": "^1.2.0", "streaming-iterables": "^4.1.2" }, + "peerDependencies": { + "libp2p": "libp2p/js-libp2p#feat/certified-addressbook" + }, "devDependencies": { "aegir": "^25.0.0", "chai": "^4.2.0", diff --git a/src/constants.js b/src/constants.js index 7d509d8..c818bf2 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,5 +1,5 @@ 'use strict' exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' -exports.MAX_NS_LENGTH = 255 // TODO: spec this +exports.MAX_NS_LENGTH = 255 exports.MAX_LIMIT = 1000 diff --git a/src/discovery.js b/src/discovery.js index b0d1c3a..831c7d2 100644 --- a/src/discovery.js +++ b/src/discovery.js @@ -6,6 +6,9 @@ log.error = debug('libp2p:redezvous:discovery:error') const { EventEmitter } = require('events') +const Envelope = require('libp2p/src/record/envelope') +const PeerRecord = require('libp2p/src/record/peer-record') + const { codes: errCodes } = require('./errors') const defaultOptions = { @@ -61,10 +64,13 @@ class Discovery extends EventEmitter { this._rendezvous._namespaces.forEach(async (ns) => { try { for await (const reg of this._rendezvous.discover(ns)) { - // TODO: interface-peer-discovery with signedPeerRecord + const envelope = await Envelope.openAndCertify(reg.signedPeerRecord, PeerRecord.DOMAIN) + const rec = PeerRecord.createFromProtobuf(envelope.payload) + + // TODO: interface-peer-discovery with signedPeerRecord instead this.emit('peer', { - id: reg.id, - multiaddrs: reg.multiaddrs + id: envelope.peerId, + multiaddrs: rec.multiaddrs }) } } catch (err) { diff --git a/src/errors.js b/src/errors.js index d130f4b..02328b3 100644 --- a/src/errors.js +++ b/src/errors.js @@ -3,6 +3,5 @@ exports.codes = { INVALID_NAMESPACE: 'ERR_INVALID_NAMESPACE', INVALID_TTL: 'ERR_INVALID_TTL', - INVALID_MULTIADDRS: 'ERR_INVALID_MULTIADDRS', NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS' } diff --git a/src/index.js b/src/index.js index 070a4ac..55ef37c 100644 --- a/src/index.js +++ b/src/index.js @@ -11,8 +11,6 @@ const { collect } = require('streaming-iterables') const { toBuffer } = require('it-buffer') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') -const multiaddr = require('multiaddr') -const PeerId = require('peer-id') const Discovery = require('./discovery') const Server = require('./server') @@ -94,7 +92,7 @@ class Rendezvous { // Create Rendezvous point if enabled if (this._serverOptions.enabled) { - this._server = new Server(this._registrar, this._serverOptions) + this._server = new Server(this._libp2p, this._serverOptions) this._server.start() } @@ -148,6 +146,8 @@ class Rendezvous { return } + log('update current registrations') + const promises = [] this._namespaces.forEach((ns) => { @@ -207,15 +207,6 @@ class Rendezvous { throw errCode(new Error('a valid ttl must be provided (bigger than 120)'), errCodes.INVALID_TTL) } - const addrs = [] - for (const m of this._libp2p.multiaddrs) { - if (!multiaddr.isMultiaddr(m)) { - throw errCode(new Error('one or more of the provided multiaddrs is not valid'), errCodes.INVALID_MULTIADDRS) - } - - addrs.push(m.buffer) - } - // Are there available rendezvous servers? if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) @@ -224,10 +215,7 @@ class Rendezvous { const message = Message.encode({ type: MESSAGE_TYPE.REGISTER, register: { - peer: { - id: this._peerId.toBytes(), - addrs - }, + signedPeerRecord: this._libp2p.peerStore.addressBook.getRawEnvelope(this._peerId), ns, ttl: ttl * 1e-3 // Convert to seconds } @@ -319,7 +307,7 @@ class Rendezvous { * Discover peers registered under a given namespace * @param {string} ns * @param {number} [limit] - * @returns {AsyncIterable<{ id: PeerId, multiaddrs: Array, ns: string, ttl: number }>} + * @returns {AsyncIterable<{ signedPeerRecord: Buffer, ns: string, ttl: number }>} */ async * discover (ns, limit) { // Are there available rendezvous servers? @@ -328,13 +316,12 @@ class Rendezvous { } const registrationTransformer = (r) => ({ - id: PeerId.createFromBytes(r.peer.id), - multiaddrs: r.peer.addrs && r.peer.addrs.map((a) => multiaddr(a)), + signedPeerRecord: r.signedPeerRecord, ns: r.ns, ttl: r.ttl * 1e3 // convert to ms }) - // Local search if Server + // Local search if Server enabled if (this._server) { const cookieSelf = this._cookiesSelf.get(ns) const { cookie: cookieS, registrations: localRegistrations } = this._server.getRegistrations(ns, { limit, cookie: cookieSelf }) diff --git a/src/proto.js b/src/proto.js index ed9574b..dce8fff 100644 --- a/src/proto.js +++ b/src/proto.js @@ -23,14 +23,12 @@ message Message { E_UNAVAILABLE = 400; } - message PeerInfo { - optional bytes id = 1; - repeated bytes addrs = 2; - } - message Register { optional string ns = 1; - optional PeerInfo peer = 2; + // signedPeerRecord contains a serialized SignedEnvelope containing a PeerRecord, + // signed by the sending node. It contains the same addresses as the listenAddrs field, but + // in a form that lets us share authenticated addrs with other peers. + optional bytes signedPeerRecord = 2; optional int64 ttl = 3; // in seconds } diff --git a/src/server/index.js b/src/server/index.js index 909a7de..35b62bf 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -4,35 +4,44 @@ const debug = require('debug') const log = debug('libp2p:redezvous-server') log.error = debug('libp2p:redezvous-server:error') +const PeerId = require('peer-id') + const { PROTOCOL_MULTICODEC, MAX_LIMIT } = require('../constants') const rpc = require('./rpc') /** * Rendezvous registration. -* @typedef {Object} Registration -* @property {string} id -* @property {PeerId} peerId -* @property {Array} addrs -* @property {number} expiration +* @typedef {Object} Register +* @property {string} ns +* @property {Buffer} signedPeerRecord +* @property {number} ttl */ +/** + * Namespace registration. + * @typedef {Object} NamespaceRegistration + * @property {string} id + * @property {number} expiration + */ + /** * Libp2p rendezvous server. */ class RendezvousServer { /** * @constructor - * @param {Registrar} registrar + * @param {Libp2p} libp2p * @param {object} options * @param {number} options.gcInterval */ - constructor (registrar, { gcInterval = 3e5 } = {}) { - this._registrar = registrar + constructor (libp2p, { gcInterval = 3e5 } = {}) { + this._registrar = libp2p.registrar + this._peerStore = libp2p.peerStore this._gcInterval = gcInterval /** * Registrations per namespace. - * @type {Map>} + * @type {Map>} */ this.nsRegistrations = new Map() @@ -67,6 +76,7 @@ class RendezvousServer { /** * Stops rendezvous server gc and clears registrations + * @returns {void} */ stop () { clearInterval(this._interval) @@ -83,16 +93,20 @@ class RendezvousServer { * @returns {void} */ _gc () { + log('gc starting') + const now = Date.now() const removedIds = [] // Iterate namespaces this.nsRegistrations.forEach((nsEntry) => { // Iterate registrations for namespaces - nsEntry.forEach((reg, idStr) => { - if (now >= reg.expiration) { + nsEntry.forEach((nsReg, idStr) => { + if (now >= nsReg.expiration) { nsEntry.delete(idStr) - removedIds.push(reg.id) + removedIds.push(nsReg.id) + + log(`gc removed namespace entry for ${idStr}`) } }) }) @@ -114,37 +128,38 @@ class RendezvousServer { * Add a peer registration to a namespace. * @param {string} ns * @param {PeerId} peerId - * @param {Array} addrs + * @param {Envelope} envelope * @param {number} ttl * @returns {void} */ - addRegistration (ns, peerId, addrs, ttl) { - const nsEntry = this.nsRegistrations.get(ns) || new Map() + addRegistration (ns, peerId, envelope, ttl) { + const nsReg = this.nsRegistrations.get(ns) || new Map() - nsEntry.set(peerId.toB58String(), { + nsReg.set(peerId.toB58String(), { id: String(Math.random() + Date.now()), - peerId, - addrs, expiration: Date.now() + ttl }) - this.nsRegistrations.set(ns, nsEntry) + this.nsRegistrations.set(ns, nsReg) + + // Store envelope in the AddressBook + this._peerStore.addressBook.consumePeerRecord(envelope) } /** - * Remove rengistration of a given namespace to a peer + * Remove registration of a given namespace to a peer * @param {string} ns * @param {PeerId} peerId * @returns {void} */ removeRegistration (ns, peerId) { - const nsEntry = this.nsRegistrations.get(ns) + const nsReg = this.nsRegistrations.get(ns) - if (nsEntry) { - nsEntry.delete(peerId.toB58String()) + if (nsReg) { + nsReg.delete(peerId.toB58String()) // Remove registrations map to namespace if empty - if (!nsEntry.size) { + if (!nsReg.size) { this.nsRegistrations.delete(ns) } log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) @@ -152,16 +167,16 @@ class RendezvousServer { } /** - * Remove registrations of a given peer + * Remove all registrations of a given peer * @param {PeerId} peerId * @returns {void} */ removePeerRegistrations (peerId) { - for (const [ns, reg] of this.nsRegistrations.entries()) { - reg.delete(peerId.toB58String()) + for (const [ns, nsReg] of this.nsRegistrations.entries()) { + nsReg.delete(peerId.toB58String()) // Remove registrations map to namespace if empty - if (!reg.size) { + if (!nsReg.size) { this.nsRegistrations.delete(ns) } } @@ -182,21 +197,25 @@ class RendezvousServer { const registrations = [] const cRegistrations = this.cookieRegistrations.get(cookie) || new Set() - for (const [idStr, reg] of nsEntry.entries()) { - if (reg.expiration <= Date.now()) { + for (const [idStr, nsReg] of nsEntry.entries()) { + if (nsReg.expiration <= Date.now()) { // Clean outdated registration from registrations and cookie record nsEntry.delete(idStr) - cRegistrations.delete(reg.id) + cRegistrations.delete(nsReg.id) continue } // If this record was already sent, continue - if (cRegistrations.has(reg.id)) { + if (cRegistrations.has(nsReg.id)) { continue } - cRegistrations.add(reg.id) - registrations.push(reg) + cRegistrations.add(nsReg.id) + registrations.push({ + ns, + signedPeerRecord: this._peerStore.addressBook.getRawEnvelope(PeerId.createFromB58String(idStr)), + ttl: Date.now() - nsReg.expiration + }) // Stop if reached limit if (registrations.length === limit) { diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index caf8026..ab14be2 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -51,12 +51,9 @@ module.exports = (rendezvousPoint) => { discoverResponse: { cookie: Buffer.from(cookie), registrations: registrations.map((r) => ({ - ns: msg.discover.ns, - peer: { - id: r.peerId.toBytes(), - addrs: r.addrs - }, - ttl: (r.expiration - Date.now()) * 1e-3 // convert to seconds + ns: r.ns, + signedPeerRecord: r.signedPeerRecord, + ttl: r.ttl * 1e-3 // convert to seconds })), status: RESPONSE_STATUS.OK } diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index d950626..5810a92 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -5,6 +5,9 @@ const debug = require('debug') const log = debug('libp2p:redezvous:protocol:register') log.error = debug('libp2p:redezvous:protocol:register:error') +const Envelope = require('libp2p/src/record/envelope') +const PeerRecord = require('libp2p/src/record/peer-record') + const { Message } = require('../../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus @@ -17,32 +20,35 @@ module.exports = (rendezvousPoint) => { * * @param {PeerId} peerId * @param {Message} msg - * @returns {Message} + * @returns {Promise} */ - return function register (peerId, msg) { + return async function register (peerId, msg) { try { log(`register ${peerId.toB58String()}: trying register on ${msg.register.ns}`) - // Validate auth - if (!msg.register.peer.id.equals(peerId.toBytes())) { - log.error('unauthorized peer id to register') + // Validate namespace + if (!msg.register.ns || msg.register.ns > MAX_NS_LENGTH) { + log.error(`invalid namespace received: ${msg.register.ns}`) return { type: MESSAGE_TYPE.REGISTER_RESPONSE, registerResponse: { - status: RESPONSE_STATUS.E_NOT_AUTHORIZED + status: RESPONSE_STATUS.E_INVALID_NAMESPACE } } } - // Validate namespace - if (!msg.register.ns || msg.register.ns > MAX_NS_LENGTH) { - log.error(`invalid namespace received: ${msg.register.ns}`) + // Open and verify envelope signature + const envelope = await Envelope.openAndCertify(msg.register.signedPeerRecord, PeerRecord.DOMAIN) + + // Validate auth + if (!envelope.peerId.equals(peerId.toBytes())) { + log.error('unauthorized peer id to register') return { type: MESSAGE_TYPE.REGISTER_RESPONSE, registerResponse: { - status: RESPONSE_STATUS.E_INVALID_NAMESPACE + status: RESPONSE_STATUS.E_NOT_AUTHORIZED } } } @@ -51,7 +57,7 @@ module.exports = (rendezvousPoint) => { rendezvousPoint.addRegistration( msg.register.ns, peerId, - msg.register.peer.addrs, + envelope, msg.register.ttl * 1e3 // convert to ms ) diff --git a/test/flows.spec.js b/test/flows.spec.js index cbacc4d..5ff5a65 100644 --- a/test/flows.spec.js +++ b/test/flows.spec.js @@ -72,8 +72,7 @@ describe('flows', () => { registers.push(reg) } expect(registers).to.have.lengthOf(1) - expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) - expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].signedPeerRecord).to.exist() expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index 4d23c5a..f233c6b 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -7,6 +7,9 @@ chai.use(require('chai-as-promised')) const { expect } = chai const sinon = require('sinon') +const Envelope = require('libp2p/src/record/envelope') +const PeerRecord = require('libp2p/src/record/peer-record') + const Rendezvous = require('../src') const { codes: errCodes } = require('../src/errors') @@ -196,11 +199,17 @@ describe('rendezvous', () => { for await (const reg of peers[2].rendezvous.discover(namespace)) { registers.push(reg) } + expect(registers).to.have.lengthOf(1) - expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) - expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].signedPeerRecord).to.exist() expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() + + // Validate envelope + const envelope = await Envelope.openAndCertify(registers[0].signedPeerRecord, PeerRecord.DOMAIN) + const rec = PeerRecord.createFromProtobuf(envelope.payload) + + expect(rec.multiaddrs).to.eql(peers[0].multiaddrs) }) it('discover find registered peer for namespace once (cookie usage)', async () => { @@ -223,8 +232,7 @@ describe('rendezvous', () => { } expect(registers).to.have.lengthOf(1) - expect(registers[0].id.toB58String()).to.eql(peers[0].peerId.toB58String()) - expect(registers[0].multiaddrs).to.eql(peers[0].multiaddrs) + expect(registers[0].signedPeerRecord).to.exist() expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() diff --git a/test/server.spec.js b/test/server.spec.js index 6fa4a82..af391f1 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -7,55 +7,54 @@ chai.use(require('chai-as-promised')) const { expect } = chai const delay = require('delay') -const sinon = require('sinon') + const multiaddr = require('multiaddr') +const Envelope = require('libp2p/src/record/envelope') +const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') -const { createPeerId } = require('./utils') +const { createPeer, createPeerId, createSignedPeerRecord } = require('./utils') -const registrar = { - handle: () => { } -} const testNamespace = 'test-namespace' -const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')].map((m) => m.buffer) +const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')] describe('rendezvous server', () => { + const signedPeerRecords = [] let rServer let peerIds + let libp2p before(async () => { peerIds = await createPeerId({ number: 3 }) - }) - afterEach(() => { - rServer && rServer.stop() + // Create a signed peer record per peer + for (const peerId of peerIds) { + const spr = await createSignedPeerRecord(peerId, multiaddrs) + signedPeerRecords.push(spr) + } }) - it('calls registrar handle on start once', () => { - rServer = new RendezvousServer(registrar) - - // Spy for handle - const spyHandle = sinon.spy(registrar, 'handle') - - rServer.start() - expect(spyHandle).to.have.property('callCount', 1) + beforeEach(async () => { + [libp2p] = await createPeer() + }) - rServer.start() - expect(spyHandle).to.have.property('callCount', 1) + afterEach(async () => { + libp2p && await libp2p.stop() + rServer && rServer.stop() }) it('can add registrations to multiple namespaces', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(otherNamespace, peerIds[0], signedPeerRecords[0], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) const { registrations: testNsRegistrations } = rServer.getRegistrations(testNamespace) expect(testNsRegistrations).to.have.lengthOf(2) @@ -65,12 +64,12 @@ describe('rendezvous server', () => { }) it('should be able to limit registrations to get', () => { - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) let r = rServer.getRegistrations(testNamespace, { limit: 1 }) expect(r.registrations).to.have.lengthOf(1) @@ -82,12 +81,12 @@ describe('rendezvous server', () => { }) it('can remove registrations from a peer in a given namespace', () => { - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) @@ -103,12 +102,12 @@ describe('rendezvous server', () => { it('can remove all registrations from a peer', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(otherNamespace, peerIds[0], signedPeerRecords[0], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) @@ -128,16 +127,16 @@ describe('rendezvous server', () => { it('can attempt to remove a registration for a non existent namespace', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) rServer.removeRegistration(otherNamespace, peerIds[0]) }) it('can attempt to remove a registration for a non existent peer', () => { - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) @@ -150,11 +149,11 @@ describe('rendezvous server', () => { }) it('gc expired records', async () => { - rServer = new RendezvousServer(registrar, { gcInterval: 300 }) + rServer = new RendezvousServer(libp2p, { gcInterval: 300 }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 500) - rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 500) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) @@ -169,21 +168,25 @@ describe('rendezvous server', () => { expect(r.registrations).to.have.lengthOf(0) }) - it('only new peers should be returned if cookie given', () => { - rServer = new RendezvousServer(registrar) + it('only new peers should be returned if cookie given', async () => { + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) expect(cookie).to.exist() expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) - expect(registrations[0].peerId.toString()).to.eql(peerIds[0].toString()) + expect(registrations[0].signedPeerRecord).to.exist() + + // Validate peer0 + const envelope = await Envelope.openAndCertify(registrations[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope.peerId.toString()).to.eql(peerIds[0].toString()) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get second registration by using the cookie const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) @@ -191,7 +194,11 @@ describe('rendezvous server', () => { expect(cookie2).to.eql(cookie) expect(registrations2).to.exist() expect(registrations2).to.have.lengthOf(1) - expect(registrations2[0].peerId.toString()).to.eql(peerIds[1].toString()) + expect(registrations2[0].signedPeerRecord).to.exist() + + // Validate peer1 + const envelope2 = await Envelope.openAndCertify(registrations2[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope2.peerId.toString()).to.eql(peerIds[1].toString()) // If no cookie provided, all registrations are given const { registrations: registrations3 } = rServer.getRegistrations(testNamespace) @@ -200,10 +207,10 @@ describe('rendezvous server', () => { }) it('no new peers should be returned if there are not new peers since latest query', () => { - rServer = new RendezvousServer(registrar) + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) @@ -219,21 +226,25 @@ describe('rendezvous server', () => { expect(registrations2).to.have.lengthOf(0) }) - it('new data for a peer should be returned if registration updated', () => { - rServer = new RendezvousServer(registrar) + it('new data for a peer should be returned if registration updated', async () => { + rServer = new RendezvousServer(libp2p) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) expect(cookie).to.exist() expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) - expect(registrations[0].peerId.toString()).to.eql(peerIds[0].toString()) + expect(registrations[0].signedPeerRecord).to.exist() + + // Validate peer0 + const envelope = await Envelope.openAndCertify(registrations[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope.peerId.toString()).to.eql(peerIds[0].toString()) // Add new registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 1000) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) // Get registrations with same cookie and no new registration const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) @@ -241,15 +252,19 @@ describe('rendezvous server', () => { expect(cookie2).to.eql(cookie) expect(registrations2).to.exist() expect(registrations2).to.have.lengthOf(1) - expect(registrations2[0].peerId.toString()).to.eql(peerIds[0].toString()) + expect(registrations2[0].signedPeerRecord).to.exist() + + // Validate peer0 + const envelope2 = await Envelope.openAndCertify(registrations2[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope2.peerId.toString()).to.eql(peerIds[0].toString()) }) it('garbage collector should remove cookies of discarded records', async () => { - rServer = new RendezvousServer(registrar, { gcInterval: 300 }) + rServer = new RendezvousServer(libp2p, { gcInterval: 300 }) rServer.start() // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], multiaddrs, 500) + rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 500) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) diff --git a/test/utils.js b/test/utils.js index 91b0d77..39de121 100644 --- a/test/utils.js +++ b/test/utils.js @@ -10,6 +10,8 @@ const pWaitFor = require('p-wait-for') const Libp2p = require('libp2p') const multiaddr = require('multiaddr') +const Envelope = require('libp2p/src/record/envelope') +const PeerRecord = require('libp2p/src/record/peer-record') const Peers = require('./fixtures/peers') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') @@ -79,3 +81,16 @@ async function connectPeers (peer, otherPeer) { } module.exports.connectPeers = connectPeers + +async function createSignedPeerRecord (peerId, multiaddrs) { + const pr = new PeerRecord({ + peerId, + multiaddrs + }) + + const envelope = await Envelope.seal(pr, peerId) + + return envelope +} + +module.exports.createSignedPeerRecord = createSignedPeerRecord From b668c8ae35873ff3ecb5bcc21fdd29eea6ab6212 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 22 Jul 2020 18:24:08 +0200 Subject: [PATCH 10/38] chore: tests --- package.json | 4 +- src/index.js | 64 ++++++++++++++------------- test/discovery.spec.js | 8 ++-- test/flows.spec.js | 96 ----------------------------------------- test/rendezvous.spec.js | 93 +++++++++++++++++++++++++++++++++++++++ 5 files changed, 133 insertions(+), 132 deletions(-) delete mode 100644 test/flows.spec.js diff --git a/package.json b/package.json index 538fedc..8358fd1 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "streaming-iterables": "^4.1.2" }, "peerDependencies": { - "libp2p": "libp2p/js-libp2p#feat/certified-addressbook" + "libp2p": "libp2p/js-libp2p#0.29.x" }, "devDependencies": { "aegir": "^25.0.0", @@ -59,7 +59,7 @@ "chai-as-promised": "^7.1.1", "delay": "^4.3.0", "dirty-chai": "^2.0.1", - "libp2p": "libp2p/js-libp2p#feat/certified-addressbook", + "libp2p": "libp2p/js-libp2p#0.29.x", "libp2p-mplex": "^0.9.5", "libp2p-noise": "^1.1.2", "libp2p-websockets": "^0.13.6", diff --git a/src/index.js b/src/index.js index 55ef37c..680b51c 100644 --- a/src/index.js +++ b/src/index.js @@ -355,38 +355,42 @@ class Rendezvous { }) // Send discover message and wait for response - const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) - const [response] = await pipe( - [message], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - - if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { - throw new Error('unexpected message received') - } - - // Iterate over registrations response - for (const r of recMessage.discoverResponse.registrations) { - // track registrations - yield registrationTransformer(r) - - // Store cookie - rpCookies.set(ns, recMessage.discoverResponse.cookie.toString()) - this._rendezvousPoints.set(id, { - connection: rp.connection, - cookies: rpCookies - }) + try { + const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) + const [response] = await pipe( + [message], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + + if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { + throw new Error('unexpected message received') + } - limit-- - if (limit === 0) { - return + // Iterate over registrations response + for (const r of recMessage.discoverResponse.registrations) { + // track registrations + yield registrationTransformer(r) + + // Store cookie + rpCookies.set(ns, recMessage.discoverResponse.cookie.toString()) + this._rendezvousPoints.set(id, { + connection: rp.connection, + cookies: rpCookies + }) + + limit-- + if (limit === 0) { + return + } } + } catch (err) { + log.error(err) } } } diff --git a/test/discovery.spec.js b/test/discovery.spec.js index b2ff1c8..6d4616f 100644 --- a/test/discovery.spec.js +++ b/test/discovery.spec.js @@ -76,7 +76,7 @@ describe('rendezvous discovery', () => { await defer.promise }) - it.skip('peer1 should not discover peer2 if it registers in a different namespace', async () => { + it('peer1 should not discover peer2 if it registers in a different namespace', async () => { const namespace1 = 'test-namespace1' const namespace2 = 'test-namespace2' await peers[1].rendezvous.register(namespace1) @@ -88,11 +88,11 @@ describe('rendezvous discovery', () => { peers[1].rendezvous.discovery.start() // Register - expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(0) await peers[2].rendezvous.register(namespace2) - expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(1) - await delay(1500) + await delay(1000) + + peers[1].rendezvous.discovery.removeAllListeners() }) }) diff --git a/test/flows.spec.js b/test/flows.spec.js deleted file mode 100644 index 5ff5a65..0000000 --- a/test/flows.spec.js +++ /dev/null @@ -1,96 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -const { expect } = chai - -const pWaitFor = require('p-wait-for') -const multiaddr = require('multiaddr') - -const Rendezvous = require('../src') - -const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] -const { createPeer } = require('./utils') - -const namespace = 'ns' - -describe('flows', () => { - describe('3 rendezvous all acting as rendezvous point', () => { - let peers - - const connectPeers = async (peer, otherPeer) => { - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) - await peer.dial(m) - - // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) - } - - beforeEach(async () => { - // Create libp2p nodes - peers = await createPeer({ - number: 3 - }) - - // Create 3 rendezvous peers - peers.forEach((peer) => { - const rendezvous = new Rendezvous({ - libp2p: peer - }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - - // Connect to testing relay node - await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) - }) - - afterEach(() => peers.map(async (libp2p) => { - await libp2p.rendezvous.stop() - await libp2p.stop() - })) - - it('discover find registered peer for namespace only when registered', async () => { - await connectPeers(peers[0], peers[1]) - await connectPeers(peers[2], peers[1]) - - const registers = [] - - // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - // Peer0 register itself on namespace (connected to Peer1) - await peers[0].rendezvous.register(namespace) - - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of peers[2].rendezvous.discover(namespace)) { - registers.push(reg) - } - expect(registers).to.have.lengthOf(1) - expect(registers[0].signedPeerRecord).to.exist() - expect(registers[0].ns).to.eql(namespace) - expect(registers[0].ttl).to.exist() - - // Peer0 unregister itself on namespace (connected to Peer1) - await peers[0].rendezvous.unregister(namespace) - - // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('discovers locally first, and if limit achieved, not go to the network', async () => { - - }) - }) - - describe('3 rendezvous, one acting as rendezvous point', () => { - - }) -}) diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index f233c6b..65cb7f2 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -6,7 +6,9 @@ chai.use(require('dirty-chai')) chai.use(require('chai-as-promised')) const { expect } = chai const sinon = require('sinon') +const pWaitFor = require('p-wait-for') +const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') @@ -14,6 +16,8 @@ const Rendezvous = require('../src') const { codes: errCodes } = require('../src/errors') const { createPeer, connectPeers } = require('./utils') +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] const namespace = 'ns' @@ -243,4 +247,93 @@ describe('rendezvous', () => { expect(registers).to.have.lengthOf(1) }) }) + + describe('flows with 3 rendezvous all acting as rendezvous point', () => { + let peers + + const connectPeers = async (peer, otherPeer) => { + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) + await peer.dial(m) + + // Wait event propagation + await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) + } + + beforeEach(async () => { + // Create libp2p nodes + peers = await createPeer({ + number: 3 + }) + + // Create 3 rendezvous peers + peers.forEach((peer) => { + const rendezvous = new Rendezvous({ + libp2p: peer + }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + + // Connect to testing relay node + await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) + }) + + afterEach(() => peers.map(async (libp2p) => { + await libp2p.rendezvous.stop() + await libp2p.stop() + })) + + it('discover find registered peer for namespace only when registered', async () => { + await connectPeers(peers[0], peers[1]) + await connectPeers(peers[2], peers[1]) + + const registers = [] + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + + // Peer0 register itself on namespace (connected to Peer1) + await peers[0].rendezvous.register(namespace) + + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of peers[2].rendezvous.discover(namespace)) { + registers.push(reg) + } + expect(registers).to.have.lengthOf(1) + expect(registers[0].signedPeerRecord).to.exist() + expect(registers[0].ns).to.eql(namespace) + expect(registers[0].ttl).to.exist() + + // Peer0 unregister itself on namespace (connected to Peer1) + await peers[0].rendezvous.unregister(namespace) + + // Peer2 does not discovery any peer registered + for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + }) + + it('discovers locally first, and if limit achieved, not go to the network', async () => { + await connectPeers(peers[0], peers[1]) + await connectPeers(peers[2], peers[1]) + + // Peer0 register itself on namespace (connected to Peer1) + await peers[1].rendezvous.register(namespace) + + const spyRendezvousPoints = sinon.spy(peers[2].rendezvous._rendezvousPoints, 'entries') + + const registers = [] + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of peers[2].rendezvous.discover(namespace, 1)) { + registers.push(reg) + } + + // No need to get the rendezvousPoints connections + expect(spyRendezvousPoints).to.have.property('callCount', 0) + expect(registers).to.have.lengthOf(1) + }) + }) }) From 7e3c541fe9df93865755b762c8368d21600c2159 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 27 Jul 2020 15:39:06 +0200 Subject: [PATCH 11/38] chore: change readme --- LIBP2P.md | 2 +- README.md | 20 ++++++++++++++++++++ src/discovery.js | 1 + test/discovery.spec.js | 1 + test/server.spec.js | 2 ++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/LIBP2P.md b/LIBP2P.md index 6359b5c..0dc47c6 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -39,4 +39,4 @@ If the discovery service is disabled, the rendezvous API also allows users to di When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. -In the event of a rendezvous client getting connected to a second rendezvous server, it will propagate its registrations to it. The rendezvous server will aso clean its registrations for a peer when it is not connected with it anymore. +In the event of a rendezvous client getting connected to a second rendezvous server, it will propagate its registrations to it. The rendezvous server will also clean its registrations for a peer when it is not connected with it anymore. diff --git a/README.md b/README.md index 949444a..83a0f9b 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,26 @@ See https://github.com/libp2p/specs/tree/master/rendezvous for more details [Vasco Santos](https://github.com/vasco-santos). +## Usage + +```js +const Libp2p = require('libp2p') +const Rendezvous = require('libp2p-rendezvous') + +const libp2p = await Libp2p.create({ + // check on js-libp2p repo the options to provide +}) +const rendezvous = new Rendezvous({ libp2p }) // Set other options below + +await node.start() +rendezvous.start() + +// ... + +rendezvous.stop() +await node.stop() +``` + ## API ### constructor diff --git a/src/discovery.js b/src/discovery.js index 831c7d2..aac1c77 100644 --- a/src/discovery.js +++ b/src/discovery.js @@ -52,6 +52,7 @@ class Discovery extends EventEmitter { * @returns {void} */ stop () { + this.removeAllListeners() clearInterval(this._interval) this._interval = null } diff --git a/test/discovery.spec.js b/test/discovery.spec.js index 6d4616f..b313404 100644 --- a/test/discovery.spec.js +++ b/test/discovery.spec.js @@ -93,6 +93,7 @@ describe('rendezvous discovery', () => { await delay(1000) peers[1].rendezvous.discovery.removeAllListeners() + peers[1].rendezvous.discovery.stop() }) }) diff --git a/test/server.spec.js b/test/server.spec.js index af391f1..88ce77d 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -279,5 +279,7 @@ describe('rendezvous server', () => { expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(0) expect(rServer.cookieRegistrations.get(cookie)).to.not.exist() + + rServer.stop() }) }) From 1a1590decc78dc2e5abcdd36a54fd84be5d74db2 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 22 Sep 2020 15:29:42 +0200 Subject: [PATCH 12/38] chore: update deps --- package.json | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/package.json b/package.json index 8358fd1..6105298 100644 --- a/package.json +++ b/package.json @@ -39,33 +39,33 @@ "coverage": "nyc --reporter=text --reporter=lcov npm test" }, "dependencies": { - "debug": "^4.1.1", + "debug": "^4.2.0", "err-code": "^2.0.3", "it-buffer": "^0.1.2", - "it-length-prefixed": "^3.0.1", + "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", - "libp2p-interfaces": "^0.3.0", - "multiaddr": "^7.5.0", - "peer-id": "^0.13.13", - "protons": "^1.2.0", - "streaming-iterables": "^4.1.2" + "libp2p-interfaces": "^0.5.1", + "multiaddr": "^8.0.0", + "peer-id": "^0.14.1", + "protons": "^2.0.0", + "streaming-iterables": "^5.0.2" }, "peerDependencies": { - "libp2p": "libp2p/js-libp2p#0.29.x" + "libp2p": "^0.29.0" }, "devDependencies": { - "aegir": "^25.0.0", + "aegir": "^26.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", - "delay": "^4.3.0", + "delay": "^4.4.0", "dirty-chai": "^2.0.1", - "libp2p": "libp2p/js-libp2p#0.29.x", - "libp2p-mplex": "^0.9.5", - "libp2p-noise": "^1.1.2", - "libp2p-websockets": "^0.13.6", + "libp2p": "^0.29.0", + "libp2p-mplex": "^0.10.0", + "libp2p-noise": "^2.0.1", + "libp2p-websockets": "^0.14.0", "p-defer": "^3.0.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", - "sinon": "^9.0.2" + "sinon": "^9.0.3" } } From b3578294959a56fa04bfedb47a90b4dcbab880c2 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 22 Sep 2020 15:39:15 +0200 Subject: [PATCH 13/38] chore: remove peer discovery interface as we will be creating libp2p.discover API --- README.md | 16 ----- src/discovery.js | 87 -------------------------- src/index.js | 6 +- test/discovery.spec.js | 135 ----------------------------------------- 4 files changed, 1 insertion(+), 243 deletions(-) delete mode 100644 src/discovery.js delete mode 100644 test/discovery.spec.js diff --git a/README.md b/README.md index 83a0f9b..15fa486 100644 --- a/README.md +++ b/README.md @@ -55,8 +55,6 @@ Creating an instance of Rendezvous. | params.server | `object` | rendezvous server options | | params.server.enabled | `boolean` | rendezvous server enabled (default: `true`) | | params.server.gcInterval | `number` | rendezvous garbage collector interval (default: `3e5`) | -| params.discovery | `object` | rendezvous peer discovery options | -| params.discovery.interval | `number` | automatic rendezvous peer discovery interval (default: `5e3`) | ### rendezvous.start @@ -72,20 +70,6 @@ Unregister the rendezvous protocol and the streams with other peers will be clos `rendezvous.stop()` -### rendezvous.discovery.start - -Starts the rendezvous automatic discovery service. - -`rendezvous.discovery.start()` - -Like other libp2p discovery protocols, it will emit `peer` events when new peers are discovered. - -### rendezvous.discovery.stop - -Stops the rendezvous automatic discovery service. - -`rendezvous.discovery.stop()` - ### rendezvous.register Registers the peer in a given namespace. diff --git a/src/discovery.js b/src/discovery.js deleted file mode 100644 index aac1c77..0000000 --- a/src/discovery.js +++ /dev/null @@ -1,87 +0,0 @@ -'use strict' - -const debug = require('debug') -const log = debug('libp2p:redezvous:discovery') -log.error = debug('libp2p:redezvous:discovery:error') - -const { EventEmitter } = require('events') - -const Envelope = require('libp2p/src/record/envelope') -const PeerRecord = require('libp2p/src/record/peer-record') - -const { codes: errCodes } = require('./errors') - -const defaultOptions = { - interval: 5e3 -} - -/** - * Libp2p Rendezvous discovery service. - */ -class Discovery extends EventEmitter { - /** - * @constructor - * @param {Rendezvous} rendezvous - * @param {Object} [options] - * @param {number} [options.interval = 5000] - */ - constructor (rendezvous, options = {}) { - super() - this._rendezvous = rendezvous - this._options = { - ...defaultOptions, - ...options - } - this._interval = undefined - } - - /** - * Start discovery service. - * @returns {void} - */ - start () { - if (this._interval) { - return - } - - this._interval = setInterval(() => this._discover(), this._options.interval) - } - - /** - * Stop discovery service. - * @returns {void} - */ - stop () { - this.removeAllListeners() - clearInterval(this._interval) - this._interval = null - } - - /** - * Iterates over the registered namespaces and tries to discover new peers - * @returns {void} - */ - _discover () { - this._rendezvous._namespaces.forEach(async (ns) => { - try { - for await (const reg of this._rendezvous.discover(ns)) { - const envelope = await Envelope.openAndCertify(reg.signedPeerRecord, PeerRecord.DOMAIN) - const rec = PeerRecord.createFromProtobuf(envelope.payload) - - // TODO: interface-peer-discovery with signedPeerRecord instead - this.emit('peer', { - id: envelope.peerId, - multiaddrs: rec.multiaddrs - }) - } - } catch (err) { - // It will fail while there are no connected rendezvous servers - if (err.code !== errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) { - throw err - } - } - }) - } -} - -module.exports = Discovery diff --git a/src/index.js b/src/index.js index 680b51c..2fd8c28 100644 --- a/src/index.js +++ b/src/index.js @@ -12,7 +12,6 @@ const { toBuffer } = require('it-buffer') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') -const Discovery = require('./discovery') const Server = require('./server') const { codes: errCodes } = require('./errors') const { PROTOCOL_MULTICODEC } = require('./constants') @@ -42,19 +41,16 @@ class Rendezvous { * @param {object} params * @param {Libp2p} params.libp2p * @param {Array} [params.namespaces = []] - * @param {object} [params.discovery] - * @param {number} [params.discovery.interval = 5e3] * @param {object} [params.server] * @param {boolean} [params.server.enabled = true] * @param {number} [params.server.gcInterval = 3e5] */ - constructor ({ libp2p, namespaces = [], discovery = {}, server = {} }) { + constructor ({ libp2p, namespaces = [], server = {} }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._registrar = libp2p.registrar this._namespaces = namespaces - this.discovery = new Discovery(this, discovery) this._serverOptions = { ...defaultServerOptions, diff --git a/test/discovery.spec.js b/test/discovery.spec.js deleted file mode 100644 index b313404..0000000 --- a/test/discovery.spec.js +++ /dev/null @@ -1,135 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai - -const delay = require('delay') -const pDefer = require('p-defer') -const testsDiscovery = require('libp2p-interfaces/src/peer-discovery/tests') - -const Rendezvous = require('../src') - -const { createPeer, connectPeers } = require('./utils') - -describe('rendezvous discovery', () => { - let peers - - // Create 3 rendezvous peers - // Peer0 will be a server - beforeEach(async () => { - peers = await createPeer({ number: 3 }) - - peers.forEach((peer, index) => { - const rendezvous = new Rendezvous({ - libp2p: peer, - discovery: { - interval: 1000 - }, - server: { - enabled: index === 0 - } - }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - }) - - // Connect rendezvous clients to server - beforeEach(async () => { - await connectPeers(peers[1], peers[0]) - await connectPeers(peers[2], peers[0]) - - expect(peers[0].rendezvous._rendezvousPoints.size).to.eql(0) - expect(peers[1].rendezvous._rendezvousPoints.size).to.eql(1) - expect(peers[2].rendezvous._rendezvousPoints.size).to.eql(1) - }) - - afterEach(async () => { - for (const peer of peers) { - peer.rendezvous.discovery.stop() - await peer.rendezvous.stop() - await peer.stop() - } - }) - - it('peer1 should discover peer2 once it registers to the same namespace', async () => { - const defer = pDefer() - const namespace = 'test-namespace' - peers[1].rendezvous._namespaces = [namespace] - - // Start discovery - peers[1].rendezvous.discovery.once('peer', (peer) => { - expect(peer.id.equals(peers[2].peerId)).to.be.true() - expect(peer.multiaddrs).to.eql(peers[2].multiaddrs) - defer.resolve() - }) - peers[1].rendezvous.discovery.start() - - // Register - expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(0) - await peers[2].rendezvous.register(namespace) - expect(peers[0].rendezvous._server.nsRegistrations.size).to.eql(1) - - await defer.promise - }) - - it('peer1 should not discover peer2 if it registers in a different namespace', async () => { - const namespace1 = 'test-namespace1' - const namespace2 = 'test-namespace2' - await peers[1].rendezvous.register(namespace1) - - // Start discovery - peers[1].rendezvous.discovery.once('peer', () => { - throw new Error('no peer should be discovered') - }) - peers[1].rendezvous.discovery.start() - - // Register - await peers[2].rendezvous.register(namespace2) - - await delay(1000) - - peers[1].rendezvous.discovery.removeAllListeners() - peers[1].rendezvous.discovery.stop() - }) -}) - -describe('interface-discovery', () => { - let peers - - beforeEach(async () => { - peers = await createPeer({ number: 2 }) - - peers.forEach((peer, index) => { - const rendezvous = new Rendezvous({ - libp2p: peer, - discovery: { - interval: 1000 - }, - namespaces: ['test-namespace'], - server: { - enabled: index === 0 - } - }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - - await connectPeers(peers[1], peers[0]) - }) - - testsDiscovery({ - setup () { - return peers[1].rendezvous.discovery - }, - teardown () { - return Promise.all(peers.map(async (libp2p) => { - await libp2p.rendezvous.stop() - await libp2p.stop() - })) - } - }) -}) From 4abd36379c0be15c821aa1ba0cd8c81cc1814b72 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 22 Sep 2020 16:07:01 +0200 Subject: [PATCH 14/38] chore: use uint8array instead of buffer --- README.md | 5 ++-- package.json | 3 +- src/index.js | 42 ++++----------------------- src/server/rpc/handlers/discover.js | 7 +++-- src/server/rpc/handlers/unregister.js | 4 ++- 5 files changed, 17 insertions(+), 44 deletions(-) diff --git a/README.md b/README.md index 15fa486..87c1628 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,13 @@ const libp2p = await Libp2p.create({ }) const rendezvous = new Rendezvous({ libp2p }) // Set other options below -await node.start() +await libp2p.start() rendezvous.start() // ... rendezvous.stop() -await node.stop() +await libp2p.stop() ``` ## API @@ -51,7 +51,6 @@ Creating an instance of Rendezvous. |------|------|-------------| | params | `object` | rendezvous parameters | | params.libp2p | `Libp2p` | a libp2p node instance | -| params.namespaces | `Array` | namespaces to keep registering and discovering over time (default: `[]`) | | params.server | `object` | rendezvous server options | | params.server.enabled | `boolean` | rendezvous server enabled (default: `true`) | | params.server.gcInterval | `number` | rendezvous garbage collector interval (default: `3e5`) | diff --git a/package.json b/package.json index 6105298..14a6973 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,8 @@ "multiaddr": "^8.0.0", "peer-id": "^0.14.1", "protons": "^2.0.0", - "streaming-iterables": "^5.0.2" + "streaming-iterables": "^5.0.2", + "uint8arrays": "^1.1.0" }, "peerDependencies": { "libp2p": "^0.29.0" diff --git a/src/index.js b/src/index.js index 2fd8c28..bd1b2ca 100644 --- a/src/index.js +++ b/src/index.js @@ -9,6 +9,8 @@ const pipe = require('it-pipe') const lp = require('it-length-prefixed') const { collect } = require('streaming-iterables') const { toBuffer } = require('it-buffer') +const fromString = require('uint8arrays/from-string') +const toString = require('uint8arrays/to-string') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') @@ -40,18 +42,15 @@ class Rendezvous { * @constructor * @param {object} params * @param {Libp2p} params.libp2p - * @param {Array} [params.namespaces = []] * @param {object} [params.server] * @param {boolean} [params.server.enabled = true] * @param {number} [params.server.gcInterval = 3e5] */ - constructor ({ libp2p, namespaces = [], server = {} }) { + constructor ({ libp2p, server = {} }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._registrar = libp2p.registrar - this._namespaces = namespaces - this._serverOptions = { ...defaultServerOptions, ...server @@ -102,7 +101,6 @@ class Rendezvous { }) this._registrarId = this._registrar.register(topology) - this._keepRegistrations() log('started') } @@ -132,31 +130,6 @@ class Rendezvous { log('stopped') } - /** - * Keep registrations updated on servers. - * @returns {void} - */ - _keepRegistrations () { - const register = () => { - if (!this._rendezvousPoints.size) { - return - } - - log('update current registrations') - - const promises = [] - - this._namespaces.forEach((ns) => { - promises.push(this.register(ns, { keep: false })) - }) - - return Promise.all(promises) - } - - register() - this._interval = setInterval(register, 1000) - } - /** * Registrar notifies a connection successfully with rendezvous protocol. * @private @@ -191,7 +164,6 @@ class Rendezvous { * @param {string} ns * @param {object} [options] * @param {number} [options.ttl = 7200e3] registration ttl in ms (minimum 120) - * @param {number} [options.keep = true] register over time to guarantee availability. * @returns {Promise} rendezvous register ttl. */ async register (ns, { ttl = 7200e3, keep = true } = {}) { @@ -247,9 +219,6 @@ class Rendezvous { // Return first ttl const [returnTtl] = await Promise.all(registerTasks) - // Keep registering if enabled - keep && this._namespaces.push(ns) - return returnTtl } @@ -295,7 +264,6 @@ class Rendezvous { unregisterTasks.push(taskFn(id)) } - this._namespaces.filter((keeptNampesace) => keeptNampesace !== ns) await Promise.all(unregisterTasks) } @@ -346,7 +314,7 @@ class Rendezvous { discover: { ns, limit, - cookie: cookie ? Buffer.from(cookie) : undefined + cookie: cookie ? fromString(cookie) : undefined } }) @@ -374,7 +342,7 @@ class Rendezvous { yield registrationTransformer(r) // Store cookie - rpCookies.set(ns, recMessage.discoverResponse.cookie.toString()) + rpCookies.set(ns, toString(recMessage.discoverResponse.cookie)) this._rendezvousPoints.set(id, { connection: rp.connection, cookies: rpCookies diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index ab14be2..c27878a 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -5,6 +5,9 @@ const debug = require('debug') const log = debug('libp2p:redezvous:protocol:discover') log.error = debug('libp2p:redezvous:protocol:discover:error') +const fromString = require('uint8arrays/from-string') +const toString = require('uint8arrays/to-string') + const { Message } = require('../../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus @@ -41,7 +44,7 @@ module.exports = (rendezvousPoint) => { // Get registrations const options = { - cookie: msg.discover.cookie ? msg.discover.cookie.toString() : undefined, + cookie: msg.discover.cookie ? toString(msg.discover.cookie) : undefined, limit: msg.discover.limit } const { registrations, cookie } = rendezvousPoint.getRegistrations(msg.discover.ns, options) @@ -49,7 +52,7 @@ module.exports = (rendezvousPoint) => { return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, discoverResponse: { - cookie: Buffer.from(cookie), + cookie: fromString(cookie), registrations: registrations.map((r) => ({ ns: r.ns, signedPeerRecord: r.signedPeerRecord, diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js index 940b4d5..bb180b3 100644 --- a/src/server/rpc/handlers/unregister.js +++ b/src/server/rpc/handlers/unregister.js @@ -5,6 +5,8 @@ const debug = require('debug') const log = debug('libp2p:redezvous:protocol:unregister') log.error = debug('libp2p:redezvous:protocol:unregister:error') +const equals = require('uint8arrays/equals') + module.exports = (rendezvousPoint) => { /** * Process `Unregister` Rendezvous messages. @@ -21,7 +23,7 @@ module.exports = (rendezvousPoint) => { } // Validate auth - if (!msg.unregister.id.equals(peerId.toBytes())) { + if (!equals(msg.unregister.id, peerId.toBytes())) { log.error('unauthorized peer id to unregister') return From 7763df2d50b277581621e27ab3abcbd5381a9f84 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 28 Sep 2020 15:33:27 +0200 Subject: [PATCH 15/38] chore: update libp2p integration doc --- .aegir.js | 8 +------- LIBP2P.md | 30 ++++++++++++------------------ 2 files changed, 13 insertions(+), 25 deletions(-) diff --git a/.aegir.js b/.aegir.js index 3b43827..c3c7f90 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,9 +8,7 @@ const WebSockets = require('libp2p-websockets') const Muxer = require('libp2p-mplex') const { NOISE: Crypto } = require('libp2p-noise') -const Rendezvous = require('.') - -let libp2p, rendezvous +let libp2p const before = async () => { // Use the last peer @@ -38,13 +36,9 @@ const before = async () => { }) await libp2p.start() - - // rendezvous = new Rendezvous({ libp2p }) - // await rendezvous.start() } const after = async () => { - // await rendezvous.stop() await libp2p.stop() } diff --git a/LIBP2P.md b/LIBP2P.md index 0dc47c6..5cbed43 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -4,39 +4,33 @@ The rendezvous protocol can be used in different contexts across libp2p. For usi ## Usage -`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled, as well as its server mode. In addition, automatic peer discovery can be enabled and namespaces to register can be specified from startup through the config. - -The rendezvous implementation also brings a discovery service that enables libp2p to automatically discover other peers in the provided namespaces and eventually connect to them. +`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled, as well as its server mode. You can configure it through libp2p as follows: ```js const Libp2p = require('libp2p') +const Rendezvous = require('libp2p-rendezvous') const node = await Libp2p.create({ - // ... required configurations - rendezvous: { - enabled: true, - namespaces: ['/namespace/1', '/namespace/2'], - discovery: { + modules: { + rendezvous: Rendezvous + }, + config: { + rendezvous: { enabled: true, - interval: 1000 - }, - server: { - enabled: true + server: { + enabled: true + } } } }) ``` -While `js-libp2p` supports the rendezvous protocol out of the box, it also provides a rendezvous API that users can interact with. This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover peers. +While `js-libp2p` supports the rendezvous protocol out of the box through its discovery API, it also provides a rendezvous API that users can interact with. This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover peers. ## Libp2p Flow -When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server and ask for nodes in given namespaces (namespaces provided for register). The rendezvous server can be added to the bootstrap nodes or manually dialed. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. - -If the discovery service is disabled, the rendezvous API also allows users to discover peers registered on provided namespaces. +When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server. The rendezvous server can be added to the bootstrap nodes or manually dialed. WHen a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. - -In the event of a rendezvous client getting connected to a second rendezvous server, it will propagate its registrations to it. The rendezvous server will also clean its registrations for a peer when it is not connected with it anymore. From 63d607bd5d5f797ea9a29ec98df465608d2f3a77 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 28 Sep 2020 15:58:43 +0200 Subject: [PATCH 16/38] chore: fix register ttl param return --- src/server/rpc/handlers/register.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index 5810a92..ab63a5f 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -65,7 +65,7 @@ module.exports = (rendezvousPoint) => { type: MESSAGE_TYPE.REGISTER_RESPONSE, registerResponse: { status: RESPONSE_STATUS.OK, - ttt: msg.register.ttl + ttl: msg.register.ttl } } } catch (err) { From 5f45c6fb89e9a9869eddaac9bb3c38c41d59bac3 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 29 Sep 2020 15:35:24 +0200 Subject: [PATCH 17/38] chore: update docs --- README.md | 5 +- package.json | 4 +- src/index.js | 73 +++++++++++++-------------- src/server/index.js | 8 +-- src/server/rpc/handlers/discover.js | 4 +- src/server/rpc/handlers/register.js | 4 +- src/server/rpc/handlers/unregister.js | 4 +- src/server/rpc/index.js | 4 +- 8 files changed, 49 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 87c1628..a73a891 100644 --- a/README.md +++ b/README.md @@ -57,12 +57,10 @@ Creating an instance of Rendezvous. ### rendezvous.start -Register the rendezvous protocol topology into libp2p and starts its internal services. The rendezvous server will be started if enabled, as well as the service to keep self registrations available. +Register the rendezvous protocol topology into libp2p. The rendezvous server will be started if enabled, as well as the service to keep self registrations available. `rendezvous.start()` -When registering to new namespaces from the API, the new namespace will be added to the registrations to keep by default. - ### rendezvous.stop Unregister the rendezvous protocol and the streams with other peers will be closed. @@ -82,7 +80,6 @@ Registers the peer in a given namespace. | namespace | `string` | namespace to register | | options | `object` | rendezvous registrations options | | options.ttl | `number` | registration ttl in ms (default: `7200e3` and minimum `120`) | -| options.keep | `boolean` | register over time to guarantee availability (default: `true`) | #### Returns diff --git a/package.json b/package.json index 14a6973..4d0ab0f 100644 --- a/package.json +++ b/package.json @@ -21,10 +21,10 @@ "bugs": { "url": "https://github.com/libp2p/js-libp2p-rendezvous/issues" }, - "homepage": "https://libp2p.io", + "homepage": "https://github.com/libp2p/js-libp2p-rendezvous", "license": "MIT", "engines": { - "node": ">=10.0.0", + "node": ">=12.0.0", "npm": ">=6.0.0" }, "scripts": { diff --git a/src/index.js b/src/index.js index bd1b2ca..d9f1086 100644 --- a/src/index.js +++ b/src/index.js @@ -1,8 +1,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous') -log.error = debug('libp2p:redezvous:error') +const log = debug('libp2p:rendezvous') +log.error = debug('libp2p:rendezvous:error') const errCode = require('err-code') const pipe = require('it-pipe') @@ -85,7 +85,7 @@ class Rendezvous { log('starting') - // Create Rendezvous point if enabled + // Create and start Rendezvous server if enabled if (this._serverOptions.enabled) { this._server = new Server(this._libp2p, this._serverOptions) this._server.start() @@ -166,7 +166,7 @@ class Rendezvous { * @param {number} [options.ttl = 7200e3] registration ttl in ms (minimum 120) * @returns {Promise} rendezvous register ttl. */ - async register (ns, { ttl = 7200e3, keep = true } = {}) { + async register (ns, { ttl = 7200e3 } = {}) { if (!ns) { throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) } @@ -319,46 +319,41 @@ class Rendezvous { }) // Send discover message and wait for response - try { - const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) - const [response] = await pipe( - [message], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - - if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { - throw new Error('unexpected message received') - } + const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) + const [response] = await pipe( + [message], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + + if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { + throw new Error('unexpected message received') + } + + // Iterate over registrations response + for (const r of recMessage.discoverResponse.registrations) { + // track registrations + yield registrationTransformer(r) + + // Store cookie + rpCookies.set(ns, toString(recMessage.discoverResponse.cookie)) + this._rendezvousPoints.set(id, { + connection: rp.connection, + cookies: rpCookies + }) - // Iterate over registrations response - for (const r of recMessage.discoverResponse.registrations) { - // track registrations - yield registrationTransformer(r) - - // Store cookie - rpCookies.set(ns, toString(recMessage.discoverResponse.cookie)) - this._rendezvousPoints.set(id, { - connection: rp.connection, - cookies: rpCookies - }) - - limit-- - if (limit === 0) { - return - } + limit-- + if (limit === 0) { + return } - } catch (err) { - log.error(err) } } } } -Rendezvous.tag = 'rendezvous' module.exports = Rendezvous diff --git a/src/server/index.js b/src/server/index.js index 35b62bf..46cf769 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -1,8 +1,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous-server') -log.error = debug('libp2p:redezvous-server:error') +const log = debug('libp2p:rendezvous-server') +log.error = debug('libp2p:rendezvous-server:error') const PeerId = require('peer-id') @@ -31,8 +31,8 @@ class RendezvousServer { /** * @constructor * @param {Libp2p} libp2p - * @param {object} options - * @param {number} options.gcInterval + * @param {object} [options] + * @param {number} [options.gcInterval = 3e5] */ constructor (libp2p, { gcInterval = 3e5 } = {}) { this._registrar = libp2p.registrar diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index c27878a..ba9c612 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -2,8 +2,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous:protocol:discover') -log.error = debug('libp2p:redezvous:protocol:discover:error') +const log = debug('libp2p:rendezvous:protocol:discover') +log.error = debug('libp2p:rendezvous:protocol:discover:error') const fromString = require('uint8arrays/from-string') const toString = require('uint8arrays/to-string') diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index ab63a5f..4431d53 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -2,8 +2,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous:protocol:register') -log.error = debug('libp2p:redezvous:protocol:register:error') +const log = debug('libp2p:rendezvous:protocol:register') +log.error = debug('libp2p:rendezvous:protocol:register:error') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js index bb180b3..fbaa1f0 100644 --- a/src/server/rpc/handlers/unregister.js +++ b/src/server/rpc/handlers/unregister.js @@ -2,8 +2,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous:protocol:unregister') -log.error = debug('libp2p:redezvous:protocol:unregister:error') +const log = debug('libp2p:rendezvous:protocol:unregister') +log.error = debug('libp2p:rendezvous:protocol:unregister:error') const equals = require('uint8arrays/equals') diff --git a/src/server/rpc/index.js b/src/server/rpc/index.js index d4f423e..57df8b0 100644 --- a/src/server/rpc/index.js +++ b/src/server/rpc/index.js @@ -1,8 +1,8 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:redezvous-point:rpc') -log.error = debug('libp2p:redezvous-point:rpc:error') +const log = debug('libp2p:rendezvous-point:rpc') +log.error = debug('libp2p:rendezvous-point:rpc:error') const pipe = require('it-pipe') const lp = require('it-length-prefixed') From 640b64fb8c2876c349ad65e6036121c1b80154b8 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 5 Oct 2020 10:43:05 +0200 Subject: [PATCH 18/38] chore: remove enabled property from libp2p integration doc --- LIBP2P.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/LIBP2P.md b/LIBP2P.md index 5cbed43..6b39177 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -18,9 +18,8 @@ const node = await Libp2p.create({ }, config: { rendezvous: { - enabled: true, server: { - enabled: true + enabled: false } } } From 8f6e148aa711e498cbfc43a98470d8e43f437579 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 16 Nov 2020 14:12:15 +0100 Subject: [PATCH 19/38] chore: apply suggestions from code review Co-authored-by: Jacob Heun --- LIBP2P.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LIBP2P.md b/LIBP2P.md index 6b39177..e3ed9dd 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -30,6 +30,6 @@ While `js-libp2p` supports the rendezvous protocol out of the box through its di ## Libp2p Flow -When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server. The rendezvous server can be added to the bootstrap nodes or manually dialed. WHen a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undiable nodes can register themselves as reachable through that relay. +When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server. The rendezvous server can be added to the bootstrap nodes or manually dialed. When a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undialable nodes can register themselves as reachable through that relay. When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. From 894ad2e7986487b67136903059b15cff11e810db Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 17 Nov 2020 13:13:27 +0100 Subject: [PATCH 20/38] chore: separate server and client rendezvous --- package.json | 14 +-- src/index.js | 62 +--------- src/server/bin.js | 93 ++++++++++++++ src/server/index.js | 21 ++-- src/server/utils.js | 41 +++++++ test/client-mode.spec.js | 47 ------- test/connectivity.spec.js | 67 ---------- test/rendezvous.spec.js | 251 ++++++++++++++++++++++---------------- test/server.spec.js | 143 +++++++++++++++------- test/utils.js | 45 ++++++- 10 files changed, 435 insertions(+), 349 deletions(-) create mode 100644 src/server/bin.js create mode 100644 src/server/utils.js delete mode 100644 test/client-mode.spec.js delete mode 100644 test/connectivity.spec.js diff --git a/package.json b/package.json index 4d0ab0f..afa01c4 100644 --- a/package.json +++ b/package.json @@ -44,26 +44,26 @@ "it-buffer": "^0.1.2", "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", + "libp2p": "^0.29.0", "libp2p-interfaces": "^0.5.1", + "libp2p-mplex": "^0.10.0", + "libp2p-noise": "^2.0.1", + "libp2p-tcp": "^0.15.1", + "libp2p-websockets": "^0.14.0", + "menoetius": "0.0.2", + "minimist": "^1.2.5", "multiaddr": "^8.0.0", "peer-id": "^0.14.1", "protons": "^2.0.0", "streaming-iterables": "^5.0.2", "uint8arrays": "^1.1.0" }, - "peerDependencies": { - "libp2p": "^0.29.0" - }, "devDependencies": { "aegir": "^26.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.4.0", "dirty-chai": "^2.0.1", - "libp2p": "^0.29.0", - "libp2p-mplex": "^0.10.0", - "libp2p-noise": "^2.0.1", - "libp2p-websockets": "^0.14.0", "p-defer": "^3.0.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", diff --git a/src/index.js b/src/index.js index d9f1086..6feb99f 100644 --- a/src/index.js +++ b/src/index.js @@ -14,17 +14,11 @@ const toString = require('uint8arrays/to-string') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') -const Server = require('./server') const { codes: errCodes } = require('./errors') const { PROTOCOL_MULTICODEC } = require('./constants') const { Message } = require('./proto') const MESSAGE_TYPE = Message.MessageType -const defaultServerOptions = { - enabled: true, - gcInterval: 3e5 -} - /** * Rendezvous point contains the connection to a rendezvous server, as well as, * the cookies per namespace that the client received. @@ -42,33 +36,17 @@ class Rendezvous { * @constructor * @param {object} params * @param {Libp2p} params.libp2p - * @param {object} [params.server] - * @param {boolean} [params.server.enabled = true] - * @param {number} [params.server.gcInterval = 3e5] */ - constructor ({ libp2p, server = {} }) { + constructor ({ libp2p }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._registrar = libp2p.registrar - this._serverOptions = { - ...defaultServerOptions, - ...server - } - /** * @type {Map} */ this._rendezvousPoints = new Map() - /** - * Client cookies per namespace for own server - * @type {Map} - */ - this._cookiesSelf = new Map() - - this._server = undefined - this._registrarId = undefined this._onPeerConnected = this._onPeerConnected.bind(this) this._onPeerDisconnected = this._onPeerDisconnected.bind(this) @@ -85,12 +63,6 @@ class Rendezvous { log('starting') - // Create and start Rendezvous server if enabled - if (this._serverOptions.enabled) { - this._server = new Server(this._libp2p, this._serverOptions) - this._server.start() - } - // register protocol with topology const topology = new MulticodecTopology({ multicodecs: PROTOCOL_MULTICODEC, @@ -105,7 +77,7 @@ class Rendezvous { } /** - * Unregister the rendezvous protocol and the streams with other peers will be closed. + * Unregister the rendezvous protocol and clear the state. * @returns {void} */ stop () { @@ -115,17 +87,11 @@ class Rendezvous { log('stopping') - clearInterval(this._interval) - // unregister protocol and handlers this._registrar.unregister(this._registrarId) - if (this._serverOptions.enabled) { - this._server.stop() - } this._registrarId = undefined this._rendezvousPoints.clear() - this._cookiesSelf.clear() log('stopped') } @@ -153,10 +119,6 @@ class Rendezvous { log('disconnected', idB58Str) this._rendezvousPoints.delete(idB58Str) - - if (this._server) { - this._server.removePeerRegistrations(peerId) - } } /** @@ -271,7 +233,7 @@ class Rendezvous { * Discover peers registered under a given namespace * @param {string} ns * @param {number} [limit] - * @returns {AsyncIterable<{ signedPeerRecord: Buffer, ns: string, ttl: number }>} + * @returns {AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>} */ async * discover (ns, limit) { // Are there available rendezvous servers? @@ -285,24 +247,6 @@ class Rendezvous { ttl: r.ttl * 1e3 // convert to ms }) - // Local search if Server enabled - if (this._server) { - const cookieSelf = this._cookiesSelf.get(ns) - const { cookie: cookieS, registrations: localRegistrations } = this._server.getRegistrations(ns, { limit, cookie: cookieSelf }) - - for (const r of localRegistrations) { - yield registrationTransformer(r) - - limit-- - if (limit === 0) { - return - } - } - - // Store cookie self - this._cookiesSelf.set(ns, cookieS) - } - // Iterate over all rendezvous points for (const [id, rp] of this._rendezvousPoints.entries()) { const rpCookies = rp.cookies || new Map() diff --git a/src/server/bin.js b/src/server/bin.js new file mode 100644 index 0000000..ada32e1 --- /dev/null +++ b/src/server/bin.js @@ -0,0 +1,93 @@ +#!/usr/bin/env node + +'use strict' + +// Usage: $0 [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsMultiaddr ] [--disableMetrics] + +/* eslint-disable no-console */ + +const debug = require('debug') +const log = debug('libp2p:rendezvous:bin') + +const fs = require('fs') +const http = require('http') +const menoetius = require('menoetius') +const argv = require('minimist')(process.argv.slice(2)) + +const TCP = require('libp2p-tcp') +const Websockets = require('libp2p-websockets') +const Muxer = require('libp2p-mplex') +const { NOISE: Crypto } = require('libp2p-noise') + +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') + +const RendezvousServer = require('./index') +const { getAnnounceAddresses, getListenAddresses } = require('./utils') + +async function main () { + // Metrics + let metricsServer + const metrics = !(argv.disableMetrics || process.env.DISABLE_METRICS) + const metricsMa = multiaddr(argv.metricsMultiaddr || argv.ma || process.env.METRICSMA || '/ip4/127.0.0.1/tcp/8003') + const metricsAddr = metricsMa.nodeAddress() + + // Multiaddrs + const listenAddresses = getListenAddresses(argv) + const announceAddresses = getAnnounceAddresses(argv) + + // PeerId + let peerId + if (argv.peerId) { + const peerData = fs.readFileSync(argv.peerId) + peerId = await PeerId.createFromJSON(JSON.parse(peerData)) + } else { + peerId = await PeerId.create() + log('You are using an automatically generated peer.') + log('If you want to keep the same address for the server you should provide a peerId with --peerId ') + } + + // Create Rendezvous server + const rendezvousServer = new RendezvousServer({ + modules: { + transport: [Websockets, TCP], + streamMuxer: [Muxer], + connEncryption: [Crypto] + }, + peerId, + addresses: { + listen: listenAddresses, + announce: announceAddresses + } + }) + + await rendezvousServer.start() + + if (metrics) { + log('enabling metrics') + metricsServer = http.createServer((req, res) => { + if (req.url !== '/metrics') { + res.statusCode = 200 + res.end() + } + }) + + menoetius.instrument(metricsServer) + + metricsServer.listen(metricsAddr.port, metricsAddr.address, () => { + console.log(`metrics server listening on ${metricsAddr.port}`) + }) + } + + const stop = async () => { + console.log('Stopping...', rendezvousServer.multiaddrs) + await rendezvousServer.stop() + metricsServer && await metricsServer.close() + process.exit(0) + } + + process.on('SIGTERM', stop) + process.on('SIGINT', stop) +} + +main() diff --git a/src/server/index.js b/src/server/index.js index 46cf769..6043b6c 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -4,6 +4,7 @@ const debug = require('debug') const log = debug('libp2p:rendezvous-server') log.error = debug('libp2p:rendezvous-server:error') +const Libp2p = require('libp2p') const PeerId = require('peer-id') const { PROTOCOL_MULTICODEC, MAX_LIMIT } = require('../constants') @@ -27,16 +28,16 @@ const rpc = require('./rpc') /** * Libp2p rendezvous server. */ -class RendezvousServer { +class RendezvousServer extends Libp2p { /** * @constructor - * @param {Libp2p} libp2p + * @param {Libp2pOptions} libp2pOptions * @param {object} [options] * @param {number} [options.gcInterval = 3e5] */ - constructor (libp2p, { gcInterval = 3e5 } = {}) { - this._registrar = libp2p.registrar - this._peerStore = libp2p.peerStore + constructor (libp2pOptions, { gcInterval = 3e5 } = {}) { + super(libp2pOptions) + this._gcInterval = gcInterval /** @@ -59,6 +60,8 @@ class RendezvousServer { * @returns {void} */ start () { + super.start() + if (this._interval) { return } @@ -69,7 +72,7 @@ class RendezvousServer { this._interval = setInterval(this._gc, this._gcInterval) // Incoming streams handling - this._registrar.handle(PROTOCOL_MULTICODEC, rpc(this)) + this.registrar.handle(PROTOCOL_MULTICODEC, rpc(this)) log('started') } @@ -79,6 +82,8 @@ class RendezvousServer { * @returns {void} */ stop () { + super.stop() + clearInterval(this._interval) this._interval = undefined @@ -143,7 +148,7 @@ class RendezvousServer { this.nsRegistrations.set(ns, nsReg) // Store envelope in the AddressBook - this._peerStore.addressBook.consumePeerRecord(envelope) + this.peerStore.addressBook.consumePeerRecord(envelope) } /** @@ -213,7 +218,7 @@ class RendezvousServer { cRegistrations.add(nsReg.id) registrations.push({ ns, - signedPeerRecord: this._peerStore.addressBook.getRawEnvelope(PeerId.createFromB58String(idStr)), + signedPeerRecord: this.peerStore.addressBook.getRawEnvelope(PeerId.createFromB58String(idStr)), ttl: Date.now() - nsReg.expiration }) diff --git a/src/server/utils.js b/src/server/utils.js new file mode 100644 index 0000000..b831fe7 --- /dev/null +++ b/src/server/utils.js @@ -0,0 +1,41 @@ +'use strict' + +const multiaddr = require('multiaddr') + +function getAnnounceAddresses(argv) { + const announceAddr = argv.announceMultiaddrs || argv.am + const announceAddresses = announceAddr ? [multiaddr(announceAddr)] : [] + + if (argv.announceMultiaddrs || argv.am) { + const flagIndex = process.argv.findIndex((e) => e === '--announceMultiaddrs' || e === '--am') + const tmpEndIndex = process.argv.slice(flagIndex + 1).findIndex((e) => e.startsWith('--')) + const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 + + for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { + announceAddresses.push(multiaddr(process.argv[i + 1])) + } + } + + return announceAddresses +} + +module.exports.getAnnounceAddresses = getAnnounceAddresses + +function getListenAddresses(argv) { + const listenAddr = argv.listenMultiaddrs || argv.lm || '/ip4/127.0.0.1/tcp/15002/ws' + const listenAddresses = [multiaddr(listenAddr)] + + if (argv.listenMultiaddrs || argv.lm) { + const flagIndex = process.argv.findIndex((e) => e === '--listenMultiaddrs' || e === '--lm') + const tmpEndIndex = process.argv.slice(flagIndex + 1).findIndex((e) => e.startsWith('--')) + const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 + + for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { + listenAddresses.push(multiaddr(process.argv[i + 1])) + } + } + + return listenAddresses +} + +module.exports.getListenAddresses = getListenAddresses diff --git a/test/client-mode.spec.js b/test/client-mode.spec.js deleted file mode 100644 index 569ea52..0000000 --- a/test/client-mode.spec.js +++ /dev/null @@ -1,47 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai -const sinon = require('sinon') - -const Rendezvous = require('../src') - -const { createPeer } = require('./utils') - -describe('client mode', () => { - let peer, rendezvous - - afterEach(async () => { - peer && await peer.stop() - rendezvous && rendezvous.stop() - }) - - it('registers a rendezvous handler by default', async () => { - [peer] = await createPeer() - rendezvous = new Rendezvous({ libp2p: peer }) - - const spyHandle = sinon.spy(peer.registrar, '_handle') - - rendezvous.start() - - expect(spyHandle).to.have.property('callCount', 1) - }) - - it('can be started only in client mode', async () => { - [peer] = await createPeer() - rendezvous = new Rendezvous({ - libp2p: peer, - server: { - enabled: false - } - }) - - const spyHandle = sinon.spy(peer.registrar, '_handle') - - rendezvous.start() - expect(spyHandle).to.have.property('callCount', 0) - }) -}) diff --git a/test/connectivity.spec.js b/test/connectivity.spec.js deleted file mode 100644 index 9f34576..0000000 --- a/test/connectivity.spec.js +++ /dev/null @@ -1,67 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -const { expect } = chai -const pWaitFor = require('p-wait-for') - -const multiaddr = require('multiaddr') - -const Rendezvous = require('../src') - -const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] -const { createPeer } = require('./utils') - -describe('connectivity', () => { - let peers - - beforeEach(async () => { - // Create libp2p nodes - peers = await createPeer({ - number: 2 - }) - - // Create && start rendezvous - peers.map((libp2p) => { - const rendezvous = new Rendezvous({ libp2p }) - rendezvous.start() - libp2p.rendezvous = rendezvous - }) - - // Connect to testing relay node - await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) - }) - - afterEach(() => peers.map(async (libp2p) => { - await libp2p.rendezvous.stop() - await libp2p.stop() - })) - - it('updates known rendezvous points', async () => { - expect(peers[0].rendezvous._rendezvousPoints.size).to.equal(0) - expect(peers[1].rendezvous._rendezvousPoints.size).to.equal(0) - - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peers[1].peerId.toB58String()}`) - const connection = await peers[0].dial(m) - - expect(peers[0].peerStore.peers.size).to.equal(2) - expect(peers[1].peerStore.peers.size).to.equal(2) - - // Wait event propagation - // Relay peer is not with rendezvous enabled - await pWaitFor(() => - peers[0].rendezvous._rendezvousPoints.size === 1 && - peers[1].rendezvous._rendezvousPoints.size === 1) - - expect(peers[0].rendezvous._rendezvousPoints.get(peers[1].peerId.toB58String())).to.exist() - expect(peers[1].rendezvous._rendezvousPoints.get(peers[0].peerId.toB58String())).to.exist() - - await connection.close() - - // Wait event propagation - await pWaitFor(() => peers[0].rendezvous._rendezvousPoints.size === 0) - }) -}) diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index 65cb7f2..fae542b 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -15,7 +15,11 @@ const PeerRecord = require('libp2p/src/record/peer-record') const Rendezvous = require('../src') const { codes: errCodes } = require('../src/errors') -const { createPeer, connectPeers } = require('./utils') +const { + createPeer, + createRendezvousServer, + connectPeers +} = require('./utils') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') const relayAddr = MULTIADDRS_WEBSOCKETS[0] @@ -66,107 +70,152 @@ describe('rendezvous', () => { }) }) + describe('connectivity', () => { + let rendezvousServer + let client + + // Create and start Libp2p nodes + beforeEach(async () => { + // Create Rendezvous Server + rendezvousServer = await createRendezvousServer() + + // Create Rendezvous client + ;[client] = await createPeer() + + const rendezvous = new Rendezvous({ libp2p: client }) + client.rendezvous = rendezvous + client.rendezvous.start() + }) + + // Connect nodes to the testing relay node + beforeEach(async () => { + await rendezvousServer.dial(relayAddr) + await client.dial(relayAddr) + }) + + afterEach(async () => { + await rendezvousServer.stop() + await client.rendezvous.stop() + await client.stop() + }) + + it('updates known rendezvous points', async () => { + expect(client.rendezvous._rendezvousPoints.size).to.equal(0) + + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${rendezvousServer.peerId.toB58String()}`) + const connection = await client.dial(m) + + expect(client.peerStore.peers.size).to.equal(2) + expect(rendezvousServer.peerStore.peers.size).to.equal(2) + + // Wait event propagation + // Relay peer is not with rendezvous enabled + await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 1) + + expect(client.rendezvous._rendezvousPoints.get(rendezvousServer.peerId.toB58String())).to.exist() + + await connection.close() + + // Wait event propagation + await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 0) + }) + }) + describe('api', () => { - let peers + let rendezvousServer + let clients + // Create and start Libp2p nodes beforeEach(async () => { - peers = await createPeer({ number: 3 }) - - // Create 3 rendezvous peers - // Peer0 will not be a server - peers.forEach((peer, index) => { - const rendezvous = new Rendezvous({ - libp2p: peer, - server: { - enabled: index !== 0 - } - }) + // Create Rendezvous Server + rendezvousServer = await createRendezvousServer() + + clients = await createPeer({ number: 2 }) + + // Create 2 rendezvous clients + clients.forEach((peer) => { + const rendezvous = new Rendezvous({ libp2p: peer }) rendezvous.start() peer.rendezvous = rendezvous }) }) afterEach(async () => { - for (const peer of peers) { + await rendezvousServer.stop() + + for (const peer of clients) { await peer.rendezvous.stop() await peer.stop() } }) it('register throws error if a namespace is not provided', async () => { - await expect(peers[0].rendezvous.register()) + await expect(clients[0].rendezvous.register()) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_NAMESPACE) }) it('register throws error if ttl is too small', async () => { - await expect(peers[0].rendezvous.register(namespace, { ttl: 10 })) + await expect(clients[0].rendezvous.register(namespace, { ttl: 10 })) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_TTL) }) it('register throws error if no connected rendezvous servers', async () => { - await expect(peers[0].rendezvous.register(namespace)) + await expect(clients[0].rendezvous.register(namespace)) .to.eventually.rejected() .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) }) it('register to a connected rendezvous server node', async () => { - await connectPeers(peers[0], peers[1]) + await connectPeers(clients[0], rendezvousServer) // Register - expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) - await peers[0].rendezvous.register(namespace) - - expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(1) - expect(peers[1].rendezvous._server.nsRegistrations.get(namespace)).to.exist() + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + await clients[0].rendezvous.register(namespace) - await peers[1].rendezvous.stop() - await peers[1].stop() + expect(rendezvousServer.nsRegistrations.size).to.eql(1) + expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() }) it('unregister throws if a namespace is not provided', async () => { - await expect(peers[0].rendezvous.unregister()) + await expect(clients[0].rendezvous.unregister()) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_NAMESPACE) }) it('register throws error if no connected rendezvous servers', async () => { - await expect(peers[0].rendezvous.unregister(namespace)) + await expect(clients[0].rendezvous.unregister(namespace)) .to.eventually.rejected() .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) }) it('unregister to a connected rendezvous server node', async () => { - await connectPeers(peers[0], peers[1]) + await connectPeers(clients[0], rendezvousServer) // Register - expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) - await peers[0].rendezvous.register(namespace) + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + await clients[0].rendezvous.register(namespace) - expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(1) - expect(peers[1].rendezvous._server.nsRegistrations.get(namespace)).to.exist() + expect(rendezvousServer.nsRegistrations.size).to.eql(1) + expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() // Unregister - await peers[0].rendezvous.unregister(namespace) - expect(peers[1].rendezvous._server.nsRegistrations.size).to.eql(0) - - await peers[1].rendezvous.stop() - await peers[1].stop() + await clients[0].rendezvous.unregister(namespace) + expect(rendezvousServer.nsRegistrations.size).to.eql(0) }) it('unregister to a connected rendezvous server node not fails if not registered', async () => { - await connectPeers(peers[0], peers[1]) + await connectPeers(clients[0], rendezvousServer) // Unregister - await peers[0].rendezvous.unregister(namespace) - - await peers[1].rendezvous.stop() + await clients[0].rendezvous.unregister(namespace) }) it('discover throws error if a namespace is not provided', async () => { try { - for await (const _ of peers[0].rendezvous.discover()) {} // eslint-disable-line + for await (const _ of clients[0].rendezvous.discover()) {} // eslint-disable-line } catch (err) { expect(err).to.exist() expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) @@ -176,31 +225,29 @@ describe('rendezvous', () => { }) it('discover does not find any register if there is none', async () => { - await connectPeers(peers[0], peers[1]) + await connectPeers(clients[0], rendezvousServer) - for await (const reg of peers[0].rendezvous.discover(namespace)) { // eslint-disable-line + for await (const reg of clients[0].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } - - await peers[1].rendezvous.stop() }) it('discover find registered peer for namespace', async () => { - await connectPeers(peers[0], peers[1]) - await connectPeers(peers[2], peers[1]) + await connectPeers(clients[0], rendezvousServer) + await connectPeers(clients[1], rendezvousServer) const registers = [] // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } // Peer0 register itself on namespace (connected to Peer1) - await peers[0].rendezvous.register(namespace) + await clients[0].rendezvous.register(namespace) // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of peers[2].rendezvous.discover(namespace)) { + for await (const reg of clients[1].rendezvous.discover(namespace)) { registers.push(reg) } @@ -213,25 +260,25 @@ describe('rendezvous', () => { const envelope = await Envelope.openAndCertify(registers[0].signedPeerRecord, PeerRecord.DOMAIN) const rec = PeerRecord.createFromProtobuf(envelope.payload) - expect(rec.multiaddrs).to.eql(peers[0].multiaddrs) + expect(rec.multiaddrs).to.eql(clients[0].multiaddrs) }) it('discover find registered peer for namespace once (cookie usage)', async () => { - await connectPeers(peers[0], peers[1]) - await connectPeers(peers[2], peers[1]) + await connectPeers(clients[0], rendezvousServer) + await connectPeers(clients[1], rendezvousServer) const registers = [] // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } // Peer0 register itself on namespace (connected to Peer1) - await peers[0].rendezvous.register(namespace) + await clients[0].rendezvous.register(namespace) // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of peers[2].rendezvous.discover(namespace)) { + for await (const reg of clients[1].rendezvous.discover(namespace)) { registers.push(reg) } @@ -240,7 +287,7 @@ describe('rendezvous', () => { expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() - for await (const reg of peers[2].rendezvous.discover(namespace)) { + for await (const reg of clients[1].rendezvous.discover(namespace)) { registers.push(reg) } @@ -248,8 +295,9 @@ describe('rendezvous', () => { }) }) - describe('flows with 3 rendezvous all acting as rendezvous point', () => { - let peers + describe('flows with two rendezvous servers available', () => { + let rendezvousServers = [] + let clients const connectPeers = async (peer, otherPeer) => { // Connect each other via relay node @@ -257,83 +305,70 @@ describe('rendezvous', () => { await peer.dial(m) // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) + await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === rendezvousServers.length) } + // Create and start Libp2p nodes beforeEach(async () => { - // Create libp2p nodes - peers = await createPeer({ - number: 3 - }) + // Create Rendezvous Server + rendezvousServers = await Promise.all([ + createRendezvousServer(), + createRendezvousServer() + ]) + + clients = await createPeer({ number: 2 }) - // Create 3 rendezvous peers - peers.forEach((peer) => { - const rendezvous = new Rendezvous({ - libp2p: peer - }) + // Create 2 rendezvous clients + clients.forEach((peer) => { + const rendezvous = new Rendezvous({ libp2p: peer }) rendezvous.start() peer.rendezvous = rendezvous }) // Connect to testing relay node - await Promise.all(peers.map((libp2p) => libp2p.dial(relayAddr))) + await Promise.all(clients.map((libp2p) => libp2p.dial(relayAddr))) + await Promise.all(rendezvousServers.map((libp2p) => libp2p.dial(relayAddr))) }) - afterEach(() => peers.map(async (libp2p) => { - await libp2p.rendezvous.stop() - await libp2p.stop() - })) + afterEach(async () => { + await Promise.all(rendezvousServers.map((libp2p) => libp2p.stop())) + await Promise.all(clients.map((libp2p) => { + libp2p.rendezvous.stop() + return libp2p.stop() + })) + }) - it('discover find registered peer for namespace only when registered', async () => { - await connectPeers(peers[0], peers[1]) - await connectPeers(peers[2], peers[1]) + it('discover find registered peer for namespace only when registered ', async () => { + // Connect all the clients to all the servers + await Promise.all(rendezvousServers.map((server) => + Promise.all(clients.map((client) => connectPeers(client, server))))) const registers = [] - // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + // Client 1 does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } - // Peer0 register itself on namespace (connected to Peer1) - await peers[0].rendezvous.register(namespace) + // Client 0 register itself on namespace (connected to Peer1) + await clients[0].rendezvous.register(namespace) - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of peers[2].rendezvous.discover(namespace)) { + // Client1 discovers Client0 + for await (const reg of clients[1].rendezvous.discover(namespace)) { registers.push(reg) } - expect(registers).to.have.lengthOf(1) + expect(registers[0].signedPeerRecord).to.exist() expect(registers[0].ns).to.eql(namespace) expect(registers[0].ttl).to.exist() - // Peer0 unregister itself on namespace (connected to Peer1) - await peers[0].rendezvous.unregister(namespace) + // Client0 unregister itself on namespace + await clients[0].rendezvous.unregister(namespace) // Peer2 does not discovery any peer registered - for await (const reg of peers[2].rendezvous.discover(namespace)) { // eslint-disable-line + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } }) - - it('discovers locally first, and if limit achieved, not go to the network', async () => { - await connectPeers(peers[0], peers[1]) - await connectPeers(peers[2], peers[1]) - - // Peer0 register itself on namespace (connected to Peer1) - await peers[1].rendezvous.register(namespace) - - const spyRendezvousPoints = sinon.spy(peers[2].rendezvous._rendezvousPoints, 'entries') - - const registers = [] - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of peers[2].rendezvous.discover(namespace, 1)) { - registers.push(reg) - } - - // No need to get the rendezvousPoints connections - expect(spyRendezvousPoints).to.have.property('callCount', 0) - expect(registers).to.have.lengthOf(1) - }) }) }) diff --git a/test/server.spec.js b/test/server.spec.js index 88ce77d..99bdf57 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -14,7 +14,11 @@ const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') -const { createPeer, createPeerId, createSignedPeerRecord } = require('./utils') +const { + createPeerId, + createSignedPeerRecord, + defaultLibp2pConfig +} = require('./utils') const testNamespace = 'test-namespace' const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')] @@ -23,10 +27,9 @@ describe('rendezvous server', () => { const signedPeerRecords = [] let rServer let peerIds - let libp2p before(async () => { - peerIds = await createPeerId({ number: 3 }) + peerIds = await createPeerId({ number: 4 }) // Create a signed peer record per peer for (const peerId of peerIds) { @@ -35,26 +38,34 @@ describe('rendezvous server', () => { } }) - beforeEach(async () => { - [libp2p] = await createPeer() + afterEach(async () => { + rServer && await rServer.stop() }) - afterEach(async () => { - libp2p && await libp2p.stop() - rServer && rServer.stop() + it('can start a rendezvous server', async () => { + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) + + await rServer.start() }) it('can add registrations to multiple namespaces', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(libp2p) + + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) const { registrations: testNsRegistrations } = rServer.getRegistrations(testNamespace) expect(testNsRegistrations).to.have.lengthOf(2) @@ -64,12 +75,15 @@ describe('rendezvous server', () => { }) it('should be able to limit registrations to get', () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) - // Add registration for peer 2 in test namespace rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) let r = rServer.getRegistrations(testNamespace, { limit: 1 }) expect(r.registrations).to.have.lengthOf(1) @@ -81,19 +95,22 @@ describe('rendezvous server', () => { }) it('can remove registrations from a peer in a given namespace', () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) - // Add registration for peer 2 in test namespace rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + // Add registration for peer 2 in test namespace + rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) expect(r.cookie).to.exist() // Remove registration for peer0 - rServer.removeRegistration(testNamespace, peerIds[0]) + rServer.removeRegistration(testNamespace, peerIds[1]) r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) @@ -102,12 +119,16 @@ describe('rendezvous server', () => { it('can remove all registrations from a peer', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(libp2p) + + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) @@ -116,7 +137,7 @@ describe('rendezvous server', () => { expect(otherR.registrations).to.have.lengthOf(1) // Remove all registrations for peer0 - rServer.removePeerRegistrations(peerIds[0]) + rServer.removePeerRegistrations(peerIds[1]) r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(0) @@ -127,33 +148,45 @@ describe('rendezvous server', () => { it('can attempt to remove a registration for a non existent namespace', () => { const otherNamespace = 'other-namespace' - rServer = new RendezvousServer(libp2p) - rServer.removeRegistration(otherNamespace, peerIds[0]) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) + + rServer.removeRegistration(otherNamespace, peerIds[1]) }) it('can attempt to remove a registration for a non existent peer', () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) // Remove registration for peer0 - rServer.removeRegistration(testNamespace, peerIds[1]) + rServer.removeRegistration(testNamespace, peerIds[2]) r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) }) it('gc expired records', async () => { - rServer = new RendezvousServer(libp2p, { gcInterval: 300 }) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }, { gcInterval: 300 }) + + await rServer.start() // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 500) - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) + rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) let r = rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) @@ -169,10 +202,13 @@ describe('rendezvous server', () => { }) it('only new peers should be returned if cookie given', async () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) @@ -183,10 +219,10 @@ describe('rendezvous server', () => { // Validate peer0 const envelope = await Envelope.openAndCertify(registrations[0].signedPeerRecord, PeerRecord.DOMAIN) - expect(envelope.peerId.toString()).to.eql(peerIds[0].toString()) + expect(envelope.peerId.toString()).to.eql(peerIds[1].toString()) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) // Get second registration by using the cookie const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) @@ -198,7 +234,7 @@ describe('rendezvous server', () => { // Validate peer1 const envelope2 = await Envelope.openAndCertify(registrations2[0].signedPeerRecord, PeerRecord.DOMAIN) - expect(envelope2.peerId.toString()).to.eql(peerIds[1].toString()) + expect(envelope2.peerId.toString()).to.eql(peerIds[2].toString()) // If no cookie provided, all registrations are given const { registrations: registrations3 } = rServer.getRegistrations(testNamespace) @@ -207,10 +243,13 @@ describe('rendezvous server', () => { }) it('no new peers should be returned if there are not new peers since latest query', () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) @@ -227,10 +266,13 @@ describe('rendezvous server', () => { }) it('new data for a peer should be returned if registration updated', async () => { - rServer = new RendezvousServer(libp2p) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) @@ -241,10 +283,10 @@ describe('rendezvous server', () => { // Validate peer0 const envelope = await Envelope.openAndCertify(registrations[0].signedPeerRecord, PeerRecord.DOMAIN) - expect(envelope.peerId.toString()).to.eql(peerIds[0].toString()) + expect(envelope.peerId.toString()).to.eql(peerIds[1].toString()) // Add new registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 1000) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get registrations with same cookie and no new registration const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) @@ -256,15 +298,18 @@ describe('rendezvous server', () => { // Validate peer0 const envelope2 = await Envelope.openAndCertify(registrations2[0].signedPeerRecord, PeerRecord.DOMAIN) - expect(envelope2.peerId.toString()).to.eql(peerIds[0].toString()) + expect(envelope2.peerId.toString()).to.eql(peerIds[1].toString()) }) it('garbage collector should remove cookies of discarded records', async () => { - rServer = new RendezvousServer(libp2p, { gcInterval: 300 }) - rServer.start() + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }, { gcInterval: 300 }) + await rServer.start() // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[0], signedPeerRecords[0], 500) + rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) // Get current registrations const { cookie, registrations } = rServer.getRegistrations(testNamespace) @@ -279,7 +324,11 @@ describe('rendezvous server', () => { expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(0) expect(rServer.cookieRegistrations.get(cookie)).to.not.exist() + }) + + describe('protocol', () => { + before(async () => { - rServer.stop() + }) }) }) diff --git a/test/utils.js b/test/utils.js index 39de121..08f7858 100644 --- a/test/utils.js +++ b/test/utils.js @@ -13,6 +13,8 @@ const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') +const RendezvousServer = require('../src/server') + const Peers = require('./fixtures/peers') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') const relayAddr = MULTIADDRS_WEBSOCKETS[0] @@ -25,14 +27,18 @@ const defaultConfig = { } } +module.exports.defaultLibp2pConfig = defaultConfig + /** * Create Perr Id. * @param {Object} [properties] - * @param {number} [properties.number] number of peers (default: 1). + * @param {number} [properties.number = 1] number of peers. * @return {Promise>} */ -async function createPeerId ({ number = 1 }) { - const peerIds = await pTimes(number, (i) => PeerId.createFromJSON(Peers[i])) +async function createPeerId ({ number = 1, fixture = true } = {}) { + const peerIds = await pTimes(number, (i) => fixture + ? PeerId.createFromJSON(Peers[i]) + : PeerId.create()) return peerIds } @@ -42,9 +48,9 @@ module.exports.createPeerId = createPeerId /** * Create libp2p nodes. * @param {Object} [properties] - * @param {Object} [properties.config] - * @param {number} [properties.number] number of peers (default: 1). - * @param {boolean} [properties.started] nodes should start (default: true) + * @param {Object} [properties.config = {}] + * @param {number} [properties.number = 1] number of peers + * @param {boolean} [properties.started = true] nodes should start * @return {Promise>} */ async function createPeer ({ number = 1, started = true, config = {} } = {}) { @@ -67,6 +73,33 @@ async function createPeer ({ number = 1, started = true, config = {} } = {}) { module.exports.createPeer = createPeer +/** + * Create rendezvous server. + * @param {Object} [properties] + * @param {Object} [properties.config = {}] + * @param {boolean} [properties.started = true] node should start + */ +async function createRendezvousServer ({ config = {}, started = true } = {}) { + const [peerId] = await createPeerId({ fixture: false }) + + const rendezvous = new RendezvousServer({ + peerId: peerId, + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultConfig, + ...config + }) + + if (started) { + await rendezvous.start() + } + + return rendezvous +} + +module.exports.createRendezvousServer = createRendezvousServer + async function connectPeers (peer, otherPeer) { // Connect to testing relay node await peer.dial(relayAddr) From ee10d69fe171bb50d509c9e0408eb167e4ba7ea0 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 17 Nov 2020 15:29:19 +0100 Subject: [PATCH 21/38] chore: add docker --- .dockerignore | 4 ++++ Dockerfile | 34 ++++++++++++++++++++++++++++++++++ src/server/utils.js | 4 ++-- 3 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 .dockerignore create mode 100644 Dockerfile diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..0ed9479 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +* +!src +!README.md +!package.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..8624b78 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,34 @@ +FROM node:lts-buster + +# Install deps +RUN apt-get update && apt-get install -y + +# Get dumb-init to allow quit running interactively +RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 && chmod +x /usr/local/bin/dumb-init + +# Setup directories for the `node` user +RUN mkdir -p /home/node/app/rendezvous/node_modules && chown -R node:node /home/node/app/rendezvous + +WORKDIR /home/node/app/rendezvous + +# Install node modules +COPY package.json ./ +# Switch to the node user for installation +USER node +RUN npm install --production + +# Copy over source files under the node user +COPY --chown=node:node ./src ./src +COPY --chown=node:node ./README.md ./ + +# rendezvous defaults to 15002 +EXPOSE 15002 + +# metrics defaults to 8003 +EXPOSE 8003 + +# Available overrides (defaults shown): +# --disableMetrics=false +# Server logging can be enabled via the DEBUG environment variable: +# DEBUG=libp2p:rendezvous:* +CMD [ "/usr/local/bin/dumb-init", "node", "src/server/bin.js"] \ No newline at end of file diff --git a/src/server/utils.js b/src/server/utils.js index b831fe7..2eddcf6 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -2,7 +2,7 @@ const multiaddr = require('multiaddr') -function getAnnounceAddresses(argv) { +function getAnnounceAddresses (argv) { const announceAddr = argv.announceMultiaddrs || argv.am const announceAddresses = announceAddr ? [multiaddr(announceAddr)] : [] @@ -21,7 +21,7 @@ function getAnnounceAddresses(argv) { module.exports.getAnnounceAddresses = getAnnounceAddresses -function getListenAddresses(argv) { +function getListenAddresses (argv) { const listenAddr = argv.listenMultiaddrs || argv.lm || '/ip4/127.0.0.1/tcp/15002/ws' const listenAddresses = [multiaddr(listenAddr)] From 3798fbbeabf863136b1ce274e827309f105b8ea8 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 17 Nov 2020 19:24:58 +0100 Subject: [PATCH 22/38] fix: changed default values and moved them into the server with propert errors and client handling --- LIBP2P.md | 122 +++++++++++++++++++++--- README.md | 131 +------------------------- package.json | 2 +- src/constants.js | 5 +- src/errors.js | 4 +- src/index.js | 64 ++++++++----- src/server/index.js | 93 +++++++++++++----- src/server/rpc/handlers/discover.js | 37 ++++++-- src/server/rpc/handlers/index.js | 1 + src/server/rpc/handlers/register.js | 37 ++++++-- src/server/rpc/handlers/unregister.js | 12 ++- src/server/rpc/index.js | 14 +-- test/rendezvous.spec.js | 2 +- test/server.spec.js | 2 +- test/utils.js | 16 ++-- 15 files changed, 321 insertions(+), 221 deletions(-) diff --git a/LIBP2P.md b/LIBP2P.md index e3ed9dd..48c7d5b 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -4,32 +4,128 @@ The rendezvous protocol can be used in different contexts across libp2p. For usi ## Usage -`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled, as well as its server mode. +`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled and customized. You can configure it through libp2p as follows: ```js const Libp2p = require('libp2p') -const Rendezvous = require('libp2p-rendezvous') const node = await Libp2p.create({ - modules: { - rendezvous: Rendezvous - }, - config: { - rendezvous: { - server: { - enabled: false - } - } + rendezvous: { + enabled: true } }) ``` -While `js-libp2p` supports the rendezvous protocol out of the box through its discovery API, it also provides a rendezvous API that users can interact with. This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to manually discover peers. - ## Libp2p Flow When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server. The rendezvous server can be added to the bootstrap nodes or manually dialed. When a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undialable nodes can register themselves as reachable through that relay. When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. + +## API + +This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to discover peers on a given namespace. + +### Options + +| Name | Type | Description | +|------|------|-------------| +| options | `object` | rendezvous parameters | +| options.enabled | `boolean` | is rendezvous enabled | + +### rendezvous.start + +Register the rendezvous protocol topology into libp2p. + +`rendezvous.start()` + +### rendezvous.stop + +Unregister the rendezvous protocol and the streams with other peers will be closed. + +`rendezvous.stop()` + +### rendezvous.register + +Registers the peer in a given namespace. + +`rendezvous.register(namespace, [options])` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to register | +| [options] | `Object` | rendezvous registrations options | +| [options.ttl=7.2e6] | `number` | registration ttl in ms | + +#### Returns + +| Type | Description | +|------|-------------| +| `Promise` | Remaining ttl value | + +#### Example + +```js +// ... +const ttl = await rendezvous.register(namespace) +``` + +### rendezvous.unregister + +Unregisters the peer from a given namespace. + +`rendezvous.unregister(namespace)` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to unregister | + +#### Returns + +| Type | Description | +|------|-------------| +| `Promise` | Operation resolved | + +#### Example + +```js +// ... +await rendezvous.register(namespace) +await rendezvous.unregister(namespace) +``` + +### rendezvous.discover + +Discovers peers registered under a given namespace. + +`rendezvous.discover(namespace, [limit])` + +#### Parameters + +| Name | Type | Description | +|------|------|-------------| +| namespace | `string` | namespace to discover | +| limit | `number` | limit of peers to discover | + +#### Returns + +| Type | Description | +|------|-------------| +| `AsyncIterable<{ signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | + +#### Example + +```js +// ... +await rendezvous.register(namespace) + +for await (const reg of rendezvous.discover(namespace)) { + console.log(reg.signedPeerRecord, reg.ns, reg.ttl) +} +``` \ No newline at end of file diff --git a/README.md b/README.md index a73a891..eb72676 100644 --- a/README.md +++ b/README.md @@ -19,136 +19,7 @@ See https://github.com/libp2p/specs/tree/master/rendezvous for more details ## Usage -```js -const Libp2p = require('libp2p') -const Rendezvous = require('libp2p-rendezvous') - -const libp2p = await Libp2p.create({ - // check on js-libp2p repo the options to provide -}) -const rendezvous = new Rendezvous({ libp2p }) // Set other options below - -await libp2p.start() -rendezvous.start() - -// ... - -rendezvous.stop() -await libp2p.stop() -``` - -## API - -### constructor - -Creating an instance of Rendezvous. - -`const rendezvous = new Rendezvous({ libp2p })` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| params | `object` | rendezvous parameters | -| params.libp2p | `Libp2p` | a libp2p node instance | -| params.server | `object` | rendezvous server options | -| params.server.enabled | `boolean` | rendezvous server enabled (default: `true`) | -| params.server.gcInterval | `number` | rendezvous garbage collector interval (default: `3e5`) | - -### rendezvous.start - -Register the rendezvous protocol topology into libp2p. The rendezvous server will be started if enabled, as well as the service to keep self registrations available. - -`rendezvous.start()` - -### rendezvous.stop - -Unregister the rendezvous protocol and the streams with other peers will be closed. - -`rendezvous.stop()` - -### rendezvous.register - -Registers the peer in a given namespace. - -`rendezvous.register(namespace, [options])` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to register | -| options | `object` | rendezvous registrations options | -| options.ttl | `number` | registration ttl in ms (default: `7200e3` and minimum `120`) | - -#### Returns - -| Type | Description | -|------|-------------| -| `Promise` | Remaining ttl value | - -#### Example - -```js -// ... -const ttl = await rendezvous.register(namespace) -``` - -### rendezvous.unregister - -Unregisters the peer from a given namespace. - -`rendezvous.unregister(namespace)` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to unregister | - -#### Returns - -| Type | Description | -|------|-------------| -| `Promise` | Operation resolved | - -#### Example - -```js -// ... -await rendezvous.register(namespace) -await rendezvous.unregister(namespace) -``` - -### rendezvous.discover - -Discovers peers registered under a given namespace. - -`rendezvous.discover(namespace, [limit])` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to discover | -| limit | `number` | limit of peers to discover | - -#### Returns - -| Type | Description | -|------|-------------| -| `AsyncIterable<{ signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | - -#### Example - -```js -// ... -await rendezvous.register(namespace) - -for await (const reg of rendezvous.discover(namespace)) { - console.log(reg.signedPeerRecord, reg.ns, reg.ttl) -} -``` +TODO ## Contribute diff --git a/package.json b/package.json index afa01c4..7f35501 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "uint8arrays": "^1.1.0" }, "devDependencies": { - "aegir": "^26.0.0", + "aegir": "^28.2.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.4.0", diff --git a/src/constants.js b/src/constants.js index c818bf2..90f3c95 100644 --- a/src/constants.js +++ b/src/constants.js @@ -2,4 +2,7 @@ exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' exports.MAX_NS_LENGTH = 255 -exports.MAX_LIMIT = 1000 +exports.MAX_DISCOVER_LIMIT = 1000 +exports.MAX_REGISTRATIONS = 1000 +exports.MIN_TTL = 7.2e6 +exports.MAX_TTL = 2.592e+8 diff --git a/src/errors.js b/src/errors.js index 02328b3..3527d5f 100644 --- a/src/errors.js +++ b/src/errors.js @@ -3,5 +3,7 @@ exports.codes = { INVALID_NAMESPACE: 'ERR_INVALID_NAMESPACE', INVALID_TTL: 'ERR_INVALID_TTL', - NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS' + INVALID_LIMIT: 'ERR_INVALID_LIMIT', + NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS', + INVALID_COOKIE: 'ERR_INVALID_COOKIE' } diff --git a/src/index.js b/src/index.js index 6feb99f..e7a31f4 100644 --- a/src/index.js +++ b/src/index.js @@ -15,27 +15,36 @@ const toString = require('uint8arrays/to-string') const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') const { codes: errCodes } = require('./errors') -const { PROTOCOL_MULTICODEC } = require('./constants') +const { + MAX_DISCOVER_LIMIT, + PROTOCOL_MULTICODEC +} = require('./constants') const { Message } = require('./proto') const MESSAGE_TYPE = Message.MessageType /** -* Rendezvous point contains the connection to a rendezvous server, as well as, -* the cookies per namespace that the client received. -* @typedef {Object} RendezvousPoint -* @property {Connection} connection -* @property {Map} cookies -*/ + * @typedef {import('libp2p')} Libp2p + */ /** - * Libp2p Rendezvous. - * A lightweight mechanism for generalized peer discovery. + * Rendezvous point contains the connection to a rendezvous server, as well as, + * the cookies per namespace that the client received. + * + * @typedef {Object} RendezvousPoint + * @property {Connection} connection + * @property {Map} cookies + */ + +/** + * @typedef {Object} RendezvousProperties + * @property {Libp2p} libp2p */ class Rendezvous { /** - * @constructor - * @param {object} params - * @param {Libp2p} params.libp2p + * Libp2p Rendezvous. A lightweight mechanism for generalized peer discovery. + * + * @class + * @param {RendezvousProperties} params */ constructor ({ libp2p }) { this._libp2p = libp2p @@ -54,6 +63,7 @@ class Rendezvous { /** * Register the rendezvous protocol in the libp2p node. + * * @returns {void} */ start () { @@ -78,6 +88,7 @@ class Rendezvous { /** * Unregister the rendezvous protocol and clear the state. + * * @returns {void} */ stop () { @@ -98,9 +109,10 @@ class Rendezvous { /** * Registrar notifies a connection successfully with rendezvous protocol. + * * @private - * @param {PeerId} peerId remote peer-id - * @param {Connection} conn connection to the peer + * @param {PeerId} peerId - remote peer-id + * @param {Connection} conn - connection to the peer */ _onPeerConnected (peerId, conn) { const idB58Str = peerId.toB58String() @@ -111,8 +123,9 @@ class Rendezvous { /** * Registrar notifies a closing connection with rendezvous protocol. + * * @private - * @param {PeerId} peerId peerId + * @param {PeerId} peerId - peerId */ _onPeerDisconnected (peerId) { const idB58Str = peerId.toB58String() @@ -123,20 +136,17 @@ class Rendezvous { /** * Register the peer in a given namespace + * * @param {string} ns * @param {object} [options] - * @param {number} [options.ttl = 7200e3] registration ttl in ms (minimum 120) + * @param {number} [options.ttl = 7.2e6] - registration ttl in ms * @returns {Promise} rendezvous register ttl. */ - async register (ns, { ttl = 7200e3 } = {}) { + async register (ns, { ttl = 7.2e6 } = {}) { if (!ns) { throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) } - if (ttl < 120) { - throw errCode(new Error('a valid ttl must be provided (bigger than 120)'), errCodes.INVALID_TTL) - } - // Are there available rendezvous servers? if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) @@ -171,6 +181,10 @@ class Rendezvous { throw new Error('unexpected message received') } + if (recMessage.registerResponse.status !== Message.ResponseStatus.OK) { + throw errCode(new Error(recMessage.registerResponse.statusText), recMessage.registerResponse.status) + } + return recMessage.registerResponse.ttl * 1e3 // convert to ms } @@ -186,6 +200,7 @@ class Rendezvous { /** * Unregister peer from the nampesapce. + * * @param {string} ns * @returns {Promise} */ @@ -231,6 +246,7 @@ class Rendezvous { /** * Discover peers registered under a given namespace + * * @param {string} ns * @param {number} [limit] * @returns {AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>} @@ -241,6 +257,10 @@ class Rendezvous { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } + if (limit > MAX_DISCOVER_LIMIT) { + throw errCode(new Error(`a smaller limit must be provided (smaller than ${MAX_DISCOVER_LIMIT})`), errCodes.INVALID_LIMIT) + } + const registrationTransformer = (r) => ({ signedPeerRecord: r.signedPeerRecord, ns: r.ns, @@ -277,6 +297,8 @@ class Rendezvous { if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { throw new Error('unexpected message received') + } else if (recMessage.discoverResponse.status !== Message.ResponseStatus.OK) { + throw errCode(new Error(recMessage.discoverResponse.statusText), recMessage.discoverResponse.status) } // Iterate over registrations response diff --git a/src/server/index.js b/src/server/index.js index 6043b6c..e36f586 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -4,50 +4,71 @@ const debug = require('debug') const log = debug('libp2p:rendezvous-server') log.error = debug('libp2p:rendezvous-server:error') +const errCode = require('err-code') + const Libp2p = require('libp2p') const PeerId = require('peer-id') -const { PROTOCOL_MULTICODEC, MAX_LIMIT } = require('../constants') +const { codes: errCodes } = require('../errors') const rpc = require('./rpc') +const { + MIN_TTL, + MAX_TTL, + MAX_NS_LENGTH, + MAX_DISCOVER_LIMIT, + PROTOCOL_MULTICODEC +} = require('../constants') /** -* Rendezvous registration. -* @typedef {Object} Register -* @property {string} ns -* @property {Buffer} signedPeerRecord -* @property {number} ttl -*/ - -/** - * Namespace registration. + * @typedef {Object} Register + * @property {string} ns + * @property {Buffer} signedPeerRecord + * @property {number} ttl + * * @typedef {Object} NamespaceRegistration * @property {string} id * @property {number} expiration */ +/** + * @typedef {Object} RendezvousServerOptions + * @property {number} [gcDelay = 3e5] garbage collector delay (default: 5 minutes) + * @property {number} [gcInterval = 7.2e6] garbage collector interval (default: 2 hours) + * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration + * @property {number} [maxTtl = MAX_TTL] maxium acceptable ttl to store a registration + * @property {number} [maxNsLength = MAX_NS_LENGTH] maxium acceptable namespace length + * @property {number} [maxDiscoverLimit = MAX_DISCOVER_LIMIT] maxium acceptable discover limit + */ + /** * Libp2p rendezvous server. */ class RendezvousServer extends Libp2p { /** - * @constructor - * @param {Libp2pOptions} libp2pOptions - * @param {object} [options] - * @param {number} [options.gcInterval = 3e5] - */ - constructor (libp2pOptions, { gcInterval = 3e5 } = {}) { + * @class + * @param {Libp2pOptions} libp2pOptions + * @param {RendezvousServerOptions} [options] + */ + constructor (libp2pOptions, options = {}) { super(libp2pOptions) - this._gcInterval = gcInterval + this._gcDelay = options.gcDelay || 3e5 + this._gcInterval = options.gcInterval || 7.2e6 + this._minTtl = options.minTtl || MIN_TTL + this._maxTtl = options.maxTtl || MAX_TTL + this._maxNsLength = options.maxNsLength || MAX_NS_LENGTH + this._maxDiscoveryLimit = options.maxDiscoverLimit || MAX_DISCOVER_LIMIT /** * Registrations per namespace. + * * @type {Map>} */ this.nsRegistrations = new Map() /** * Registration ids per cookie. + * * @type {Map>} */ this.cookieRegistrations = new Map() @@ -57,6 +78,7 @@ class RendezvousServer extends Libp2p { /** * Start rendezvous server for handling rendezvous streams and gc. + * * @returns {void} */ start () { @@ -69,36 +91,40 @@ class RendezvousServer extends Libp2p { log('starting') // Garbage collection - this._interval = setInterval(this._gc, this._gcInterval) + this._timeout = setInterval(this._gc, this._gcDelay) // Incoming streams handling - this.registrar.handle(PROTOCOL_MULTICODEC, rpc(this)) + this.handle(PROTOCOL_MULTICODEC, rpc(this)) log('started') } /** * Stops rendezvous server gc and clears registrations + * * @returns {void} */ stop () { - super.stop() + this.unhandle(PROTOCOL_MULTICODEC) - clearInterval(this._interval) + clearTimeout(this._timeout) this._interval = undefined this.nsRegistrations.clear() this.cookieRegistrations.clear() + super.stop() log('stopped') } /** * Garbage collector to removed outdated registrations. + * * @returns {void} */ _gc () { log('gc starting') + // TODO: delete addressBook const now = Date.now() const removedIds = [] @@ -127,10 +153,17 @@ class RendezvousServer extends Libp2p { this.cookieRegistrations.delete(key) } } + + if (!this._timeout) { + return + } + + this._timeout = setInterval(this._gc, this._gcInterval) } /** * Add a peer registration to a namespace. + * * @param {string} ns * @param {PeerId} peerId * @param {Envelope} envelope @@ -153,6 +186,7 @@ class RendezvousServer extends Libp2p { /** * Remove registration of a given namespace to a peer + * * @param {string} ns * @param {PeerId} peerId * @returns {void} @@ -173,6 +207,7 @@ class RendezvousServer extends Libp2p { /** * Remove all registrations of a given peer + * * @param {PeerId} peerId * @returns {void} */ @@ -191,16 +226,28 @@ class RendezvousServer extends Libp2p { /** * Get registrations for a namespace + * * @param {string} ns * @param {object} [options] * @param {number} [options.limit] * @param {string} [options.cookie] * @returns {{ registrations: Array, cookie: string }} */ - getRegistrations (ns, { limit = MAX_LIMIT, cookie = String(Math.random() + Date.now()) } = {}) { + getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { const nsEntry = this.nsRegistrations.get(ns) || new Map() const registrations = [] - const cRegistrations = this.cookieRegistrations.get(cookie) || new Set() + + // Get the cookie registration if provided, create a cookie otherwise + let cRegistrations = new Set() + if (cookie) { + cRegistrations = this.cookieRegistrations.get(cookie) + } else { + cookie = String(Math.random() + Date.now()) + } + + if (!cRegistrations) { + throw errCode(new Error('no registrations for the given cookie'), errCodes.INVALID_COOKIE) + } for (const [idStr, nsReg] of nsEntry.entries()) { if (nsReg.expiration <= Date.now()) { diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index ba9c612..222e071 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -12,8 +12,16 @@ const { Message } = require('../../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus -const { MAX_NS_LENGTH, MAX_LIMIT } = require('../../../constants') +const { codes: errCodes } = require('../../../errors') +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('../..')} RendezvousPoint + */ + +/** + * @param {RendezvousPoint} rendezvousPoint + */ module.exports = (rendezvousPoint) => { /** * Process `Discover` Rendezvous messages. @@ -24,22 +32,24 @@ module.exports = (rendezvousPoint) => { */ return function discover (peerId, msg) { try { - log(`discover ${peerId.toB58String()}: discover on ${msg.discover.ns}`) + const namespace = msg.discover.ns + log(`discover ${peerId.toB58String()}: discover on ${namespace}`) // Validate namespace - if (!msg.discover.ns || msg.discover.ns > MAX_NS_LENGTH) { - log.error(`invalid namespace received: ${msg.discover.ns}`) + if (!namespace || namespace > rendezvousPoint._maxNsLength) { + log.error(`invalid namespace received: ${namespace}`) return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, discoverResponse: { - status: RESPONSE_STATUS.E_INVALID_NAMESPACE + status: RESPONSE_STATUS.E_INVALID_NAMESPACE, + statusText: `invalid namespace received: "${namespace}". It should be smaller than ${rendezvousPoint._maxNsLength}` } } } - if (!msg.discover.limit || msg.discover.limit <= 0 || msg.discover.limit > MAX_LIMIT) { - msg.discover.limit = MAX_LIMIT + if (!msg.discover.limit || msg.discover.limit <= 0 || msg.discover.limit > rendezvousPoint._maxDiscoveryLimit) { + msg.discover.limit = rendezvousPoint._maxDiscoveryLimit } // Get registrations @@ -47,7 +57,8 @@ module.exports = (rendezvousPoint) => { cookie: msg.discover.cookie ? toString(msg.discover.cookie) : undefined, limit: msg.discover.limit } - const { registrations, cookie } = rendezvousPoint.getRegistrations(msg.discover.ns, options) + + const { registrations, cookie } = rendezvousPoint.getRegistrations(namespace, options) return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, @@ -63,6 +74,16 @@ module.exports = (rendezvousPoint) => { } } catch (err) { log.error(err) + + if (err.code === errCodes.INVALID_COOKIE) { + return { + type: MESSAGE_TYPE.DISCOVER_RESPONSE, + discoverResponse: { + status: RESPONSE_STATUS.E_INVALID_COOKIE, + statusText: `invalid cookie received: "${toString(msg.discover.cookie)}"` + } + } + } } return { diff --git a/src/server/rpc/handlers/index.js b/src/server/rpc/handlers/index.js index 89248ab..9daaf91 100644 --- a/src/server/rpc/handlers/index.js +++ b/src/server/rpc/handlers/index.js @@ -12,6 +12,7 @@ module.exports = (server) => { /** * Get the message handler matching the passed in type. + * * @param {number} type * @returns {function(PeerId, Message, function(Error, Message))} */ diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index 4431d53..0c81b8a 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -12,8 +12,14 @@ const { Message } = require('../../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus -const { MAX_NS_LENGTH } = require('../../../constants') +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('../..')} RendezvousPoint + */ +/** + * @param {RendezvousPoint} rendezvousPoint + */ module.exports = (rendezvousPoint) => { /** * Process `Register` Rendezvous messages. @@ -24,20 +30,37 @@ module.exports = (rendezvousPoint) => { */ return async function register (peerId, msg) { try { - log(`register ${peerId.toB58String()}: trying register on ${msg.register.ns}`) + const namespace = msg.register.ns // Validate namespace - if (!msg.register.ns || msg.register.ns > MAX_NS_LENGTH) { - log.error(`invalid namespace received: ${msg.register.ns}`) + if (!namespace || namespace > rendezvousPoint._maxNsLength) { + log.error(`invalid namespace received: ${namespace}`) return { type: MESSAGE_TYPE.REGISTER_RESPONSE, registerResponse: { - status: RESPONSE_STATUS.E_INVALID_NAMESPACE + status: RESPONSE_STATUS.E_INVALID_NAMESPACE, + statusText: `invalid namespace received: "${namespace}". It should be smaller than ${rendezvousPoint._maxNsLength}` } } } + // Validate ttl + const ttl = msg.register.ttl * 1e3 // convert to ms + if (!ttl || ttl < rendezvousPoint._minTtl || ttl > rendezvousPoint._maxTtl) { + log.error(`invalid ttl received: ${ttl}`) + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.E_INVALID_TTL, + statusText: `invalid ttl received: "${ttl}". It should be bigger than ${rendezvousPoint._minTtl} and smaller than ${rendezvousPoint._maxTtl}` + } + } + } + + log(`register ${peerId.toB58String()}: trying register on ${namespace} by ${ttl} ms`) + // Open and verify envelope signature const envelope = await Envelope.openAndCertify(msg.register.signedPeerRecord, PeerRecord.DOMAIN) @@ -55,10 +78,10 @@ module.exports = (rendezvousPoint) => { // Add registration rendezvousPoint.addRegistration( - msg.register.ns, + namespace, peerId, envelope, - msg.register.ttl * 1e3 // convert to ms + ttl ) return { diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js index fbaa1f0..8f9c56c 100644 --- a/src/server/rpc/handlers/unregister.js +++ b/src/server/rpc/handlers/unregister.js @@ -7,6 +7,14 @@ log.error = debug('libp2p:rendezvous:protocol:unregister:error') const equals = require('uint8arrays/equals') +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('../..')} RendezvousPoint + */ + +/** + * @param {RendezvousPoint} rendezvousPoint + */ module.exports = (rendezvousPoint) => { /** * Process `Unregister` Rendezvous messages. @@ -19,13 +27,13 @@ module.exports = (rendezvousPoint) => { log(`unregister ${peerId.toB58String()}: trying unregister from ${msg.unregister.ns}`) if (!msg.unregister.id && !msg.unregister.ns) { - throw new Error('no peerId or namespace provided') + log.error('no peerId or namespace provided') + return } // Validate auth if (!equals(msg.unregister.id, peerId.toBytes())) { log.error('unauthorized peer id to unregister') - return } diff --git a/src/server/rpc/index.js b/src/server/rpc/index.js index 57df8b0..f2269a1 100644 --- a/src/server/rpc/index.js +++ b/src/server/rpc/index.js @@ -15,11 +15,12 @@ module.exports = (rendezvous) => { const getMessageHandler = handlers(rendezvous) /** - * Process incoming Rendezvous messages. - * @param {PeerId} peerId - * @param {Message} msg - * @returns {Promise} - */ + * Process incoming Rendezvous messages. + * + * @param {PeerId} peerId + * @param {Message} msg + * @returns {Promise} + */ function handleMessage (peerId, msg) { const handler = getMessageHandler(msg.type) @@ -33,9 +34,10 @@ module.exports = (rendezvous) => { /** * Handle incoming streams on the rendezvous protocol. + * * @param {Object} props * @param {DuplexStream} props.stream - * @param {Connection} props.connection connection + * @param {Connection} props.connection - connection * @returns {Promise} */ return async function onIncomingStream ({ stream, connection }) { diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index fae542b..5df346b 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -156,7 +156,7 @@ describe('rendezvous', () => { .and.have.property('code', errCodes.INVALID_NAMESPACE) }) - it('register throws error if ttl is too small', async () => { + it.skip('register throws error if ttl is too small', async () => { await expect(clients[0].rendezvous.register(namespace, { ttl: 10 })) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_TTL) diff --git a/test/server.spec.js b/test/server.spec.js index 99bdf57..debe0cb 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -305,7 +305,7 @@ describe('rendezvous server', () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }, { gcInterval: 300 }) + }, { gcDelay: 300, gcInterval: 300 }) await rServer.start() // Add registration for peer 1 in test namespace diff --git a/test/utils.js b/test/utils.js index 08f7858..5b96c02 100644 --- a/test/utils.js +++ b/test/utils.js @@ -31,9 +31,11 @@ module.exports.defaultLibp2pConfig = defaultConfig /** * Create Perr Id. + * * @param {Object} [properties] - * @param {number} [properties.number = 1] number of peers. - * @return {Promise>} + * @param {number} [properties.number = 1] - number of peers. + * @param {boolean} [properties.fixture = true] + * @returns {Promise>} */ async function createPeerId ({ number = 1, fixture = true } = {}) { const peerIds = await pTimes(number, (i) => fixture @@ -47,11 +49,12 @@ module.exports.createPeerId = createPeerId /** * Create libp2p nodes. + * * @param {Object} [properties] * @param {Object} [properties.config = {}] - * @param {number} [properties.number = 1] number of peers - * @param {boolean} [properties.started = true] nodes should start - * @return {Promise>} + * @param {number} [properties.number = 1] - number of peers + * @param {boolean} [properties.started = true] - nodes should start + * @returns {Promise>} */ async function createPeer ({ number = 1, started = true, config = {} } = {}) { const peerIds = await pTimes(number, (i) => PeerId.createFromJSON(Peers[i])) @@ -75,9 +78,10 @@ module.exports.createPeer = createPeer /** * Create rendezvous server. + * * @param {Object} [properties] * @param {Object} [properties.config = {}] - * @param {boolean} [properties.started = true] node should start + * @param {boolean} [properties.started = true] - node should start */ async function createRendezvousServer ({ config = {}, started = true } = {}) { const [peerId] = await createPeerId({ fixture: false }) From cdf2f6b7c0f4cdc3573ca8e1c6c9fef6d57c5509 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 17 Nov 2020 21:34:17 +0100 Subject: [PATCH 23/38] chore: update docs and constants --- README.md | 52 ++++++++++++++++++++++++++++++++++++++++- package.json | 3 +++ src/constants.js | 6 +---- src/index.js | 8 ++----- src/server/constants.js | 8 +++++++ src/server/index.js | 2 +- test/rendezvous.spec.js | 8 +------ 7 files changed, 67 insertions(+), 20 deletions(-) create mode 100644 src/server/constants.js diff --git a/README.md b/README.md index eb72676..d2451d0 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. Any node implementing the rendezvous protocol can act as a rendezvous point, allowing the discovery of relevant peers in a decentralized fashion. -See https://github.com/libp2p/specs/tree/master/rendezvous for more details +See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more details. ## Lead Maintainer @@ -19,6 +19,56 @@ See https://github.com/libp2p/specs/tree/master/rendezvous for more details ## Usage +### Install + +```bash +> npm install --global libp2p-rendezvous +``` + +Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. + +It accepts several arguments: `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsMultiaddr` and `--disableMetrics` + +```sh +libp2p-rendezvous-server [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsMultiaddr ] [--disableMetrics] +``` + +For further customization (e.g. swapping the muxer, using other transports) it is recommended to create a server via the API. + +#### PeerId + +You can create a [PeerId](https://github.com/libp2p/js-peer-id) via its [CLI](https://github.com/libp2p/js-peer-id#cli). + +```sh +libp2p-rendezvous-server --peerId id.json +``` + +#### Multiaddrs + +You can specify the libp2p rendezvous server listen and announce multiaddrs. This server is configured with [libp2p-tcp](https://github.com/libp2p/js-libp2p-tcp) and [libp2p-websockets](https://github.com/libp2p/js-libp2p-websockets) and addresses with this transports should be used. It can always be modified via the API. + +```sh +libp2p-rendezvous-server --peerId id.json --listenMultiaddrs '/ip4/127.0.0.1/tcp/15002/ws' '/ip4/127.0.0.1/tcp/8000' --announceMultiaddrs '/dns4/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' '/dns6/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' +``` + +By default it listens on `/ip4/127.0.0.1/tcp/15002/ws` and has no announce multiaddrs specified. + +#### Metrics + +Metrics are enabled by default on `/ip4/127.0.0.1/tcp/8003` via Prometheus. This address can also be modified with: + +```sh +libp2p-rendezvous-server --metricsMultiaddr '/ip4/127.0.0.1/tcp/8000' +``` + +Moreover, metrics can also be disabled with: + +```sh +libp2p-rendezvous-server --disableMetrics +``` + +## Docker Setup + TODO ## Contribute diff --git a/package.json b/package.json index 7f35501..8423bd9 100644 --- a/package.json +++ b/package.json @@ -4,6 +4,9 @@ "description": "Javascript implementation of the rendezvous protocol for libp2p", "leadMaintainer": "Vasco Santos ", "main": "src/index.js", + "bin": { + "libp2p-rendezvous-server": "src/server/bin.js" + }, "files": [ "dist", "src" diff --git a/src/constants.js b/src/constants.js index 90f3c95..4a2b937 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,8 +1,4 @@ 'use strict' exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' -exports.MAX_NS_LENGTH = 255 -exports.MAX_DISCOVER_LIMIT = 1000 -exports.MAX_REGISTRATIONS = 1000 -exports.MIN_TTL = 7.2e6 -exports.MAX_TTL = 2.592e+8 +exports.MAX_DISCOVER_LIMIT = 50 diff --git a/src/index.js b/src/index.js index e7a31f4..70b4f79 100644 --- a/src/index.js +++ b/src/index.js @@ -248,19 +248,15 @@ class Rendezvous { * Discover peers registered under a given namespace * * @param {string} ns - * @param {number} [limit] + * @param {number} [limit = MAX_DISCOVER_LIMIT] * @returns {AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>} */ - async * discover (ns, limit) { + async * discover (ns, limit = MAX_DISCOVER_LIMIT) { // Are there available rendezvous servers? if (!this._rendezvousPoints.size) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } - if (limit > MAX_DISCOVER_LIMIT) { - throw errCode(new Error(`a smaller limit must be provided (smaller than ${MAX_DISCOVER_LIMIT})`), errCodes.INVALID_LIMIT) - } - const registrationTransformer = (r) => ({ signedPeerRecord: r.signedPeerRecord, ns: r.ns, diff --git a/src/server/constants.js b/src/server/constants.js new file mode 100644 index 0000000..fd8391a --- /dev/null +++ b/src/server/constants.js @@ -0,0 +1,8 @@ +'use strict' + +exports.MAX_NS_LENGTH = 255 +exports.MAX_DISCOVER_LIMIT = 1000 +exports.MAX_REGISTRATIONS = 1000 +exports.MIN_TTL = 7.2e6 +exports.MAX_TTL = 2.592e+8 +exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' diff --git a/src/server/index.js b/src/server/index.js index e36f586..12468b4 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -17,7 +17,7 @@ const { MAX_NS_LENGTH, MAX_DISCOVER_LIMIT, PROTOCOL_MULTICODEC -} = require('../constants') +} = require('./constants') /** * @typedef {Object} Register diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index 5df346b..bbbd056 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -122,7 +122,7 @@ describe('rendezvous', () => { }) }) - describe('api', () => { + describe('api with one rendezvous server', () => { let rendezvousServer let clients @@ -156,12 +156,6 @@ describe('rendezvous', () => { .and.have.property('code', errCodes.INVALID_NAMESPACE) }) - it.skip('register throws error if ttl is too small', async () => { - await expect(clients[0].rendezvous.register(namespace, { ttl: 10 })) - .to.eventually.rejected() - .and.have.property('code', errCodes.INVALID_TTL) - }) - it('register throws error if no connected rendezvous servers', async () => { await expect(clients[0].rendezvous.register(namespace)) .to.eventually.rejected() From 9fe06913d4823a5e3c6994fdf34e5f652c25b4f1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Wed, 18 Nov 2020 21:15:55 +0100 Subject: [PATCH 24/38] chore: add tests for protocol with direct connection to server --- package.json | 2 +- src/server/rpc/handlers/discover.js | 2 +- src/server/rpc/handlers/register.js | 2 +- test/rendezvous.spec.js | 65 +++++++- test/server.spec.js | 249 ++++++++++++++++++++++++++++ 5 files changed, 315 insertions(+), 5 deletions(-) diff --git a/package.json b/package.json index 8423bd9..835731b 100644 --- a/package.json +++ b/package.json @@ -47,7 +47,7 @@ "it-buffer": "^0.1.2", "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", - "libp2p": "^0.29.0", + "libp2p": "libp2p/js-libp2p#0.30.x", "libp2p-interfaces": "^0.5.1", "libp2p-mplex": "^0.10.0", "libp2p-noise": "^2.0.1", diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index 222e071..e3760b7 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -36,7 +36,7 @@ module.exports = (rendezvousPoint) => { log(`discover ${peerId.toB58String()}: discover on ${namespace}`) // Validate namespace - if (!namespace || namespace > rendezvousPoint._maxNsLength) { + if (!namespace || namespace.length > rendezvousPoint._maxNsLength) { log.error(`invalid namespace received: ${namespace}`) return { diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index 0c81b8a..237f603 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -33,7 +33,7 @@ module.exports = (rendezvousPoint) => { const namespace = msg.register.ns // Validate namespace - if (!namespace || namespace > rendezvousPoint._maxNsLength) { + if (!namespace || namespace.length > rendezvousPoint._maxNsLength) { log.error(`invalid namespace received: ${namespace}`) return { diff --git a/test/rendezvous.spec.js b/test/rendezvous.spec.js index bbbd056..5690845 100644 --- a/test/rendezvous.spec.js +++ b/test/rendezvous.spec.js @@ -6,6 +6,7 @@ chai.use(require('dirty-chai')) chai.use(require('chai-as-promised')) const { expect } = chai const sinon = require('sinon') + const pWaitFor = require('p-wait-for') const multiaddr = require('multiaddr') @@ -15,9 +16,13 @@ const PeerRecord = require('libp2p/src/record/peer-record') const Rendezvous = require('../src') const { codes: errCodes } = require('../src/errors') +const { Message } = require('../src/proto') +const RESPONSE_STATUS = Message.ResponseStatus + const { createPeer, createRendezvousServer, + createSignedPeerRecord, connectPeers } = require('./utils') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') @@ -142,6 +147,7 @@ describe('rendezvous', () => { }) afterEach(async () => { + sinon.restore() await rendezvousServer.stop() for (const peer of clients) { @@ -165,7 +171,6 @@ describe('rendezvous', () => { it('register to a connected rendezvous server node', async () => { await connectPeers(clients[0], rendezvousServer) - // Register expect(rendezvousServer.nsRegistrations.size).to.eql(0) await clients[0].rendezvous.register(namespace) @@ -173,13 +178,55 @@ describe('rendezvous', () => { expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() }) + it('register throws an error with an invalid namespace', async () => { + const badNamespace = 'x'.repeat(300) + await connectPeers(clients[0], rendezvousServer) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + + await expect(clients[0].rendezvous.register(badNamespace)) + .to.eventually.rejected() + .and.have.property('code', RESPONSE_STATUS.E_INVALID_NAMESPACE) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + }) + + it('register throws an error with an invalid ttl', async () => { + const badTtl = 5e10 + await connectPeers(clients[0], rendezvousServer) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + + await expect(clients[0].rendezvous.register(namespace, { ttl: badTtl })) + .to.eventually.rejected() + .and.have.property('code', RESPONSE_STATUS.E_INVALID_TTL) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + }) + + it('register throws an error with an invalid peerId', async () => { + const badSignedPeerRecord = await createSignedPeerRecord(clients[1].peerId, [multiaddr('/ip4/127.0.0.1/tcp/100')]) + await connectPeers(clients[0], rendezvousServer) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + + const stub = sinon.stub(clients[0].peerStore.addressBook, 'getRawEnvelope') + stub.onCall(0).returns(badSignedPeerRecord.marshal()) + + await expect(clients[0].rendezvous.register(namespace)) + .to.eventually.rejected() + .and.have.property('code', RESPONSE_STATUS.E_NOT_AUTHORIZED) + + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + }) + it('unregister throws if a namespace is not provided', async () => { await expect(clients[0].rendezvous.unregister()) .to.eventually.rejected() .and.have.property('code', errCodes.INVALID_NAMESPACE) }) - it('register throws error if no connected rendezvous servers', async () => { + it('unregister throws error if no connected rendezvous servers', async () => { await expect(clients[0].rendezvous.unregister(namespace)) .to.eventually.rejected() .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) @@ -218,6 +265,20 @@ describe('rendezvous', () => { throw new Error('discover should throw error if a namespace is not provided') }) + it('discover throws error if a namespace is invalid', async () => { + const badNamespace = 'x'.repeat(300) + + await connectPeers(clients[0], rendezvousServer) + try { + for await (const _ of clients[0].rendezvous.discover(badNamespace)) { } // eslint-disable-line + } catch (err) { + expect(err).to.exist() + expect(err.code).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) + return + } + throw new Error('discover should throw error if a namespace is not provided') + }) + it('discover does not find any register if there is none', async () => { await connectPeers(clients[0], rendezvousServer) diff --git a/test/server.spec.js b/test/server.spec.js index debe0cb..1e9efa4 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -7,12 +7,24 @@ chai.use(require('chai-as-promised')) const { expect } = chai const delay = require('delay') +const pipe = require('it-pipe') +const lp = require('it-length-prefixed') +const { collect } = require('streaming-iterables') +const { toBuffer } = require('it-buffer') const multiaddr = require('multiaddr') +const PeerId = require('peer-id') +const Libp2p = require('libp2p') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') +const { + PROTOCOL_MULTICODEC +} = require('../src/server/constants') +const { Message } = require('../src/proto') +const MESSAGE_TYPE = Message.MessageType +const RESPONSE_STATUS = Message.ResponseStatus const { createPeerId, @@ -20,6 +32,9 @@ const { defaultLibp2pConfig } = require('./utils') +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + const testNamespace = 'test-namespace' const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')] @@ -301,6 +316,20 @@ describe('rendezvous server', () => { expect(envelope2.peerId.toString()).to.eql(peerIds[1].toString()) }) + it('get registrations should throw if no stored cookie is provided', () => { + const badCookie = String(Math.random() + Date.now()) + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }) + + const fn = () => { + rServer.getRegistrations(testNamespace, { cookie: badCookie }) + } + + expect(fn).to.throw('no registrations for the given cookie') + }) + it('garbage collector should remove cookies of discarded records', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, @@ -327,8 +356,228 @@ describe('rendezvous server', () => { }) describe('protocol', () => { + const ns = 'test-ns' + const ttl = 7.2e6 * 1e-3 + + let rServer + let client + let peerIds + let multiaddrServer + before(async () => { + peerIds = await createPeerId({ number: 4 }) + }) + + // Create client and server and connect them + beforeEach(async () => { + rServer = new RendezvousServer({ + peerId: peerIds[0], + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerIds[0].toB58String()}`) + + client = await Libp2p.create({ + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + + await Promise.all([rServer, client].map((n) => n.start())) + }) + + afterEach(async () => { + await Promise.all([rServer, client].map((n) => n.stop())) + }) + + it('can register a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(Message.ResponseStatus.OK) + + expect(rServer.nsRegistrations.size).to.eql(1) + }) + + it('fails to register if invalid namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns: 'x'.repeat(300), + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('fails to register if invalid ttl', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl: 5e10 * 1e-3 + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_TTL) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('fails to register if invalid signed peer record', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(PeerId.createFromCID(relayAddr.getPeerId())), + ns, + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) + }) + describe('with previous registrations', () => { + beforeEach(async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + })], + lp.encode(), + stream, + async (source) => { + for await (const _ of source) { } // eslint-disable-line + } + ) + + expect(rServer.nsRegistrations.size).to.eql(1) + }) + + it('can unregister a namespace', async () => { + expect(rServer.nsRegistrations.size).to.eql(1) + + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + await pipe( + [Message.encode({ + type: MESSAGE_TYPE.UNREGISTER, + unregister: { + id: client.peerId.toBytes(), + ns + } + })], + lp.encode(), + stream, + async (source) => { + for await (const _ of source) { } // eslint-disable-line + } + ) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('can discover a peer registered into a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.DISCOVER, + discover: { + ns, + limit: 50 + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.DISCOVER_RESPONSE) + expect(recMessage.discoverResponse.status).to.eql(Message.ResponseStatus.OK) + expect(recMessage.discoverResponse.registrations).to.exist() + expect(recMessage.discoverResponse.registrations).to.have.lengthOf(1) + }) }) }) }) From 52fa2bd2a186e24ca7674bcb868ee3d0ca94ebe4 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 19 Nov 2020 12:07:41 +0100 Subject: [PATCH 25/38] chore: DoS protection with max registrations --- src/server/index.js | 29 +++++++++-- src/server/rpc/handlers/register.js | 15 ++++++ test/server.spec.js | 76 +++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 4 deletions(-) diff --git a/src/server/index.js b/src/server/index.js index 12468b4..aec7303 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -16,6 +16,7 @@ const { MAX_TTL, MAX_NS_LENGTH, MAX_DISCOVER_LIMIT, + MAX_REGISTRATIONS, PROTOCOL_MULTICODEC } = require('./constants') @@ -26,7 +27,7 @@ const { * @property {number} ttl * * @typedef {Object} NamespaceRegistration - * @property {string} id + * @property {string} id random generated id to map cookies * @property {number} expiration */ @@ -37,7 +38,8 @@ const { * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration * @property {number} [maxTtl = MAX_TTL] maxium acceptable ttl to store a registration * @property {number} [maxNsLength = MAX_NS_LENGTH] maxium acceptable namespace length - * @property {number} [maxDiscoverLimit = MAX_DISCOVER_LIMIT] maxium acceptable discover limit + * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maxium acceptable discover limit + * @property {number} [maxRegistrations = MAX_REGISTRATIONS] maxium acceptable registrations per peer */ /** @@ -57,10 +59,11 @@ class RendezvousServer extends Libp2p { this._minTtl = options.minTtl || MIN_TTL this._maxTtl = options.maxTtl || MAX_TTL this._maxNsLength = options.maxNsLength || MAX_NS_LENGTH - this._maxDiscoveryLimit = options.maxDiscoverLimit || MAX_DISCOVER_LIMIT + this._maxDiscoveryLimit = options.maxDiscoveryLimit || MAX_DISCOVER_LIMIT + this._maxRegistrations = options.maxRegistrations || MAX_REGISTRATIONS /** - * Registrations per namespace. + * Registrations per namespace, where a registration maps peer id strings to a namespace reg. * * @type {Map>} */ @@ -283,6 +286,24 @@ class RendezvousServer extends Libp2p { cookie } } + + /** + * Get all the namespaces a given peer has registrations. + * + * @param {PeerId} peerId + * @returns {Array} + */ + getRegistrationsFromPeer (peerId) { + const namespaces = [] + + this.nsRegistrations.forEach((nsEntry, namespace) => { + if (nsEntry.has(peerId.toB58String())) { + namespaces.push(namespace) + } + }) + + return namespaces + } } module.exports = RendezvousServer diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index 237f603..d8b0835 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -59,6 +59,21 @@ module.exports = (rendezvousPoint) => { } } + // Now check how many registrations we have for this peer + // simple limit to defend against trivial DoS attacks + // example: a peer connects and keeps registering until it fills our memory + const peerRegistrations = rendezvousPoint.getRegistrationsFromPeer(peerId) + if (peerRegistrations.length >= rendezvousPoint._maxRegistrations) { + log.error('unauthorized peer to register, too many registrations') + + return { + type: MESSAGE_TYPE.REGISTER_RESPONSE, + registerResponse: { + status: RESPONSE_STATUS.E_NOT_AUTHORIZED + } + } + } + log(`register ${peerId.toB58String()}: trying register on ${namespace} by ${ttl} ms`) // Open and verify envelope signature diff --git a/test/server.spec.js b/test/server.spec.js index 1e9efa4..8ef1067 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -580,4 +580,80 @@ describe('rendezvous server', () => { }) }) }) + + describe('DoS attack protection', () => { + const ns = 'test-ns' + const ttl = 7.2e6 * 1e-3 + + let rServer + let client + let peerId + let multiaddrServer + + // Create client and server and connect them + beforeEach(async () => { + [peerId] = await createPeerId() + + rServer = new RendezvousServer({ + peerId: peerId, + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }, { maxRegistrations: 1 }) // Maximum of one registration + + multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerId.toB58String()}`) + + client = await Libp2p.create({ + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + + await Promise.all([rServer, client].map((n) => n.start())) + }) + + afterEach(async () => { + await Promise.all([rServer, client].map((n) => n.stop())) + }) + + it('can register a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const responses = await pipe( + [ + Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + }), + Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + }) + ], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + expect(rServer.nsRegistrations.size).to.eql(1) + + const recMessage = Message.decode(responses[1]) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) + }) + }) }) From 06d53ac334ad295d82e69080eda004ec2c6fd450 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sat, 21 Nov 2020 15:55:19 +0100 Subject: [PATCH 26/38] chore: refactor client --- package.json | 1 - src/errors.js | 5 +- src/index.js | 112 ++++--- src/server/errors.js | 5 + src/server/index.js | 2 +- src/server/rpc/handlers/discover.js | 2 +- .../api.spec.js} | 162 +++------ test/client/connectivity.spec.js | 68 ++++ test/client/lifecycle.spec.js | 62 ++++ test/dos-attack-protection.spec.js | 107 ++++++ test/protocol.spec.js | 258 ++++++++++++++ test/server.spec.js | 317 ------------------ 12 files changed, 618 insertions(+), 483 deletions(-) create mode 100644 src/server/errors.js rename test/{rendezvous.spec.js => client/api.spec.js} (76%) create mode 100644 test/client/connectivity.spec.js create mode 100644 test/client/lifecycle.spec.js create mode 100644 test/dos-attack-protection.spec.js create mode 100644 test/protocol.spec.js diff --git a/package.json b/package.json index 835731b..652d4e5 100644 --- a/package.json +++ b/package.json @@ -48,7 +48,6 @@ "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", "libp2p": "libp2p/js-libp2p#0.30.x", - "libp2p-interfaces": "^0.5.1", "libp2p-mplex": "^0.10.0", "libp2p-noise": "^2.0.1", "libp2p-tcp": "^0.15.1", diff --git a/src/errors.js b/src/errors.js index 3527d5f..9b0e8e3 100644 --- a/src/errors.js +++ b/src/errors.js @@ -2,8 +2,5 @@ exports.codes = { INVALID_NAMESPACE: 'ERR_INVALID_NAMESPACE', - INVALID_TTL: 'ERR_INVALID_TTL', - INVALID_LIMIT: 'ERR_INVALID_LIMIT', - NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS', - INVALID_COOKIE: 'ERR_INVALID_COOKIE' + NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS' } diff --git a/src/index.js b/src/index.js index 70b4f79..022a4a8 100644 --- a/src/index.js +++ b/src/index.js @@ -12,7 +12,7 @@ const { toBuffer } = require('it-buffer') const fromString = require('uint8arrays/from-string') const toString = require('uint8arrays/to-string') -const MulticodecTopology = require('libp2p-interfaces/src/topology/multicodec-topology') +const PeerId = require('peer-id') const { codes: errCodes } = require('./errors') const { @@ -27,11 +27,9 @@ const MESSAGE_TYPE = Message.MessageType */ /** - * Rendezvous point contains the connection to a rendezvous server, as well as, - * the cookies per namespace that the client received. + * Rendezvous point contains the cookies per namespace that the client received. * * @typedef {Object} RendezvousPoint - * @property {Connection} connection * @property {Map} cookies */ @@ -44,21 +42,24 @@ class Rendezvous { * Libp2p Rendezvous. A lightweight mechanism for generalized peer discovery. * * @class - * @param {RendezvousProperties} params + * @param {RendezvousProperties & RendezvousOptions} params */ - constructor ({ libp2p }) { + constructor ({ libp2p, maxRendezvousPoints }) { this._libp2p = libp2p this._peerId = libp2p.peerId - this._registrar = libp2p.registrar + this._peerStore = libp2p.peerStore + this._connectionManager = libp2p.connectionManager + + this._maxRendezvousPoints = maxRendezvousPoints + + this._isStarted = false /** * @type {Map} */ this._rendezvousPoints = new Map() - this._registrarId = undefined - this._onPeerConnected = this._onPeerConnected.bind(this) - this._onPeerDisconnected = this._onPeerDisconnected.bind(this) + this._onProtocolChange = this._onProtocolChange.bind(this) } /** @@ -67,71 +68,63 @@ class Rendezvous { * @returns {void} */ start () { - if (this._registrarId) { + if (this._isStarted) { return } log('starting') - // register protocol with topology - const topology = new MulticodecTopology({ - multicodecs: PROTOCOL_MULTICODEC, - handlers: { - onConnect: this._onPeerConnected, - onDisconnect: this._onPeerDisconnected - } - }) - this._registrarId = this._registrar.register(topology) + this._peerStore.on('change:protocols', this._onProtocolChange) + this._isStarted = true log('started') } /** - * Unregister the rendezvous protocol and clear the state. + * Clear the rendezvous state and remove listeners. * * @returns {void} */ stop () { - if (!this._registrarId) { + if (!this._isStarted) { return } log('stopping') - // unregister protocol and handlers - this._registrar.unregister(this._registrarId) - - this._registrarId = undefined + this._peerStore.removeListener('change:protocols', this._onProtocolChange) this._rendezvousPoints.clear() + this._isStarted = false log('stopped') } /** - * Registrar notifies a connection successfully with rendezvous protocol. - * - * @private - * @param {PeerId} peerId - remote peer-id - * @param {Connection} conn - connection to the peer - */ - _onPeerConnected (peerId, conn) { - const idB58Str = peerId.toB58String() - log('connected', idB58Str) - - this._rendezvousPoints.set(idB58Str, { connection: conn }) - } - - /** - * Registrar notifies a closing connection with rendezvous protocol. + * Check if a peer supports the rendezvous protocol. + * If the protocol is not supported, check if it was supported before and remove it as a rendezvous point. + * If the protocol is supported, add it to the known rendezvous points. * - * @private - * @param {PeerId} peerId - peerId + * @param {Object} props + * @param {PeerId} props.peerId + * @param {Array} props.protocols + * @returns {void} */ - _onPeerDisconnected (peerId) { - const idB58Str = peerId.toB58String() - log('disconnected', idB58Str) + _onProtocolChange ({ peerId, protocols }) { + const id = peerId.toB58String() + + // Check if it has the protocol + const hasProtocol = protocols.find(protocol => protocol === PROTOCOL_MULTICODEC) + const hasRendezvousPoint = this._rendezvousPoints.has(id) + + // If no protocol, check if we were keeping the peer before + if (!hasProtocol && hasRendezvousPoint) { + this._rendezvousPoints.delete(id) + log(`removed ${id} from rendezvous points as it does not suport ${PROTOCOL_MULTICODEC} anymore`) + } else if (hasProtocol && !this._rendezvousPoints.has(id)) { + this._rendezvousPoints.set(id, { cookies: new Map() }) + } - this._rendezvousPoints.delete(idB58Str) + // TODO: Hint that connection can be discarded? } /** @@ -152,6 +145,10 @@ class Rendezvous { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } + // TODO: we should protect from getting to many rendezvous points and sending to all + // Should we have a custom max number of servers and a custom sorter function? + // Default to peers already connected + const message = Message.encode({ type: MESSAGE_TYPE.REGISTER, register: { @@ -163,7 +160,7 @@ class Rendezvous { const registerTasks = [] const taskFn = async (id) => { - const { connection } = this._rendezvousPoints.get(id) + const connection = await this._libp2p.dial(PeerId.createFromCID(id)) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( @@ -175,6 +172,10 @@ class Rendezvous { collect ) + if (!connection.streams.length) { + await connection.close() + } + const recMessage = Message.decode(response) if (!recMessage.type === MESSAGE_TYPE.REGISTER_RESPONSE) { @@ -193,6 +194,7 @@ class Rendezvous { } // Return first ttl + // pAny here? const [returnTtl] = await Promise.all(registerTasks) return returnTtl @@ -224,7 +226,7 @@ class Rendezvous { const unregisterTasks = [] const taskFn = async (id) => { - const { connection } = this._rendezvousPoints.get(id) + const connection = await this._libp2p.dial(PeerId.createFromCID(id)) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) await pipe( @@ -235,6 +237,10 @@ class Rendezvous { for await (const _ of source) { } // eslint-disable-line } ) + + if (!connection.streams.length) { + await connection.close() + } } for (const id of this._rendezvousPoints.keys()) { @@ -279,7 +285,8 @@ class Rendezvous { }) // Send discover message and wait for response - const { stream } = await rp.connection.newStream(PROTOCOL_MULTICODEC) + const connection = await this._libp2p.dial(PeerId.createFromCID(id)) + const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( [message], lp.encode(), @@ -289,6 +296,10 @@ class Rendezvous { collect ) + if (!connection.streams.length) { + await connection.close() + } + const recMessage = Message.decode(response) if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { @@ -305,7 +316,6 @@ class Rendezvous { // Store cookie rpCookies.set(ns, toString(recMessage.discoverResponse.cookie)) this._rendezvousPoints.set(id, { - connection: rp.connection, cookies: rpCookies }) diff --git a/src/server/errors.js b/src/server/errors.js new file mode 100644 index 0000000..efb6560 --- /dev/null +++ b/src/server/errors.js @@ -0,0 +1,5 @@ +'use strict' + +exports.codes = { + INVALID_COOKIE: 'ERR_INVALID_COOKIE' +} diff --git a/src/server/index.js b/src/server/index.js index aec7303..78751f6 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -9,7 +9,7 @@ const errCode = require('err-code') const Libp2p = require('libp2p') const PeerId = require('peer-id') -const { codes: errCodes } = require('../errors') +const { codes: errCodes } = require('./errors') const rpc = require('./rpc') const { MIN_TTL, diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index e3760b7..cccd1ab 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -12,7 +12,7 @@ const { Message } = require('../../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus -const { codes: errCodes } = require('../../../errors') +const { codes: errCodes } = require('../../errors') /** * @typedef {import('peer-id')} PeerId diff --git a/test/rendezvous.spec.js b/test/client/api.spec.js similarity index 76% rename from test/rendezvous.spec.js rename to test/client/api.spec.js index 5690845..36a249e 100644 --- a/test/rendezvous.spec.js +++ b/test/client/api.spec.js @@ -13,10 +13,10 @@ const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') -const Rendezvous = require('../src') -const { codes: errCodes } = require('../src/errors') +const Rendezvous = require('../../src') +const { codes: errCodes } = require('../../src/errors') -const { Message } = require('../src/proto') +const { Message } = require('../../src/proto') const RESPONSE_STATUS = Message.ResponseStatus const { @@ -24,110 +24,14 @@ const { createRendezvousServer, createSignedPeerRecord, connectPeers -} = require('./utils') -const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +} = require('../utils') +const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') const relayAddr = MULTIADDRS_WEBSOCKETS[0] const namespace = 'ns' -describe('rendezvous', () => { - describe('start and stop', () => { - let peer, rendezvous - - beforeEach(async () => { - [peer] = await createPeer() - rendezvous = new Rendezvous({ libp2p: peer }) - }) - - afterEach(async () => { - await rendezvous.stop() - await peer.stop() - }) - - it('can be started and stopped', async () => { - const spyRegister = sinon.spy(peer.registrar, 'register') - const spyUnregister = sinon.spy(peer.registrar, 'unregister') - - await rendezvous.start() - await rendezvous.stop() - - expect(spyRegister).to.have.property('callCount', 1) - expect(spyUnregister).to.have.property('callCount', 1) - }) - - it('registers the protocol once, if multiple starts', async () => { - const spyRegister = sinon.spy(peer.registrar, 'register') - - await rendezvous.start() - await rendezvous.start() - - expect(spyRegister).to.have.property('callCount', 1) - - await rendezvous.stop() - }) - - it('only unregisters on stop if already started', async () => { - const spyUnregister = sinon.spy(peer.registrar, 'unregister') - - await rendezvous.stop() - - expect(spyUnregister).to.have.property('callCount', 0) - }) - }) - - describe('connectivity', () => { - let rendezvousServer - let client - - // Create and start Libp2p nodes - beforeEach(async () => { - // Create Rendezvous Server - rendezvousServer = await createRendezvousServer() - - // Create Rendezvous client - ;[client] = await createPeer() - - const rendezvous = new Rendezvous({ libp2p: client }) - client.rendezvous = rendezvous - client.rendezvous.start() - }) - - // Connect nodes to the testing relay node - beforeEach(async () => { - await rendezvousServer.dial(relayAddr) - await client.dial(relayAddr) - }) - - afterEach(async () => { - await rendezvousServer.stop() - await client.rendezvous.stop() - await client.stop() - }) - - it('updates known rendezvous points', async () => { - expect(client.rendezvous._rendezvousPoints.size).to.equal(0) - - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${rendezvousServer.peerId.toB58String()}`) - const connection = await client.dial(m) - - expect(client.peerStore.peers.size).to.equal(2) - expect(rendezvousServer.peerStore.peers.size).to.equal(2) - - // Wait event propagation - // Relay peer is not with rendezvous enabled - await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 1) - - expect(client.rendezvous._rendezvousPoints.get(rendezvousServer.peerId.toB58String())).to.exist() - - await connection.close() - - // Wait event propagation - await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 0) - }) - }) - - describe('api with one rendezvous server', () => { +describe('rendezvous api', () => { + describe('one rendezvous server', () => { let rendezvousServer let clients @@ -178,6 +82,17 @@ describe('rendezvous', () => { expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() }) + it('register opens and closes connection on register', async () => { + await connectPeers(clients[0], rendezvousServer) + + clients[0].hangUp(rendezvousServer.peerId) + + const connsBeforeRegister = clients[0].connectionManager.size + await clients[0].rendezvous.register(namespace) + + expect(clients[0].connectionManager.size).to.eql(connsBeforeRegister) + }) + it('register throws an error with an invalid namespace', async () => { const badNamespace = 'x'.repeat(300) await connectPeers(clients[0], rendezvousServer) @@ -247,6 +162,20 @@ describe('rendezvous', () => { expect(rendezvousServer.nsRegistrations.size).to.eql(0) }) + it('unregister opens and closes connection on register', async () => { + await connectPeers(clients[0], rendezvousServer) + + // Register + await clients[0].rendezvous.register(namespace) + expect(rendezvousServer.nsRegistrations.size).to.eql(1) + + const connsBeforeRegister = clients[0].connectionManager.size + await clients[0].rendezvous.unregister(namespace) + expect(rendezvousServer.nsRegistrations.size).to.eql(0) + + expect(clients[0].connectionManager.size).to.eql(connsBeforeRegister) + }) + it('unregister to a connected rendezvous server node not fails if not registered', async () => { await connectPeers(clients[0], rendezvousServer) @@ -256,7 +185,7 @@ describe('rendezvous', () => { it('discover throws error if a namespace is not provided', async () => { try { - for await (const _ of clients[0].rendezvous.discover()) {} // eslint-disable-line + for await (const _ of clients[0].rendezvous.discover()) { } // eslint-disable-line } catch (err) { expect(err).to.exist() expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) @@ -287,7 +216,7 @@ describe('rendezvous', () => { } }) - it('discover find registered peer for namespace', async () => { + it('discover finds registered peer for namespace', async () => { await connectPeers(clients[0], rendezvousServer) await connectPeers(clients[1], rendezvousServer) @@ -318,14 +247,31 @@ describe('rendezvous', () => { expect(rec.multiaddrs).to.eql(clients[0].multiaddrs) }) - it('discover find registered peer for namespace once (cookie usage)', async () => { + it('discover opens and closes connection on register', async () => { + await connectPeers(clients[0], rendezvousServer) + await connectPeers(clients[1], rendezvousServer) + + // Register + await clients[0].rendezvous.register(namespace) + expect(rendezvousServer.nsRegistrations.size).to.eql(1) + + // Hangup from one + clients[1].hangUp(rendezvousServer.peerId) + const connsBeforeRegister = clients[1].connectionManager.size + + for await (const _ of clients[1].rendezvous.discover(namespace)) {} // eslint-disable-line + + expect(clients[1].connectionManager.size).to.eql(connsBeforeRegister) + }) + + it('discover finds registered peer for namespace once (cookie usage)', async () => { await connectPeers(clients[0], rendezvousServer) await connectPeers(clients[1], rendezvousServer) const registers = [] // Peer2 does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + for await (const _ of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } @@ -350,7 +296,7 @@ describe('rendezvous', () => { }) }) - describe('flows with two rendezvous servers available', () => { + describe('multiple rendezvous servers available', () => { let rendezvousServers = [] let clients diff --git a/test/client/connectivity.spec.js b/test/client/connectivity.spec.js new file mode 100644 index 0000000..bd0ed05 --- /dev/null +++ b/test/client/connectivity.spec.js @@ -0,0 +1,68 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai + +const pWaitFor = require('p-wait-for') +const multiaddr = require('multiaddr') + +const Rendezvous = require('../../src') + +const { + createPeer, + createRendezvousServer +} = require('../utils') +const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + +describe('rendezvous connectivity', () => { + let rendezvousServer + let client + + // Create and start Libp2p nodes + beforeEach(async () => { + // Create Rendezvous Server + rendezvousServer = await createRendezvousServer() + + // Create Rendezvous client + ;[client] = await createPeer() + + const rendezvous = new Rendezvous({ libp2p: client }) + client.rendezvous = rendezvous + client.rendezvous.start() + }) + + // Connect nodes to the testing relay node + beforeEach(async () => { + await rendezvousServer.dial(relayAddr) + await client.dial(relayAddr) + }) + + afterEach(async () => { + await rendezvousServer.stop() + await client.rendezvous.stop() + await client.stop() + }) + + it('updates known rendezvous points', async () => { + expect(client.rendezvous._rendezvousPoints.size).to.equal(0) + + // Connect each other via relay node + const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${rendezvousServer.peerId.toB58String()}`) + const connection = await client.dial(m) + + expect(client.peerStore.peers.size).to.equal(2) + expect(rendezvousServer.peerStore.peers.size).to.equal(2) + + // Wait event propagation + // Relay peer is not with rendezvous enabled + await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 1) + + expect(client.rendezvous._rendezvousPoints.get(rendezvousServer.peerId.toB58String())).to.exist() + + await connection.close() + }) +}) diff --git a/test/client/lifecycle.spec.js b/test/client/lifecycle.spec.js new file mode 100644 index 0000000..e06a6dc --- /dev/null +++ b/test/client/lifecycle.spec.js @@ -0,0 +1,62 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai +const sinon = require('sinon') + +const Rendezvous = require('../../src') + +const { + createPeer +} = require('../utils') + +describe('rendezvous lifecycle', () => { + let peer, rendezvous + + beforeEach(async () => { + [peer] = await createPeer() + rendezvous = new Rendezvous({ libp2p: peer }) + }) + + afterEach(async () => { + await rendezvous.stop() + await peer.stop() + }) + + it('can be started and stopped', async () => { + const spyPeerStoreListener = sinon.spy(peer.peerStore, 'on') + const spyPeerStoreRemoveListener = sinon.spy(peer.peerStore, 'removeListener') + + await rendezvous.start() + + expect(spyPeerStoreListener).to.have.property('callCount', 1) + expect(spyPeerStoreRemoveListener).to.have.property('callCount', 0) + + await rendezvous.stop() + + expect(spyPeerStoreListener).to.have.property('callCount', 1) + expect(spyPeerStoreRemoveListener).to.have.property('callCount', 1) + }) + + it('adds event handlers once, if multiple starts', async () => { + const spyPeerStoreListener = sinon.spy(peer.peerStore, 'on') + + await rendezvous.start() + await rendezvous.start() + + expect(spyPeerStoreListener).to.have.property('callCount', 1) + + await rendezvous.stop() + }) + + it('only removes handlers on stop if already started', async () => { + const spyPeerStoreRemoveListener = sinon.spy(peer.peerStore, 'removeListener') + + await rendezvous.stop() + + expect(spyPeerStoreRemoveListener).to.have.property('callCount', 0) + }) +}) diff --git a/test/dos-attack-protection.spec.js b/test/dos-attack-protection.spec.js new file mode 100644 index 0000000..9fc474a --- /dev/null +++ b/test/dos-attack-protection.spec.js @@ -0,0 +1,107 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai + +const pipe = require('it-pipe') +const lp = require('it-length-prefixed') +const { collect } = require('streaming-iterables') +const { toBuffer } = require('it-buffer') + +const multiaddr = require('multiaddr') +const Libp2p = require('libp2p') + +const RendezvousServer = require('../src/server') +const { + PROTOCOL_MULTICODEC +} = require('../src/server/constants') +const { Message } = require('../src/proto') +const MESSAGE_TYPE = Message.MessageType +const RESPONSE_STATUS = Message.ResponseStatus + +const { + createPeerId, + defaultLibp2pConfig +} = require('./utils') + +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + +describe('DoS attack protection', () => { + const ns = 'test-ns' + const ttl = 7.2e6 * 1e-3 + + let rServer + let client + let peerId + let multiaddrServer + + // Create client and server and connect them + beforeEach(async () => { + [peerId] = await createPeerId() + + rServer = new RendezvousServer({ + peerId: peerId, + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }, { maxRegistrations: 1 }) // Maximum of one registration + + multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerId.toB58String()}`) + + client = await Libp2p.create({ + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + + await Promise.all([rServer, client].map((n) => n.start())) + }) + + afterEach(async () => { + await Promise.all([rServer, client].map((n) => n.stop())) + }) + + it('can register a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const responses = await pipe( + [ + Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + }), + Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + }) + ], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + expect(rServer.nsRegistrations.size).to.eql(1) + + const recMessage = Message.decode(responses[1]) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) + }) +}) diff --git a/test/protocol.spec.js b/test/protocol.spec.js new file mode 100644 index 0000000..2a13678 --- /dev/null +++ b/test/protocol.spec.js @@ -0,0 +1,258 @@ +'use strict' +/* eslint-env mocha */ + +const chai = require('chai') +chai.use(require('dirty-chai')) +chai.use(require('chai-as-promised')) +const { expect } = chai + +const pipe = require('it-pipe') +const lp = require('it-length-prefixed') +const { collect } = require('streaming-iterables') +const { toBuffer } = require('it-buffer') + +const multiaddr = require('multiaddr') +const PeerId = require('peer-id') +const Libp2p = require('libp2p') + +const RendezvousServer = require('../src/server') +const { + PROTOCOL_MULTICODEC +} = require('../src/server/constants') +const { Message } = require('../src/proto') +const MESSAGE_TYPE = Message.MessageType +const RESPONSE_STATUS = Message.ResponseStatus + +const { + createPeerId, + defaultLibp2pConfig +} = require('./utils') + +const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') +const relayAddr = MULTIADDRS_WEBSOCKETS[0] + +describe('protocol', () => { + const ns = 'test-ns' + const ttl = 7.2e6 * 1e-3 + + let rServer + let client + let peerIds + let multiaddrServer + + before(async () => { + peerIds = await createPeerId({ number: 4 }) + }) + + // Create client and server and connect them + beforeEach(async () => { + rServer = new RendezvousServer({ + peerId: peerIds[0], + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerIds[0].toB58String()}`) + + client = await Libp2p.create({ + addresses: { + listen: [`${relayAddr}/p2p-circuit`] + }, + ...defaultLibp2pConfig + }) + + await Promise.all([rServer, client].map((n) => n.start())) + }) + + afterEach(async () => { + await Promise.all([rServer, client].map((n) => n.stop())) + }) + + it('can register a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(Message.ResponseStatus.OK) + + expect(rServer.nsRegistrations.size).to.eql(1) + }) + + it('fails to register if invalid namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns: 'x'.repeat(300), + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('fails to register if invalid ttl', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl: 5e10 * 1e-3 + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_TTL) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('fails to register if invalid signed peer record', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(PeerId.createFromCID(relayAddr.getPeerId())), + ns, + ttl + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) + expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) + }) + + describe('with previous registrations', () => { + beforeEach(async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + await pipe( + [Message.encode({ + type: MESSAGE_TYPE.REGISTER, + register: { + signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), + ns, + ttl + } + })], + lp.encode(), + stream, + async (source) => { + for await (const _ of source) { } // eslint-disable-line + } + ) + + expect(rServer.nsRegistrations.size).to.eql(1) + }) + + it('can unregister a namespace', async () => { + expect(rServer.nsRegistrations.size).to.eql(1) + + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + await pipe( + [Message.encode({ + type: MESSAGE_TYPE.UNREGISTER, + unregister: { + id: client.peerId.toBytes(), + ns + } + })], + lp.encode(), + stream, + async (source) => { + for await (const _ of source) { } // eslint-disable-line + } + ) + + expect(rServer.nsRegistrations.size).to.eql(0) + }) + + it('can discover a peer registered into a namespace', async () => { + const conn = await client.dial(multiaddrServer) + const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) + + const [response] = await pipe( + [Message.encode({ + type: MESSAGE_TYPE.DISCOVER, + discover: { + ns, + limit: 50 + } + })], + lp.encode(), + stream, + lp.decode(), + toBuffer, + collect + ) + + const recMessage = Message.decode(response) + expect(recMessage).to.exist() + expect(recMessage).to.exist() + expect(recMessage.type).to.eql(MESSAGE_TYPE.DISCOVER_RESPONSE) + expect(recMessage.discoverResponse.status).to.eql(Message.ResponseStatus.OK) + expect(recMessage.discoverResponse.registrations).to.exist() + expect(recMessage.discoverResponse.registrations).to.have.lengthOf(1) + }) + }) +}) diff --git a/test/server.spec.js b/test/server.spec.js index 8ef1067..1b82b7b 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -7,24 +7,12 @@ chai.use(require('chai-as-promised')) const { expect } = chai const delay = require('delay') -const pipe = require('it-pipe') -const lp = require('it-length-prefixed') -const { collect } = require('streaming-iterables') -const { toBuffer } = require('it-buffer') const multiaddr = require('multiaddr') -const PeerId = require('peer-id') -const Libp2p = require('libp2p') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') -const { - PROTOCOL_MULTICODEC -} = require('../src/server/constants') -const { Message } = require('../src/proto') -const MESSAGE_TYPE = Message.MessageType -const RESPONSE_STATUS = Message.ResponseStatus const { createPeerId, @@ -32,9 +20,6 @@ const { defaultLibp2pConfig } = require('./utils') -const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] - const testNamespace = 'test-namespace' const multiaddrs = [multiaddr('/ip4/127.0.0.1/tcp/0')] @@ -354,306 +339,4 @@ describe('rendezvous server', () => { expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(0) expect(rServer.cookieRegistrations.get(cookie)).to.not.exist() }) - - describe('protocol', () => { - const ns = 'test-ns' - const ttl = 7.2e6 * 1e-3 - - let rServer - let client - let peerIds - let multiaddrServer - - before(async () => { - peerIds = await createPeerId({ number: 4 }) - }) - - // Create client and server and connect them - beforeEach(async () => { - rServer = new RendezvousServer({ - peerId: peerIds[0], - addresses: { - listen: [`${relayAddr}/p2p-circuit`] - }, - ...defaultLibp2pConfig - }) - multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerIds[0].toB58String()}`) - - client = await Libp2p.create({ - addresses: { - listen: [`${relayAddr}/p2p-circuit`] - }, - ...defaultLibp2pConfig - }) - - await Promise.all([rServer, client].map((n) => n.start())) - }) - - afterEach(async () => { - await Promise.all([rServer, client].map((n) => n.stop())) - }) - - it('can register a namespace', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns, - ttl - } - })], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) - expect(recMessage.registerResponse.status).to.eql(Message.ResponseStatus.OK) - - expect(rServer.nsRegistrations.size).to.eql(1) - }) - - it('fails to register if invalid namespace', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns: 'x'.repeat(300), - ttl - } - })], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) - expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) - - expect(rServer.nsRegistrations.size).to.eql(0) - }) - - it('fails to register if invalid ttl', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns, - ttl: 5e10 * 1e-3 - } - })], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) - expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_TTL) - - expect(rServer.nsRegistrations.size).to.eql(0) - }) - - it('fails to register if invalid signed peer record', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(PeerId.createFromCID(relayAddr.getPeerId())), - ns, - ttl - } - })], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) - expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) - }) - - describe('with previous registrations', () => { - beforeEach(async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - await pipe( - [Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns, - ttl - } - })], - lp.encode(), - stream, - async (source) => { - for await (const _ of source) { } // eslint-disable-line - } - ) - - expect(rServer.nsRegistrations.size).to.eql(1) - }) - - it('can unregister a namespace', async () => { - expect(rServer.nsRegistrations.size).to.eql(1) - - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - await pipe( - [Message.encode({ - type: MESSAGE_TYPE.UNREGISTER, - unregister: { - id: client.peerId.toBytes(), - ns - } - })], - lp.encode(), - stream, - async (source) => { - for await (const _ of source) { } // eslint-disable-line - } - ) - - expect(rServer.nsRegistrations.size).to.eql(0) - }) - - it('can discover a peer registered into a namespace', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [Message.encode({ - type: MESSAGE_TYPE.DISCOVER, - discover: { - ns, - limit: 50 - } - })], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - const recMessage = Message.decode(response) - expect(recMessage).to.exist() - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.DISCOVER_RESPONSE) - expect(recMessage.discoverResponse.status).to.eql(Message.ResponseStatus.OK) - expect(recMessage.discoverResponse.registrations).to.exist() - expect(recMessage.discoverResponse.registrations).to.have.lengthOf(1) - }) - }) - }) - - describe('DoS attack protection', () => { - const ns = 'test-ns' - const ttl = 7.2e6 * 1e-3 - - let rServer - let client - let peerId - let multiaddrServer - - // Create client and server and connect them - beforeEach(async () => { - [peerId] = await createPeerId() - - rServer = new RendezvousServer({ - peerId: peerId, - addresses: { - listen: [`${relayAddr}/p2p-circuit`] - }, - ...defaultLibp2pConfig - }, { maxRegistrations: 1 }) // Maximum of one registration - - multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerId.toB58String()}`) - - client = await Libp2p.create({ - addresses: { - listen: [`${relayAddr}/p2p-circuit`] - }, - ...defaultLibp2pConfig - }) - - await Promise.all([rServer, client].map((n) => n.start())) - }) - - afterEach(async () => { - await Promise.all([rServer, client].map((n) => n.stop())) - }) - - it('can register a namespace', async () => { - const conn = await client.dial(multiaddrServer) - const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) - - const responses = await pipe( - [ - Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns, - ttl - } - }), - Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: client.peerStore.addressBook.getRawEnvelope(client.peerId), - ns, - ttl - } - }) - ], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - expect(rServer.nsRegistrations.size).to.eql(1) - - const recMessage = Message.decode(responses[1]) - expect(recMessage).to.exist() - expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) - expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) - }) - }) }) From d501d973e38a0ffff87d73ca22658e25f2f827c0 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Tue, 8 Dec 2020 19:22:49 +0100 Subject: [PATCH 27/38] chore: add datastore and types --- README.md | 34 ++-- mysql/.env | 6 + mysql/Dockerfile | 10 + mysql/docker-compose.yml | 21 ++ package.json | 9 +- src/index.js | 118 ++++-------- src/server/bin.js | 22 ++- src/server/datastores/interface.ts | 51 +++++ src/server/datastores/memory.js | 211 ++++++++++++++++++++ src/server/datastores/mysql.js | 266 ++++++++++++++++++++++++++ src/server/index.js | 180 +++++------------ src/server/rpc/handlers/discover.js | 11 +- src/server/rpc/handlers/register.js | 13 +- src/server/rpc/handlers/unregister.js | 12 +- src/server/rpc/index.js | 11 +- src/server/utils.js | 8 +- test/client/api.spec.js | 195 +++++++------------ test/client/connectivity.spec.js | 68 ------- test/client/lifecycle.spec.js | 62 ------ test/dos-attack-protection.spec.js | 17 +- test/protocol.spec.js | 24 +-- test/server.spec.js | 163 ++++++++-------- test/utils.js | 20 +- tsconfig.json | 9 + 24 files changed, 908 insertions(+), 633 deletions(-) create mode 100644 mysql/.env create mode 100644 mysql/Dockerfile create mode 100644 mysql/docker-compose.yml create mode 100644 src/server/datastores/interface.ts create mode 100644 src/server/datastores/memory.js create mode 100644 src/server/datastores/mysql.js delete mode 100644 test/client/connectivity.spec.js delete mode 100644 test/client/lifecycle.spec.js create mode 100644 tsconfig.json diff --git a/README.md b/README.md index d2451d0..d0e55ef 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# js-libp2p-rendezvous +# js-libp2p-rendezvous [![](https://img.shields.io/badge/made%20by-Protocol%20Labs-blue.svg?style=flat-square)](http://protocol.ai) [![](https://img.shields.io/badge/project-libp2p-yellow.svg?style=flat-square)](http://libp2p.io/) @@ -7,16 +7,26 @@ > Javascript implementation of the rendezvous protocol for libp2p +## Lead Maintainer + +[Vasco Santos](https://github.com/vasco-santos). + +## Table of Contents + +- [Overview](#overview) +- [Usage](#usage) + - [Install](#install) + - [CLI](#cli) + - [Docker Setup](#docker-setup) +- [Contribute](#contribute) +- [License](#license) + ## Overview Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. Any node implementing the rendezvous protocol can act as a rendezvous point, allowing the discovery of relevant peers in a decentralized fashion. See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more details. -## Lead Maintainer - -[Vasco Santos](https://github.com/vasco-santos). - ## Usage ### Install @@ -27,10 +37,12 @@ See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. -It accepts several arguments: `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsMultiaddr` and `--disableMetrics` +### CLI + +After installing the rendezvous server, you can use its binary. It accepts several arguments: `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsPort` and `--disableMetrics` ```sh -libp2p-rendezvous-server [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsMultiaddr ] [--disableMetrics] +libp2p-rendezvous-server [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] ``` For further customization (e.g. swapping the muxer, using other transports) it is recommended to create a server via the API. @@ -55,10 +67,10 @@ By default it listens on `/ip4/127.0.0.1/tcp/15002/ws` and has no announce multi #### Metrics -Metrics are enabled by default on `/ip4/127.0.0.1/tcp/8003` via Prometheus. This address can also be modified with: +Metrics are enabled by default on `/ip4/127.0.0.1/tcp/8003` via Prometheus. This port can also be modified with: ```sh -libp2p-rendezvous-server --metricsMultiaddr '/ip4/127.0.0.1/tcp/8000' +libp2p-rendezvous-server --metricsPort '8008' ``` Moreover, metrics can also be disabled with: @@ -67,13 +79,13 @@ Moreover, metrics can also be disabled with: libp2p-rendezvous-server --disableMetrics ``` -## Docker Setup +### Docker Setup TODO ## Contribute -Feel free to join in. All welcome. Open an [issue](https://github.com/libp2p/js-libp2p-pubsub-peer-discovery/issues)! +Feel free to join in. All welcome. Open an [issue](https://github.com/libp2p/js-libp2p-rendezvous/issues)! This repository falls under the IPFS [Code of Conduct](https://github.com/ipfs/community/blob/master/code-of-conduct.md). diff --git a/mysql/.env b/mysql/.env new file mode 100644 index 0000000..b2276f3 --- /dev/null +++ b/mysql/.env @@ -0,0 +1,6 @@ +# MySQL +DATABASE=libp2p_rendezvous_db +ROOT_USER=root +ROOT_PASSWORD=my-secret-pw +USER=libp2p +PASSWORD=dev \ No newline at end of file diff --git a/mysql/Dockerfile b/mysql/Dockerfile new file mode 100644 index 0000000..44f5a21 --- /dev/null +++ b/mysql/Dockerfile @@ -0,0 +1,10 @@ +# Derived from official mysql image (our base image) +FROM mysql + +# Add env variables +ENV MYSQL_DATABASE libp2p-rendezvous +ENV MYSQL_ROOT_PASSWORD=my-secret-pw +ENV MYSQL_USER=vsantos +ENV MYSQL_PASSWORD=my-secret-pw + +EXPOSE 3306 diff --git a/mysql/docker-compose.yml b/mysql/docker-compose.yml new file mode 100644 index 0000000..45cba3f --- /dev/null +++ b/mysql/docker-compose.yml @@ -0,0 +1,21 @@ +version: '3.1' +services: + db: + image: mysql + volumes: + - mysql-db:/var/lib/mysql + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + # MYSQL_ROOT_PASSWORD: my-secret-pw + # MYSQL_USER: libp2p + # MYSQL_PASSWORD: my-secret-pw + # MYSQL_DATABASE: libp2p_rendezvous_db + MYSQL_DATABASE: ${DATABASE} + MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD} + MYSQL_USER: ${USER} + MYSQL_PASSWORD: ${PASSWORD} + ports: + - "3306:3306" +volumes: + mysql-db: diff --git a/package.json b/package.json index 652d4e5..4b9054f 100644 --- a/package.json +++ b/package.json @@ -44,10 +44,11 @@ "dependencies": { "debug": "^4.2.0", "err-code": "^2.0.3", + "es6-promisify": "^6.1.1", "it-buffer": "^0.1.2", "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", - "libp2p": "libp2p/js-libp2p#0.30.x", + "libp2p": "libp2p/js-libp2p#chore/add-typedfs-with-post-install", "libp2p-mplex": "^0.10.0", "libp2p-noise": "^2.0.1", "libp2p-tcp": "^0.15.1", @@ -55,13 +56,13 @@ "menoetius": "0.0.2", "minimist": "^1.2.5", "multiaddr": "^8.0.0", + "mysql": "^2.18.1", "peer-id": "^0.14.1", "protons": "^2.0.0", - "streaming-iterables": "^5.0.2", - "uint8arrays": "^1.1.0" + "streaming-iterables": "^5.0.2" }, "devDependencies": { - "aegir": "^28.2.0", + "aegir": "^29.2.2", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "delay": "^4.4.0", diff --git a/src/index.js b/src/index.js index 022a4a8..be16fd4 100644 --- a/src/index.js +++ b/src/index.js @@ -1,19 +1,18 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous') -log.error = debug('libp2p:rendezvous:error') +const log = Object.assign(debug('libp2p:rendezvous'), { + error: debug('libp2p:rendezvous:err') +}) const errCode = require('err-code') -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const lp = require('it-length-prefixed') const { collect } = require('streaming-iterables') const { toBuffer } = require('it-buffer') const fromString = require('uint8arrays/from-string') const toString = require('uint8arrays/to-string') -const PeerId = require('peer-id') - const { codes: errCodes } = require('./errors') const { MAX_DISCOVER_LIMIT, @@ -24,19 +23,17 @@ const MESSAGE_TYPE = Message.MessageType /** * @typedef {import('libp2p')} Libp2p - */ - -/** - * Rendezvous point contains the cookies per namespace that the client received. - * - * @typedef {Object} RendezvousPoint - * @property {Map} cookies + * @typedef {import('multiaddr')} Multiaddr */ /** * @typedef {Object} RendezvousProperties * @property {Libp2p} libp2p + * + * @typedef {Object} RendezvousOptions + * @property {Multiaddr[]} rendezvousPoints */ + class Rendezvous { /** * Libp2p Rendezvous. A lightweight mechanism for generalized peer discovery. @@ -44,22 +41,21 @@ class Rendezvous { * @class * @param {RendezvousProperties & RendezvousOptions} params */ - constructor ({ libp2p, maxRendezvousPoints }) { + constructor ({ libp2p, rendezvousPoints }) { this._libp2p = libp2p this._peerId = libp2p.peerId this._peerStore = libp2p.peerStore this._connectionManager = libp2p.connectionManager - - this._maxRendezvousPoints = maxRendezvousPoints + this._rendezvousPoints = rendezvousPoints this._isStarted = false /** - * @type {Map} + * Map namespaces to a map of rendezvous point identifier to cookie. + * + * @type {Map>} */ - this._rendezvousPoints = new Map() - - this._onProtocolChange = this._onProtocolChange.bind(this) + this._cookies = new Map() } /** @@ -72,11 +68,7 @@ class Rendezvous { return } - log('starting') - - this._peerStore.on('change:protocols', this._onProtocolChange) this._isStarted = true - log('started') } @@ -90,43 +82,11 @@ class Rendezvous { return } - log('stopping') - - this._peerStore.removeListener('change:protocols', this._onProtocolChange) - this._rendezvousPoints.clear() - this._isStarted = false + this._cookies.clear() log('stopped') } - /** - * Check if a peer supports the rendezvous protocol. - * If the protocol is not supported, check if it was supported before and remove it as a rendezvous point. - * If the protocol is supported, add it to the known rendezvous points. - * - * @param {Object} props - * @param {PeerId} props.peerId - * @param {Array} props.protocols - * @returns {void} - */ - _onProtocolChange ({ peerId, protocols }) { - const id = peerId.toB58String() - - // Check if it has the protocol - const hasProtocol = protocols.find(protocol => protocol === PROTOCOL_MULTICODEC) - const hasRendezvousPoint = this._rendezvousPoints.has(id) - - // If no protocol, check if we were keeping the peer before - if (!hasProtocol && hasRendezvousPoint) { - this._rendezvousPoints.delete(id) - log(`removed ${id} from rendezvous points as it does not suport ${PROTOCOL_MULTICODEC} anymore`) - } else if (hasProtocol && !this._rendezvousPoints.has(id)) { - this._rendezvousPoints.set(id, { cookies: new Map() }) - } - - // TODO: Hint that connection can be discarded? - } - /** * Register the peer in a given namespace * @@ -141,14 +101,10 @@ class Rendezvous { } // Are there available rendezvous servers? - if (!this._rendezvousPoints.size) { + if (!this._rendezvousPoints || !this._rendezvousPoints.length) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } - // TODO: we should protect from getting to many rendezvous points and sending to all - // Should we have a custom max number of servers and a custom sorter function? - // Default to peers already connected - const message = Message.encode({ type: MESSAGE_TYPE.REGISTER, register: { @@ -159,8 +115,9 @@ class Rendezvous { }) const registerTasks = [] - const taskFn = async (id) => { - const connection = await this._libp2p.dial(PeerId.createFromCID(id)) + + const taskFn = async (/** @type {Multiaddr} **/ m) => { + const connection = await this._libp2p.dial(m) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( @@ -172,6 +129,7 @@ class Rendezvous { collect ) + // Close connection if not any other open streams if (!connection.streams.length) { await connection.close() } @@ -189,12 +147,12 @@ class Rendezvous { return recMessage.registerResponse.ttl * 1e3 // convert to ms } - for (const id of this._rendezvousPoints.keys()) { - registerTasks.push(taskFn(id)) + for (const m of this._rendezvousPoints) { + registerTasks.push(taskFn(m)) } // Return first ttl - // pAny here? + // TODO: consider pAny const [returnTtl] = await Promise.all(registerTasks) return returnTtl @@ -212,7 +170,7 @@ class Rendezvous { } // Are there available rendezvous servers? - if (!this._rendezvousPoints.size) { + if (!this._rendezvousPoints || !this._rendezvousPoints.length) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } @@ -225,8 +183,8 @@ class Rendezvous { }) const unregisterTasks = [] - const taskFn = async (id) => { - const connection = await this._libp2p.dial(PeerId.createFromCID(id)) + const taskFn = async (/** @type {Multiaddr} **/ m) => { + const connection = await this._libp2p.dial(m) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) await pipe( @@ -238,13 +196,14 @@ class Rendezvous { } ) + // Close connection if not any other open streams if (!connection.streams.length) { await connection.close() } } - for (const id of this._rendezvousPoints.keys()) { - unregisterTasks.push(taskFn(id)) + for (const m of this._rendezvousPoints) { + unregisterTasks.push(taskFn(m)) } await Promise.all(unregisterTasks) @@ -259,7 +218,7 @@ class Rendezvous { */ async * discover (ns, limit = MAX_DISCOVER_LIMIT) { // Are there available rendezvous servers? - if (!this._rendezvousPoints.size) { + if (!this._rendezvousPoints || !this._rendezvousPoints.length) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) } @@ -270,11 +229,11 @@ class Rendezvous { }) // Iterate over all rendezvous points - for (const [id, rp] of this._rendezvousPoints.entries()) { - const rpCookies = rp.cookies || new Map() + for (const m of this._rendezvousPoints) { + const namespaseCookies = this._cookies.get(ns) || new Map() // Check if we have a cookie and encode discover message - const cookie = rpCookies.get(ns) + const cookie = namespaseCookies.get(m.toString()) const message = Message.encode({ type: MESSAGE_TYPE.DISCOVER, discover: { @@ -285,7 +244,7 @@ class Rendezvous { }) // Send discover message and wait for response - const connection = await this._libp2p.dial(PeerId.createFromCID(id)) + const connection = await this._libp2p.dial(m) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) const [response] = await pipe( [message], @@ -314,10 +273,9 @@ class Rendezvous { yield registrationTransformer(r) // Store cookie - rpCookies.set(ns, toString(recMessage.discoverResponse.cookie)) - this._rendezvousPoints.set(id, { - cookies: rpCookies - }) + const nsCookies = this._cookies.get(ns) || new Map() + nsCookies.set(m.toString(), toString(recMessage.discoverResponse.cookie)) + this._cookies.set(ns, nsCookies) limit-- if (limit === 0) { diff --git a/src/server/bin.js b/src/server/bin.js index ada32e1..b21efab 100644 --- a/src/server/bin.js +++ b/src/server/bin.js @@ -2,7 +2,7 @@ 'use strict' -// Usage: $0 [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsMultiaddr ] [--disableMetrics] +// Usage: $0 [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] /* eslint-disable no-console */ @@ -19,18 +19,19 @@ const Websockets = require('libp2p-websockets') const Muxer = require('libp2p-mplex') const { NOISE: Crypto } = require('libp2p-noise') -const multiaddr = require('multiaddr') const PeerId = require('peer-id') const RendezvousServer = require('./index') +const Datastore = require('./datastores/memory') const { getAnnounceAddresses, getListenAddresses } = require('./utils') async function main () { // Metrics let metricsServer const metrics = !(argv.disableMetrics || process.env.DISABLE_METRICS) - const metricsMa = multiaddr(argv.metricsMultiaddr || argv.ma || process.env.METRICSMA || '/ip4/127.0.0.1/tcp/8003') - const metricsAddr = metricsMa.nodeAddress() + const metricsPort = argv.metricsPort || argv.mp || process.env.METRICS_PORT || '8003' + // const metricsMa = multiaddr(argv.metricsMultiaddr || argv.ma || process.env.METRICSMA || '/ip4/127.0.0.1/tcp/8003') + // const metricsAddr = metricsMa.nodeAddress() // Multiaddrs const listenAddresses = getListenAddresses(argv) @@ -40,7 +41,7 @@ async function main () { let peerId if (argv.peerId) { const peerData = fs.readFileSync(argv.peerId) - peerId = await PeerId.createFromJSON(JSON.parse(peerData)) + peerId = await PeerId.createFromJSON(JSON.parse(peerData.toString())) } else { peerId = await PeerId.create() log('You are using an automatically generated peer.') @@ -48,6 +49,7 @@ async function main () { } // Create Rendezvous server + const datastore = new Datastore() const rendezvousServer = new RendezvousServer({ modules: { transport: [Websockets, TCP], @@ -59,9 +61,11 @@ async function main () { listen: listenAddresses, announce: announceAddresses } - }) + }, { datastore }) await rendezvousServer.start() + console.log('Rendezvous server listening on:') + rendezvousServer.multiaddrs.forEach((m) => console.log(m)) if (metrics) { log('enabling metrics') @@ -74,13 +78,13 @@ async function main () { menoetius.instrument(metricsServer) - metricsServer.listen(metricsAddr.port, metricsAddr.address, () => { - console.log(`metrics server listening on ${metricsAddr.port}`) + metricsServer.listen(metricsPort, '0.0.0.0', () => { + console.log(`metrics server listening on ${metricsPort.port}`) }) } const stop = async () => { - console.log('Stopping...', rendezvousServer.multiaddrs) + console.log('Stopping...') await rendezvousServer.stop() metricsServer && await metricsServer.close() process.exit(0) diff --git a/src/server/datastores/interface.ts b/src/server/datastores/interface.ts new file mode 100644 index 0000000..8913214 --- /dev/null +++ b/src/server/datastores/interface.ts @@ -0,0 +1,51 @@ +import PeerId from 'peer-id' + +export interface DatastoreFactory { + new (options?: DatastoreOptions): Datastore; +} + +export interface Datastore { + /** + * Setup datastore. + */ + start (): Promise; + /** + * Tear down datastore. + */ + stop (): void; + /** + * Add a rendezvous registrations. + */ + addRegistration (namespace: string, peerId: PeerId, signedPeerRecord: Uint8Array, ttl: number): Promise; + /** + * Get rendezvous registrations for a given namespace. + */ + getRegistrations (namespace: string, query?: RegistrationQuery): Promise<{ registrations: Registration[], cookie?: string }>; + /** + * Get number of registrations of a given peer. + */ + getNumberOfRegistrationsFromPeer (peerId: PeerId): Promise; + /** + * Remove registration of a given namespace to a peer. + */ + removeRegistration (ns: string, peerId: PeerId): Promise; + /** + * Remove all registrations of a given peer. + */ + removePeerRegistrations (peerId: PeerId): Promise; + /** + * Reset content + */ + reset (): Promise; +} + +export type RegistrationQuery = { + limit?: number; + cookie?: string; +} + +export type Registration = { + ns: string; + signedPeerRecord: Uint8Array; + ttl: number; +} diff --git a/src/server/datastores/memory.js b/src/server/datastores/memory.js new file mode 100644 index 0000000..3e49447 --- /dev/null +++ b/src/server/datastores/memory.js @@ -0,0 +1,211 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:rendezvous-server:memory') +log.error = debug('libp2p:rendezvous-server:memory:error') + +const errCode = require('err-code') +const { codes: errCodes } = require('../errors') + +const PeerId = require('peer-id') + +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('./interface').Datastore} Datastore + * @typedef {import('./interface').Registration} Registration + */ + +/** * + * + * @typedef {Object} NamespaceRegistration + * @property {string} id random generated id to map cookies + * @property {Uint8Array} signedPeerRecord + * @property {number} expiration + */ + +/** + * @implements {Datastore} + */ +class Memory { + /** + * Memory datastore for libp2p rendezvous. + */ + constructor () { + /** + * Registrations per namespace, where a registration maps peer id strings to a namespace reg. + * + * @type {Map>} + */ + this.nsRegistrations = new Map() + + /** + * Registration ids per cookie. + * + * @type {Map>} + */ + this.cookieRegistrations = new Map() + } + + /** + * @returns {Promise} + */ + start () { + return Promise.resolve() + } + + stop () { + this.nsRegistrations.clear() + this.cookieRegistrations.clear() + } + + reset () { + return Promise.resolve() + } + + /** + * Add an entry to the registration table. + * + * @param {string} ns + * @param {PeerId} peerId + * @param {Uint8Array} signedPeerRecord + * @param {number} ttl + * @returns {Promise} + */ + addRegistration (ns, peerId, signedPeerRecord, ttl) { + const nsReg = this.nsRegistrations.get(ns) || new Map() + + nsReg.set(peerId.toB58String(), { + id: String(Math.random() + Date.now()), + expiration: Date.now() + ttl, + signedPeerRecord + }) + + this.nsRegistrations.set(ns, nsReg) + + return Promise.resolve() + } + + /** + * Get registrations for a given namespace + * + * @param {string} ns + * @param {object} [options] + * @param {number} [options.limit = 10] + * @param {string} [options.cookie] + * @returns {Promise<{ registrations: Array, cookie: string }>} + */ + getRegistrations (ns, { limit = 10, cookie } = {}) { + const nsEntry = this.nsRegistrations.get(ns) || new Map() + const registrations = [] + + // Get the cookie registration if provided, create a cookie otherwise + let cRegistrations + if (cookie) { + cRegistrations = this.cookieRegistrations.get(cookie) + } else { + cRegistrations = new Set() + cookie = String(Math.random() + Date.now()) + } + + if (!cRegistrations) { + throw errCode(new Error('no registrations for the given cookie'), errCodes.INVALID_COOKIE) + } + + for (const [idStr, nsReg] of nsEntry.entries()) { + if (nsReg.expiration <= Date.now()) { + // Clean outdated registration from registrations and cookie record + nsEntry.delete(idStr) + cRegistrations.delete(nsReg.id) + continue + } + + // If this record was already sent, continue + if (cRegistrations.has(nsReg.id)) { + continue + } + + cRegistrations.add(nsReg.id) + registrations.push({ + ns, + signedPeerRecord: nsReg.signedPeerRecord, + ttl: nsReg.expiration - Date.now() // TODO: do not add if invalid? + }) + + // Stop if reached limit + if (registrations.length === limit) { + break + } + } + + // Save cookie registrations + this.cookieRegistrations.set(cookie, cRegistrations) + + return Promise.resolve({ + registrations, + cookie + }) + } + + /** + * Get number of registrations of a given peer. + * + * @param {PeerId} peerId + * @returns {Promise} + */ + getNumberOfRegistrationsFromPeer (peerId) { + const namespaces = [] + + this.nsRegistrations.forEach((nsEntry, namespace) => { + if (nsEntry.has(peerId.toB58String())) { + namespaces.push(namespace) + } + }) + + return Promise.resolve(namespaces.length) + } + + /** + * Remove registration of a given namespace to a peer + * + * @param {string} ns + * @param {PeerId} peerId + * @returns {Promise} + */ + removeRegistration (ns, peerId) { + const nsReg = this.nsRegistrations.get(ns) + + if (nsReg) { + nsReg.delete(peerId.toB58String()) + + // Remove registrations map to namespace if empty + if (!nsReg.size) { + this.nsRegistrations.delete(ns) + } + log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) + } + + return Promise.resolve() + } + + /** + * Remove all registrations of a given peer + * + * @param {PeerId} peerId + * @returns {Promise} + */ + removePeerRegistrations (peerId) { + for (const [ns, nsReg] of this.nsRegistrations.entries()) { + nsReg.delete(peerId.toB58String()) + + // Remove registrations map to namespace if empty + if (!nsReg.size) { + this.nsRegistrations.delete(ns) + } + } + + log('removed existing registrations for peer', peerId.toB58String()) + return Promise.resolve() + } +} + +module.exports = Memory diff --git a/src/server/datastores/mysql.js b/src/server/datastores/mysql.js new file mode 100644 index 0000000..4efca90 --- /dev/null +++ b/src/server/datastores/mysql.js @@ -0,0 +1,266 @@ +'use strict' + +const debug = require('debug') +const log = debug('libp2p:rendezvous-server:mysql') +log.error = debug('libp2p:rendezvous-server:mysql:error') + +const mysql = require('mysql') + +/** + * @typedef {import('peer-id')} PeerId + * @typedef {import('./interface').Datastore} Datastore + * @typedef {import('./interface').Registration} Registration + */ + +/** + * @typedef {object} MySqlOptions + * @param {string} host + * @param {string} user + * @param {string} password + * @param {string} database + * @param {boolean} [insecureAuth = true] + * @param {boolean} [multipleStatements = true] + */ + +/** + * @implements {Datastore} + */ +class Mysql { + /** + * Database manager for libp2p rendezvous. + * + * @param {MySqlOptions} options + */ + constructor ({ host, user, password, database, insecureAuth = true, multipleStatements = true }) { + this.options = { + host, + user, + password, + database, + insecureAuth, + multipleStatements + } + } + + /** + * Starts DB connection and creates needed tables if needed + * + * @returns {Promise} + */ + async start () { + this.conn = mysql.createConnection(this.options) + + await this._initDB() + } + + /** + * Closes Database connection + */ + stop () { + this.conn.end() + } + + reset () { + return Promise.resolve() + } + + /** + * Add an entry to the registration table. + * + * @param {string} namespace + * @param {PeerId} peerId + * @param {Uint8Array} signedPeerRecord + * @param {number} ttl + * @returns {Promise} + */ + addRegistration (namespace, peerId, signedPeerRecord, ttl) { + return new Promise((resolve, reject) => { + this.conn.query('INSERT INTO ?? SET ?', + ['registration', { + namespace, + peer_id: peerId, + signed_peer_record: Buffer.from(signedPeerRecord), + expiration: new Date(Date.now() + ttl) + }], (err) => { + if (err) { + return reject(err) + } + resolve() + } + ) + }) + } + + /** + * Get registrations for a given namespace + * + * @param {string} namespace + * @param {object} [options] + * @param {number} [options.limit = 10] + * @param {string} [options.cookie] + * @returns {Promise<{ registrations: Array, cookie?: string }>} + */ + async getRegistrations (namespace, { limit = 10, cookie } = {}) { + // TODO: transaction + const cookieWhereNotExists = () => { + if (!cookie) return '' + return ` AND NOT EXISTS ( + SELECT null + FROM cookie c + WHERE r.id = c.reg_id AND c.namespace = r.namespace + )` + } + + const results = await new Promise((resolve, reject) => { + this.conn.query( + `SELECT id, namespace, signed_peer_record, expiration FROM registration r + WHERE namespace = ? AND expiration >= NOW()${cookieWhereNotExists()} + ORDER BY expiration DESC + LIMIT ?`, + [namespace, limit], + (err, results) => { + if (err) { + return reject(err) + } + resolve(results) + } + ) + }) + + if (!results.length) { + return { + registrations: [], + cookie + } + } + + cookie = cookie || String(Math.random() + Date.now()) + + // Store in cookies if results available + await new Promise((resolve, reject) => { + this.conn.query( + `INSERT INTO ?? (id, namespace, reg_id) VALUES ${results.map((entry) => + `("${this.conn.escape(cookie)}", "${this.conn.escape(entry.namespace)}", "${this.conn.escape(entry.id)}")` + )}`, ['cookie'] + , (err) => { + if (err) { + return reject(err) + } + resolve() + }) + }) + + return { + registrations: results.map((r) => ({ + id: r.id, + ns: r.namespace, + signedPeerRecord: new Uint8Array(r.signed_peer_record), + ttl: r.expiration + })), + cookie + } + } + + /** + * Get number of registrations of a given peer. + * + * @param {PeerId} peerId + * @returns {Promise} + */ + getNumberOfRegistrationsFromPeer (peerId) { + const id = peerId.toB58String() + + return new Promise((resolve, reject) => { + this.conn.query('SELECT COUNT(1) FROM registration WHERE peer_id = ?', + [id], + (err, res) => { + if (err) { + return reject(err) + } + resolve(res[0]['COUNT(1)']) + } + ) + }) + } + + /** + * Remove registration of a given namespace to a peer + * + * @param {string} ns + * @param {PeerId} peerId + * @returns {Promise} + */ + removeRegistration (ns, peerId) { + const id = peerId.toB58String() + + return new Promise((resolve, reject) => { + this.conn.query('DELETE FROM registration WHERE peer_id = ? AND namespace = ?', + [id, ns], + (err) => { + if (err) { + return reject(err) + } + resolve() + }) + }) + } + + /** + * Remove all registrations of a given peer + * + * @param {PeerId} peerId + * @returns {Promise} + */ + removePeerRegistrations (peerId) { + const id = peerId.toB58String() + + return new Promise((resolve, reject) => { + this.conn.query('DELETE FROM registration WHERE peer_id = ?', + [id], + (err) => { + if (err) { + return reject(err) + } + resolve() + }) + }) + } + + /** + * Initialize Database if tables do not exist. + * + * @returns {Promise} + */ + _initDB () { + return new Promise((resolve, reject) => { + this.conn.query(` + CREATE TABLE IF NOT EXISTS registration ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT, + namespace varchar(255) NOT NULL, + peer_id varchar(255) NOT NULL, + signed_peer_record blob NOT NULL, + expiration timestamp NOT NULL, + PRIMARY KEY (id), + INDEX (namespace, expiration, peer_id) + ); + + CREATE TABLE IF NOT EXISTS cookie ( + id varchar(21), + namespace varchar(255), + reg_id INT UNSIGNED, + created_at datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id, namespace, reg_id), + FOREIGN KEY (reg_id) REFERENCES registration(id), + INDEX (created_at) + ); + `, (err) => { + if (err) { + return reject(err) + } + resolve() + }) + }) + } +} + +module.exports = Mysql diff --git a/src/server/index.js b/src/server/index.js index 78751f6..a4fbbe8 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -1,15 +1,13 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous-server') -log.error = debug('libp2p:rendezvous-server:error') - -const errCode = require('err-code') +const log = Object.assign(debug('libp2p:rendezvous-server'), { + error: debug('libp2p:rendezvous-server:err') +}) const Libp2p = require('libp2p') const PeerId = require('peer-id') -const { codes: errCodes } = require('./errors') const rpc = require('./rpc') const { MIN_TTL, @@ -21,10 +19,8 @@ const { } = require('./constants') /** - * @typedef {Object} Register - * @property {string} ns - * @property {Buffer} signedPeerRecord - * @property {number} ttl + * @typedef {import('./datastores/interface').Datastore} Datastore + * @typedef {import('./datastores/interface').Registration} Registration * * @typedef {Object} NamespaceRegistration * @property {string} id random generated id to map cookies @@ -33,6 +29,7 @@ const { /** * @typedef {Object} RendezvousServerOptions + * @property {Datastore} datastore * @property {number} [gcDelay = 3e5] garbage collector delay (default: 5 minutes) * @property {number} [gcInterval = 7.2e6] garbage collector interval (default: 2 hours) * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration @@ -48,10 +45,10 @@ const { class RendezvousServer extends Libp2p { /** * @class - * @param {Libp2pOptions} libp2pOptions - * @param {RendezvousServerOptions} [options] + * @param {import('libp2p').Libp2pOptions} libp2pOptions + * @param {RendezvousServerOptions} options */ - constructor (libp2pOptions, options = {}) { + constructor (libp2pOptions, options) { super(libp2pOptions) this._gcDelay = options.gcDelay || 3e5 @@ -62,6 +59,9 @@ class RendezvousServer extends Libp2p { this._maxDiscoveryLimit = options.maxDiscoveryLimit || MAX_DISCOVER_LIMIT this._maxRegistrations = options.maxRegistrations || MAX_REGISTRATIONS + this.datastore = options.datastore + + // TODO: REMOVE! /** * Registrations per namespace, where a registration maps peer id strings to a namespace reg. * @@ -82,19 +82,22 @@ class RendezvousServer extends Libp2p { /** * Start rendezvous server for handling rendezvous streams and gc. * - * @returns {void} + * @returns {Promise} */ - start () { + async start () { super.start() - if (this._interval) { - return - } + // if (this._interval) { + // return + // } log('starting') + await this.datastore.start() + + // TODO: + use module // Garbage collection - this._timeout = setInterval(this._gc, this._gcDelay) + // this._timeout = setInterval(this._gc, this._gcDelay) // Incoming streams handling this.handle(PROTOCOL_MULTICODEC, rpc(this)) @@ -105,19 +108,19 @@ class RendezvousServer extends Libp2p { /** * Stops rendezvous server gc and clears registrations * - * @returns {void} + * @returns {Promise} */ stop () { this.unhandle(PROTOCOL_MULTICODEC) - clearTimeout(this._timeout) - this._interval = undefined + // clearTimeout(this._timeout) - this.nsRegistrations.clear() - this.cookieRegistrations.clear() + this.datastore.stop() super.stop() log('stopped') + + return Promise.resolve() } /** @@ -150,18 +153,18 @@ class RendezvousServer extends Libp2p { const filteredIds = Array.from(idSet).filter((id) => !removedIds.includes(id)) if (filteredIds && filteredIds.length) { - this.cookieRegistrations.set(key, filteredIds) + this.cookieRegistrations.set(key, new Set(filteredIds)) } else { // Empty this.cookieRegistrations.delete(key) } } - if (!this._timeout) { - return - } + // if (!this._timeout) { + // return + // } - this._timeout = setInterval(this._gc, this._gcInterval) + // this._timeout = setInterval(this._gc, this._gcInterval) } /** @@ -169,22 +172,13 @@ class RendezvousServer extends Libp2p { * * @param {string} ns * @param {PeerId} peerId - * @param {Envelope} envelope + * @param {Uint8Array} signedPeerRecord * @param {number} ttl - * @returns {void} + * @returns {Promise} */ - addRegistration (ns, peerId, envelope, ttl) { - const nsReg = this.nsRegistrations.get(ns) || new Map() - - nsReg.set(peerId.toB58String(), { - id: String(Math.random() + Date.now()), - expiration: Date.now() + ttl - }) - - this.nsRegistrations.set(ns, nsReg) - - // Store envelope in the AddressBook - this.peerStore.addressBook.consumePeerRecord(envelope) + async addRegistration (ns, peerId, signedPeerRecord, ttl) { + await this.datastore.addRegistration(ns, peerId, signedPeerRecord, ttl) + log(`added registration for the namespace ${ns} with peer ${peerId.toB58String()}`) } /** @@ -192,39 +186,22 @@ class RendezvousServer extends Libp2p { * * @param {string} ns * @param {PeerId} peerId - * @returns {void} + * @returns {Promise} */ - removeRegistration (ns, peerId) { - const nsReg = this.nsRegistrations.get(ns) - - if (nsReg) { - nsReg.delete(peerId.toB58String()) - - // Remove registrations map to namespace if empty - if (!nsReg.size) { - this.nsRegistrations.delete(ns) - } - log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) - } + async removeRegistration (ns, peerId) { + await this.datastore.removeRegistration(ns, peerId) + log(`removed existing registrations for the namespace ${ns} - peer ${peerId.toB58String()} pair`) } /** * Remove all registrations of a given peer * * @param {PeerId} peerId - * @returns {void} + * @returns {Promise} */ - removePeerRegistrations (peerId) { - for (const [ns, nsReg] of this.nsRegistrations.entries()) { - nsReg.delete(peerId.toB58String()) - - // Remove registrations map to namespace if empty - if (!nsReg.size) { - this.nsRegistrations.delete(ns) - } - } - - log('removed existing registrations for peer', peerId.toB58String()) + async removePeerRegistrations (peerId) { + await this.datastore.removePeerRegistrations(peerId) + log(`removed existing registrations for peer ${peerId.toB58String()}`) } /** @@ -234,75 +211,20 @@ class RendezvousServer extends Libp2p { * @param {object} [options] * @param {number} [options.limit] * @param {string} [options.cookie] - * @returns {{ registrations: Array, cookie: string }} + * @returns {Promise<{ registrations: Array, cookie?: string }>} */ - getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { - const nsEntry = this.nsRegistrations.get(ns) || new Map() - const registrations = [] - - // Get the cookie registration if provided, create a cookie otherwise - let cRegistrations = new Set() - if (cookie) { - cRegistrations = this.cookieRegistrations.get(cookie) - } else { - cookie = String(Math.random() + Date.now()) - } - - if (!cRegistrations) { - throw errCode(new Error('no registrations for the given cookie'), errCodes.INVALID_COOKIE) - } - - for (const [idStr, nsReg] of nsEntry.entries()) { - if (nsReg.expiration <= Date.now()) { - // Clean outdated registration from registrations and cookie record - nsEntry.delete(idStr) - cRegistrations.delete(nsReg.id) - continue - } - - // If this record was already sent, continue - if (cRegistrations.has(nsReg.id)) { - continue - } - - cRegistrations.add(nsReg.id) - registrations.push({ - ns, - signedPeerRecord: this.peerStore.addressBook.getRawEnvelope(PeerId.createFromB58String(idStr)), - ttl: Date.now() - nsReg.expiration - }) - - // Stop if reached limit - if (registrations.length === limit) { - break - } - } - - // Save cookie registrations - this.cookieRegistrations.set(cookie, cRegistrations) - - return { - registrations, - cookie - } + async getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { + return await this.datastore.getRegistrations(ns, { limit, cookie }) } /** - * Get all the namespaces a given peer has registrations. + * Get number of registrations of a given peer. * * @param {PeerId} peerId - * @returns {Array} + * @returns {Promise} */ - getRegistrationsFromPeer (peerId) { - const namespaces = [] - - this.nsRegistrations.forEach((nsEntry, namespace) => { - if (nsEntry.has(peerId.toB58String())) { - namespaces.push(namespace) - } - }) - - return namespaces + async getNumberOfRegistrationsFromPeer (peerId) { + return await this.datastore.getNumberOfRegistrationsFromPeer(peerId) } } diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index cccd1ab..d4a301b 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -2,8 +2,9 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous:protocol:discover') -log.error = debug('libp2p:rendezvous:protocol:discover:error') +const log = Object.assign(debug('libp2p:rendezvous-server:rpc:discover'), { + error: debug('libp2p:rendezvous-server:rpc:discover:err') +}) const fromString = require('uint8arrays/from-string') const toString = require('uint8arrays/to-string') @@ -28,9 +29,9 @@ module.exports = (rendezvousPoint) => { * * @param {PeerId} peerId * @param {Message} msg - * @returns {Message} + * @returns {Promise} */ - return function discover (peerId, msg) { + return async function discover (peerId, msg) { try { const namespace = msg.discover.ns log(`discover ${peerId.toB58String()}: discover on ${namespace}`) @@ -58,7 +59,7 @@ module.exports = (rendezvousPoint) => { limit: msg.discover.limit } - const { registrations, cookie } = rendezvousPoint.getRegistrations(namespace, options) + const { registrations, cookie } = await rendezvousPoint.getRegistrations(namespace, options) return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index d8b0835..606a3dc 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -2,8 +2,9 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous:protocol:register') -log.error = debug('libp2p:rendezvous:protocol:register:error') +const log = Object.assign(debug('libp2p:rendezvous-server:rpc:register'), { + error: debug('libp2p:rendezvous-server:rpc:register:err') +}) const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') @@ -62,8 +63,8 @@ module.exports = (rendezvousPoint) => { // Now check how many registrations we have for this peer // simple limit to defend against trivial DoS attacks // example: a peer connects and keeps registering until it fills our memory - const peerRegistrations = rendezvousPoint.getRegistrationsFromPeer(peerId) - if (peerRegistrations.length >= rendezvousPoint._maxRegistrations) { + const peerRegistrations = await rendezvousPoint.getNumberOfRegistrationsFromPeer(peerId) + if (peerRegistrations >= rendezvousPoint._maxRegistrations) { log.error('unauthorized peer to register, too many registrations') return { @@ -92,10 +93,10 @@ module.exports = (rendezvousPoint) => { } // Add registration - rendezvousPoint.addRegistration( + await rendezvousPoint.addRegistration( namespace, peerId, - envelope, + msg.register.signedPeerRecord, ttl ) diff --git a/src/server/rpc/handlers/unregister.js b/src/server/rpc/handlers/unregister.js index 8f9c56c..0fa708c 100644 --- a/src/server/rpc/handlers/unregister.js +++ b/src/server/rpc/handlers/unregister.js @@ -2,8 +2,9 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous:protocol:unregister') -log.error = debug('libp2p:rendezvous:protocol:unregister:error') +const log = Object.assign(debug('libp2p:rendezvous-server:rpc:unregister'), { + error: debug('libp2p:rendezvous-server:rpc:unregister:err') +}) const equals = require('uint8arrays/equals') @@ -21,8 +22,9 @@ module.exports = (rendezvousPoint) => { * * @param {PeerId} peerId * @param {Message} msg + * @returns {Promise} */ - return function unregister (peerId, msg) { + return async function unregister (peerId, msg) { try { log(`unregister ${peerId.toB58String()}: trying unregister from ${msg.unregister.ns}`) @@ -39,9 +41,9 @@ module.exports = (rendezvousPoint) => { // Remove registration if (!msg.unregister.ns) { - rendezvousPoint.removePeerRegistrations(peerId) + await rendezvousPoint.removePeerRegistrations(peerId) } else { - rendezvousPoint.removeRegistration(msg.unregister.ns, peerId) + await rendezvousPoint.removeRegistration(msg.unregister.ns, peerId) } } catch (err) { log.error(err) diff --git a/src/server/rpc/index.js b/src/server/rpc/index.js index f2269a1..8c86143 100644 --- a/src/server/rpc/index.js +++ b/src/server/rpc/index.js @@ -1,10 +1,11 @@ 'use strict' const debug = require('debug') -const log = debug('libp2p:rendezvous-point:rpc') -log.error = debug('libp2p:rendezvous-point:rpc:error') +const log = Object.assign(debug('libp2p:rendezvous-server:rpc'), { + error: debug('libp2p:rendezvous-server:rpc:err') +}) -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const lp = require('it-length-prefixed') const { toBuffer } = require('it-buffer') @@ -17,9 +18,9 @@ module.exports = (rendezvous) => { /** * Process incoming Rendezvous messages. * - * @param {PeerId} peerId + * @param {import('peer-id')} peerId * @param {Message} msg - * @returns {Promise} + * @returns {Promise | undefined} */ function handleMessage (peerId, msg) { const handler = getMessageHandler(msg.type) diff --git a/src/server/utils.js b/src/server/utils.js index 2eddcf6..97918ca 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -4,7 +4,7 @@ const multiaddr = require('multiaddr') function getAnnounceAddresses (argv) { const announceAddr = argv.announceMultiaddrs || argv.am - const announceAddresses = announceAddr ? [multiaddr(announceAddr)] : [] + const announceAddresses = announceAddr ? [multiaddr(announceAddr).toString()] : [] if (argv.announceMultiaddrs || argv.am) { const flagIndex = process.argv.findIndex((e) => e === '--announceMultiaddrs' || e === '--am') @@ -12,7 +12,7 @@ function getAnnounceAddresses (argv) { const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { - announceAddresses.push(multiaddr(process.argv[i + 1])) + announceAddresses.push(multiaddr(process.argv[i + 1]).toString()) } } @@ -23,7 +23,7 @@ module.exports.getAnnounceAddresses = getAnnounceAddresses function getListenAddresses (argv) { const listenAddr = argv.listenMultiaddrs || argv.lm || '/ip4/127.0.0.1/tcp/15002/ws' - const listenAddresses = [multiaddr(listenAddr)] + const listenAddresses = [multiaddr(listenAddr).toString()] if (argv.listenMultiaddrs || argv.lm) { const flagIndex = process.argv.findIndex((e) => e === '--listenMultiaddrs' || e === '--lm') @@ -31,7 +31,7 @@ function getListenAddresses (argv) { const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { - listenAddresses.push(multiaddr(process.argv[i + 1])) + listenAddresses.push(multiaddr(process.argv[i + 1]).toString()) } } diff --git a/test/client/api.spec.js b/test/client/api.spec.js index 36a249e..c7829d3 100644 --- a/test/client/api.spec.js +++ b/test/client/api.spec.js @@ -1,10 +1,7 @@ 'use strict' /* eslint-env mocha */ -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai +const { expect } = require('aegir/utils/chai') const sinon = require('sinon') const pWaitFor = require('p-wait-for') @@ -22,8 +19,7 @@ const RESPONSE_STATUS = Message.ResponseStatus const { createPeer, createRendezvousServer, - createSignedPeerRecord, - connectPeers + createSignedPeerRecord } = require('../utils') const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') const relayAddr = MULTIADDRS_WEBSOCKETS[0] @@ -31,15 +27,11 @@ const relayAddr = MULTIADDRS_WEBSOCKETS[0] const namespace = 'ns' describe('rendezvous api', () => { - describe('one rendezvous server', () => { - let rendezvousServer + describe('no rendezvous server', () => { let clients // Create and start Libp2p nodes beforeEach(async () => { - // Create Rendezvous Server - rendezvousServer = await createRendezvousServer() - clients = await createPeer({ number: 2 }) // Create 2 rendezvous clients @@ -52,7 +44,6 @@ describe('rendezvous api', () => { afterEach(async () => { sinon.restore() - await rendezvousServer.stop() for (const peer of clients) { await peer.rendezvous.stop() @@ -60,70 +51,88 @@ describe('rendezvous api', () => { } }) - it('register throws error if a namespace is not provided', async () => { - await expect(clients[0].rendezvous.register()) + it('register throws error if no rendezvous servers', async () => { + await expect(clients[0].rendezvous.register(namespace)) .to.eventually.rejected() - .and.have.property('code', errCodes.INVALID_NAMESPACE) + .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) }) - it('register throws error if no connected rendezvous servers', async () => { - await expect(clients[0].rendezvous.register(namespace)) + it('unregister throws error if no rendezvous servers', async () => { + await expect(clients[0].rendezvous.unregister(namespace)) .to.eventually.rejected() .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) }) - it('register to a connected rendezvous server node', async () => { - await connectPeers(clients[0], rendezvousServer) + it('discover throws error if no rendezvous servers', async () => { + try { + for await (const _ of clients[0].rendezvous.discover()) { } // eslint-disable-line + } catch (err) { + expect(err).to.exist() + expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + return + } + throw new Error('discover should throw error if no rendezvous servers') + }) + }) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) - await clients[0].rendezvous.register(namespace) + describe('one rendezvous server', () => { + let rendezvousServer + let clients - expect(rendezvousServer.nsRegistrations.size).to.eql(1) - expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() - }) + // Create and start Libp2p + beforeEach(async () => { + // Create Rendezvous Server + rendezvousServer = await createRendezvousServer() + await pWaitFor(() => rendezvousServer.multiaddrs.length > 0) + const rendezvousServerMultiaddr = `${rendezvousServer.multiaddrs[0]}/p2p/${rendezvousServer.peerId.toB58String()}` - it('register opens and closes connection on register', async () => { - await connectPeers(clients[0], rendezvousServer) + // Create 2 rendezvous clients + clients = await createPeer({ number: 2 }) + clients.forEach((peer) => { + const rendezvous = new Rendezvous({ libp2p: peer, rendezvousPoints: [rendezvousServerMultiaddr] }) + rendezvous.start() + peer.rendezvous = rendezvous + }) + }) - clients[0].hangUp(rendezvousServer.peerId) + afterEach(async () => { + sinon.restore() + await rendezvousServer.stop() - const connsBeforeRegister = clients[0].connectionManager.size - await clients[0].rendezvous.register(namespace) + for (const peer of clients) { + await peer.rendezvous.stop() + await peer.stop() + } + }) - expect(clients[0].connectionManager.size).to.eql(connsBeforeRegister) + it('register throws error if a namespace is not provided', async () => { + await expect(clients[0].rendezvous.register()) + .to.eventually.rejected() + .and.have.property('code', errCodes.INVALID_NAMESPACE) }) it('register throws an error with an invalid namespace', async () => { const badNamespace = 'x'.repeat(300) - await connectPeers(clients[0], rendezvousServer) - - expect(rendezvousServer.nsRegistrations.size).to.eql(0) await expect(clients[0].rendezvous.register(badNamespace)) .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_INVALID_NAMESPACE) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) }) it('register throws an error with an invalid ttl', async () => { const badTtl = 5e10 - await connectPeers(clients[0], rendezvousServer) - - expect(rendezvousServer.nsRegistrations.size).to.eql(0) await expect(clients[0].rendezvous.register(namespace, { ttl: badTtl })) .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_INVALID_TTL) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) }) it('register throws an error with an invalid peerId', async () => { const badSignedPeerRecord = await createSignedPeerRecord(clients[1].peerId, [multiaddr('/ip4/127.0.0.1/tcp/100')]) - await connectPeers(clients[0], rendezvousServer) - - expect(rendezvousServer.nsRegistrations.size).to.eql(0) const stub = sinon.stub(clients[0].peerStore.addressBook, 'getRawEnvelope') stub.onCall(0).returns(badSignedPeerRecord.marshal()) @@ -132,7 +141,15 @@ describe('rendezvous api', () => { .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_NOT_AUTHORIZED) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + }) + + it('registers with an available rendezvous server node', async () => { + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + await clients[0].rendezvous.register(namespace) + + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(1) + expect(rendezvousServer.datastore.nsRegistrations.get(namespace)).to.exist() }) it('unregister throws if a namespace is not provided', async () => { @@ -141,63 +158,26 @@ describe('rendezvous api', () => { .and.have.property('code', errCodes.INVALID_NAMESPACE) }) - it('unregister throws error if no connected rendezvous servers', async () => { - await expect(clients[0].rendezvous.unregister(namespace)) - .to.eventually.rejected() - .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - }) - - it('unregister to a connected rendezvous server node', async () => { - await connectPeers(clients[0], rendezvousServer) - + it('unregisters with an available rendezvous server node', async () => { // Register - expect(rendezvousServer.nsRegistrations.size).to.eql(0) + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) await clients[0].rendezvous.register(namespace) - expect(rendezvousServer.nsRegistrations.size).to.eql(1) - expect(rendezvousServer.nsRegistrations.get(namespace)).to.exist() + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(1) + expect(rendezvousServer.datastore.nsRegistrations.get(namespace)).to.exist() // Unregister await clients[0].rendezvous.unregister(namespace) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) + expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) }) - it('unregister opens and closes connection on register', async () => { - await connectPeers(clients[0], rendezvousServer) - - // Register - await clients[0].rendezvous.register(namespace) - expect(rendezvousServer.nsRegistrations.size).to.eql(1) - - const connsBeforeRegister = clients[0].connectionManager.size - await clients[0].rendezvous.unregister(namespace) - expect(rendezvousServer.nsRegistrations.size).to.eql(0) - - expect(clients[0].connectionManager.size).to.eql(connsBeforeRegister) - }) - - it('unregister to a connected rendezvous server node not fails if not registered', async () => { - await connectPeers(clients[0], rendezvousServer) - - // Unregister + it('unregister not fails if not registered', async () => { await clients[0].rendezvous.unregister(namespace) }) - it('discover throws error if a namespace is not provided', async () => { - try { - for await (const _ of clients[0].rendezvous.discover()) { } // eslint-disable-line - } catch (err) { - expect(err).to.exist() - expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - return - } - throw new Error('discover should throw error if a namespace is not provided') - }) - it('discover throws error if a namespace is invalid', async () => { const badNamespace = 'x'.repeat(300) - await connectPeers(clients[0], rendezvousServer) try { for await (const _ of clients[0].rendezvous.discover(badNamespace)) { } // eslint-disable-line } catch (err) { @@ -209,17 +189,12 @@ describe('rendezvous api', () => { }) it('discover does not find any register if there is none', async () => { - await connectPeers(clients[0], rendezvousServer) - for await (const reg of clients[0].rendezvous.discover(namespace)) { // eslint-disable-line throw new Error('no registers should exist') } }) it('discover finds registered peer for namespace', async () => { - await connectPeers(clients[0], rendezvousServer) - await connectPeers(clients[1], rendezvousServer) - const registers = [] // Peer2 does not discovery any peer registered @@ -247,27 +222,7 @@ describe('rendezvous api', () => { expect(rec.multiaddrs).to.eql(clients[0].multiaddrs) }) - it('discover opens and closes connection on register', async () => { - await connectPeers(clients[0], rendezvousServer) - await connectPeers(clients[1], rendezvousServer) - - // Register - await clients[0].rendezvous.register(namespace) - expect(rendezvousServer.nsRegistrations.size).to.eql(1) - - // Hangup from one - clients[1].hangUp(rendezvousServer.peerId) - const connsBeforeRegister = clients[1].connectionManager.size - - for await (const _ of clients[1].rendezvous.discover(namespace)) {} // eslint-disable-line - - expect(clients[1].connectionManager.size).to.eql(connsBeforeRegister) - }) - it('discover finds registered peer for namespace once (cookie usage)', async () => { - await connectPeers(clients[0], rendezvousServer) - await connectPeers(clients[1], rendezvousServer) - const registers = [] // Peer2 does not discovery any peer registered @@ -300,15 +255,6 @@ describe('rendezvous api', () => { let rendezvousServers = [] let clients - const connectPeers = async (peer, otherPeer) => { - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) - await peer.dial(m) - - // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === rendezvousServers.length) - } - // Create and start Libp2p nodes beforeEach(async () => { // Create Rendezvous Server @@ -316,12 +262,13 @@ describe('rendezvous api', () => { createRendezvousServer(), createRendezvousServer() ]) - - clients = await createPeer({ number: 2 }) + await pWaitFor(() => rendezvousServers[0].multiaddrs.length > 0 && rendezvousServers[1].multiaddrs.length > 0) + const rendezvousServerMultiaddrs = rendezvousServers.map((rendezvousServer) => `${rendezvousServer.multiaddrs[0]}/p2p/${rendezvousServer.peerId.toB58String()}`) // Create 2 rendezvous clients + clients = await createPeer({ number: 2 }) clients.forEach((peer) => { - const rendezvous = new Rendezvous({ libp2p: peer }) + const rendezvous = new Rendezvous({ libp2p: peer, rendezvousPoints: rendezvousServerMultiaddrs }) rendezvous.start() peer.rendezvous = rendezvous }) @@ -340,10 +287,6 @@ describe('rendezvous api', () => { }) it('discover find registered peer for namespace only when registered ', async () => { - // Connect all the clients to all the servers - await Promise.all(rendezvousServers.map((server) => - Promise.all(clients.map((client) => connectPeers(client, server))))) - const registers = [] // Client 1 does not discovery any peer registered diff --git a/test/client/connectivity.spec.js b/test/client/connectivity.spec.js deleted file mode 100644 index bd0ed05..0000000 --- a/test/client/connectivity.spec.js +++ /dev/null @@ -1,68 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai - -const pWaitFor = require('p-wait-for') -const multiaddr = require('multiaddr') - -const Rendezvous = require('../../src') - -const { - createPeer, - createRendezvousServer -} = require('../utils') -const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] - -describe('rendezvous connectivity', () => { - let rendezvousServer - let client - - // Create and start Libp2p nodes - beforeEach(async () => { - // Create Rendezvous Server - rendezvousServer = await createRendezvousServer() - - // Create Rendezvous client - ;[client] = await createPeer() - - const rendezvous = new Rendezvous({ libp2p: client }) - client.rendezvous = rendezvous - client.rendezvous.start() - }) - - // Connect nodes to the testing relay node - beforeEach(async () => { - await rendezvousServer.dial(relayAddr) - await client.dial(relayAddr) - }) - - afterEach(async () => { - await rendezvousServer.stop() - await client.rendezvous.stop() - await client.stop() - }) - - it('updates known rendezvous points', async () => { - expect(client.rendezvous._rendezvousPoints.size).to.equal(0) - - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${rendezvousServer.peerId.toB58String()}`) - const connection = await client.dial(m) - - expect(client.peerStore.peers.size).to.equal(2) - expect(rendezvousServer.peerStore.peers.size).to.equal(2) - - // Wait event propagation - // Relay peer is not with rendezvous enabled - await pWaitFor(() => client.rendezvous._rendezvousPoints.size === 1) - - expect(client.rendezvous._rendezvousPoints.get(rendezvousServer.peerId.toB58String())).to.exist() - - await connection.close() - }) -}) diff --git a/test/client/lifecycle.spec.js b/test/client/lifecycle.spec.js deleted file mode 100644 index e06a6dc..0000000 --- a/test/client/lifecycle.spec.js +++ /dev/null @@ -1,62 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai -const sinon = require('sinon') - -const Rendezvous = require('../../src') - -const { - createPeer -} = require('../utils') - -describe('rendezvous lifecycle', () => { - let peer, rendezvous - - beforeEach(async () => { - [peer] = await createPeer() - rendezvous = new Rendezvous({ libp2p: peer }) - }) - - afterEach(async () => { - await rendezvous.stop() - await peer.stop() - }) - - it('can be started and stopped', async () => { - const spyPeerStoreListener = sinon.spy(peer.peerStore, 'on') - const spyPeerStoreRemoveListener = sinon.spy(peer.peerStore, 'removeListener') - - await rendezvous.start() - - expect(spyPeerStoreListener).to.have.property('callCount', 1) - expect(spyPeerStoreRemoveListener).to.have.property('callCount', 0) - - await rendezvous.stop() - - expect(spyPeerStoreListener).to.have.property('callCount', 1) - expect(spyPeerStoreRemoveListener).to.have.property('callCount', 1) - }) - - it('adds event handlers once, if multiple starts', async () => { - const spyPeerStoreListener = sinon.spy(peer.peerStore, 'on') - - await rendezvous.start() - await rendezvous.start() - - expect(spyPeerStoreListener).to.have.property('callCount', 1) - - await rendezvous.stop() - }) - - it('only removes handlers on stop if already started', async () => { - const spyPeerStoreRemoveListener = sinon.spy(peer.peerStore, 'removeListener') - - await rendezvous.stop() - - expect(spyPeerStoreRemoveListener).to.have.property('callCount', 0) - }) -}) diff --git a/test/dos-attack-protection.spec.js b/test/dos-attack-protection.spec.js index 9fc474a..7259c27 100644 --- a/test/dos-attack-protection.spec.js +++ b/test/dos-attack-protection.spec.js @@ -1,12 +1,9 @@ 'use strict' /* eslint-env mocha */ -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai +const { expect } = require('aegir/utils/chai') -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const lp = require('it-length-prefixed') const { collect } = require('streaming-iterables') const { toBuffer } = require('it-buffer') @@ -15,6 +12,7 @@ const multiaddr = require('multiaddr') const Libp2p = require('libp2p') const RendezvousServer = require('../src/server') +const Datastore = require('../src/server/datastores/memory') const { PROTOCOL_MULTICODEC } = require('../src/server/constants') @@ -34,6 +32,7 @@ describe('DoS attack protection', () => { const ns = 'test-ns' const ttl = 7.2e6 * 1e-3 + let datastore let rServer let client let peerId @@ -43,13 +42,14 @@ describe('DoS attack protection', () => { beforeEach(async () => { [peerId] = await createPeerId() + datastore = new Datastore() rServer = new RendezvousServer({ peerId: peerId, addresses: { listen: [`${relayAddr}/p2p-circuit`] }, ...defaultLibp2pConfig - }, { maxRegistrations: 1 }) // Maximum of one registration + }, { maxRegistrations: 1, datastore }) // Maximum of one registration multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerId.toB58String()}`) @@ -97,11 +97,12 @@ describe('DoS attack protection', () => { collect ) - expect(rServer.nsRegistrations.size).to.eql(1) - const recMessage = Message.decode(responses[1]) expect(recMessage).to.exist() expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) + + // Only one record + expect(rServer.datastore.nsRegistrations.size).to.eql(1) }) }) diff --git a/test/protocol.spec.js b/test/protocol.spec.js index 2a13678..5606e8a 100644 --- a/test/protocol.spec.js +++ b/test/protocol.spec.js @@ -1,12 +1,9 @@ 'use strict' /* eslint-env mocha */ -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai +const { expect } = require('aegir/utils/chai') -const pipe = require('it-pipe') +const { pipe } = require('it-pipe') const lp = require('it-length-prefixed') const { collect } = require('streaming-iterables') const { toBuffer } = require('it-buffer') @@ -16,6 +13,7 @@ const PeerId = require('peer-id') const Libp2p = require('libp2p') const RendezvousServer = require('../src/server') +const Datastore = require('../src/server/datastores/memory') const { PROTOCOL_MULTICODEC } = require('../src/server/constants') @@ -35,6 +33,7 @@ describe('protocol', () => { const ns = 'test-ns' const ttl = 7.2e6 * 1e-3 + let datastore let rServer let client let peerIds @@ -46,13 +45,14 @@ describe('protocol', () => { // Create client and server and connect them beforeEach(async () => { + datastore = new Datastore() rServer = new RendezvousServer({ peerId: peerIds[0], addresses: { listen: [`${relayAddr}/p2p-circuit`] }, ...defaultLibp2pConfig - }) + }, { datastore }) multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerIds[0].toB58String()}`) client = await Libp2p.create({ @@ -94,7 +94,7 @@ describe('protocol', () => { expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(Message.ResponseStatus.OK) - expect(rServer.nsRegistrations.size).to.eql(1) + expect(rServer.datastore.nsRegistrations.size).to.eql(1) }) it('fails to register if invalid namespace', async () => { @@ -122,7 +122,7 @@ describe('protocol', () => { expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) - expect(rServer.nsRegistrations.size).to.eql(0) + expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('fails to register if invalid ttl', async () => { @@ -150,7 +150,7 @@ describe('protocol', () => { expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_TTL) - expect(rServer.nsRegistrations.size).to.eql(0) + expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('fails to register if invalid signed peer record', async () => { @@ -200,11 +200,11 @@ describe('protocol', () => { } ) - expect(rServer.nsRegistrations.size).to.eql(1) + expect(rServer.datastore.nsRegistrations.size).to.eql(1) }) it('can unregister a namespace', async () => { - expect(rServer.nsRegistrations.size).to.eql(1) + expect(rServer.datastore.nsRegistrations.size).to.eql(1) const conn = await client.dial(multiaddrServer) const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) @@ -224,7 +224,7 @@ describe('protocol', () => { } ) - expect(rServer.nsRegistrations.size).to.eql(0) + expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('can discover a peer registered into a namespace', async () => { diff --git a/test/server.spec.js b/test/server.spec.js index 1b82b7b..b8cea64 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -1,11 +1,7 @@ 'use strict' /* eslint-env mocha */ -const chai = require('chai') -chai.use(require('dirty-chai')) -chai.use(require('chai-as-promised')) -const { expect } = chai - +const { expect } = require('aegir/utils/chai') const delay = require('delay') const multiaddr = require('multiaddr') @@ -13,7 +9,8 @@ const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') - +const Datastore = require('../src/server/datastores/memory') +const { codes: errCodes } = require('../src/server/errors') const { createPeerId, createSignedPeerRecord, @@ -27,6 +24,7 @@ describe('rendezvous server', () => { const signedPeerRecords = [] let rServer let peerIds + let datastore before(async () => { peerIds = await createPeerId({ number: 4 }) @@ -34,11 +32,14 @@ describe('rendezvous server', () => { // Create a signed peer record per peer for (const peerId of peerIds) { const spr = await createSignedPeerRecord(peerId, multiaddrs) - signedPeerRecords.push(spr) + signedPeerRecords.push(spr.marshal()) } + + datastore = new Datastore() }) afterEach(async () => { + datastore = new Datastore() rServer && await rServer.stop() }) @@ -46,133 +47,133 @@ describe('rendezvous server', () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) await rServer.start() }) - it('can add registrations to multiple namespaces', () => { + it('can add registrations to multiple namespaces', async () => { const otherNamespace = 'other-namespace' rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) - const { registrations: testNsRegistrations } = rServer.getRegistrations(testNamespace) + const { registrations: testNsRegistrations } = await rServer.getRegistrations(testNamespace) expect(testNsRegistrations).to.have.lengthOf(2) - const { registrations: otherNsRegistrations } = rServer.getRegistrations(otherNamespace) + const { registrations: otherNsRegistrations } = await rServer.getRegistrations(otherNamespace) expect(otherNsRegistrations).to.have.lengthOf(1) }) - it('should be able to limit registrations to get', () => { + it('should be able to limit registrations to get', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) - let r = rServer.getRegistrations(testNamespace, { limit: 1 }) + let r = await rServer.getRegistrations(testNamespace, { limit: 1 }) expect(r.registrations).to.have.lengthOf(1) expect(r.cookie).to.exist() - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) expect(r.cookie).to.exist() }) - it('can remove registrations from a peer in a given namespace', () => { + it('can remove registrations from a peer in a given namespace', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) - let r = rServer.getRegistrations(testNamespace) + let r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) expect(r.cookie).to.exist() // Remove registration for peer0 - rServer.removeRegistration(testNamespace, peerIds[1]) + await rServer.removeRegistration(testNamespace, peerIds[1]) - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) expect(r.cookie).to.exist() }) - it('can remove all registrations from a peer', () => { + it('can remove all registrations from a peer', async () => { const otherNamespace = 'other-namespace' rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Add registration for peer 1 in a different namespace - rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(otherNamespace, peerIds[1], signedPeerRecords[1], 1000) - let r = rServer.getRegistrations(testNamespace) + let r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) - let otherR = rServer.getRegistrations(otherNamespace) + let otherR = await rServer.getRegistrations(otherNamespace) expect(otherR.registrations).to.have.lengthOf(1) // Remove all registrations for peer0 - rServer.removePeerRegistrations(peerIds[1]) + await rServer.removePeerRegistrations(peerIds[1]) - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(0) - otherR = rServer.getRegistrations(otherNamespace) + otherR = await rServer.getRegistrations(otherNamespace) expect(otherR.registrations).to.have.lengthOf(0) }) - it('can attempt to remove a registration for a non existent namespace', () => { + it('can attempt to remove a registration for a non existent namespace', async () => { const otherNamespace = 'other-namespace' rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) - rServer.removeRegistration(otherNamespace, peerIds[1]) + await rServer.removeRegistration(otherNamespace, peerIds[1]) }) - it('can attempt to remove a registration for a non existent peer', () => { + it('can attempt to remove a registration for a non existent peer', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) - let r = rServer.getRegistrations(testNamespace) + let r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) // Remove registration for peer0 - rServer.removeRegistration(testNamespace, peerIds[2]) + await rServer.removeRegistration(testNamespace, peerIds[2]) - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) }) @@ -180,24 +181,24 @@ describe('rendezvous server', () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }, { gcInterval: 300 }) + }, { datastore, gcInterval: 300 }) await rServer.start() // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) - rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) - let r = rServer.getRegistrations(testNamespace) + let r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) // wait for firt record to be removed await delay(650) - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) await delay(400) - r = rServer.getRegistrations(testNamespace) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(0) }) @@ -205,13 +206,13 @@ describe('rendezvous server', () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations - const { cookie, registrations } = rServer.getRegistrations(testNamespace) + const { cookie, registrations } = await rServer.getRegistrations(testNamespace) expect(cookie).to.exist() expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) @@ -222,10 +223,10 @@ describe('rendezvous server', () => { expect(envelope.peerId.toString()).to.eql(peerIds[1].toString()) // Add registration for peer 2 in test namespace - rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) // Get second registration by using the cookie - const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + const { cookie: cookie2, registrations: registrations2 } = await rServer.getRegistrations(testNamespace, { cookie }) expect(cookie2).to.exist() expect(cookie2).to.eql(cookie) expect(registrations2).to.exist() @@ -237,28 +238,28 @@ describe('rendezvous server', () => { expect(envelope2.peerId.toString()).to.eql(peerIds[2].toString()) // If no cookie provided, all registrations are given - const { registrations: registrations3 } = rServer.getRegistrations(testNamespace) + const { registrations: registrations3 } = await rServer.getRegistrations(testNamespace) expect(registrations3).to.exist() expect(registrations3).to.have.lengthOf(2) }) - it('no new peers should be returned if there are not new peers since latest query', () => { + it('no new peers should be returned if there are not new peers since latest query', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations - const { cookie, registrations } = rServer.getRegistrations(testNamespace) + const { cookie, registrations } = await rServer.getRegistrations(testNamespace) expect(cookie).to.exist() expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) // Get registrations with same cookie and no new registration - const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + const { cookie: cookie2, registrations: registrations2 } = await rServer.getRegistrations(testNamespace, { cookie }) expect(cookie2).to.exist() expect(cookie2).to.eql(cookie) expect(registrations2).to.exist() @@ -269,13 +270,13 @@ describe('rendezvous server', () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) + }, { datastore }) // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get current registrations - const { cookie, registrations } = rServer.getRegistrations(testNamespace) + const { cookie, registrations } = await rServer.getRegistrations(testNamespace) expect(cookie).to.exist() expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) @@ -286,10 +287,10 @@ describe('rendezvous server', () => { expect(envelope.peerId.toString()).to.eql(peerIds[1].toString()) // Add new registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) // Get registrations with same cookie and no new registration - const { cookie: cookie2, registrations: registrations2 } = rServer.getRegistrations(testNamespace, { cookie }) + const { cookie: cookie2, registrations: registrations2 } = await rServer.getRegistrations(testNamespace, { cookie }) expect(cookie2).to.exist() expect(cookie2).to.eql(cookie) expect(registrations2).to.exist() @@ -301,42 +302,40 @@ describe('rendezvous server', () => { expect(envelope2.peerId.toString()).to.eql(peerIds[1].toString()) }) - it('get registrations should throw if no stored cookie is provided', () => { + it('get registrations should throw if no stored cookie is provided', async () => { const badCookie = String(Math.random() + Date.now()) rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }) - - const fn = () => { - rServer.getRegistrations(testNamespace, { cookie: badCookie }) - } + }, { datastore }) - expect(fn).to.throw('no registrations for the given cookie') + await expect(rServer.getRegistrations(testNamespace, { cookie: badCookie })) + .to.eventually.be.rejectedWith(Error) + .and.to.have.property('code', errCodes.INVALID_COOKIE) }) it('garbage collector should remove cookies of discarded records', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }, { gcDelay: 300, gcInterval: 300 }) + }, { datastore, gcDelay: 300, gcInterval: 300 }) await rServer.start() // Add registration for peer 1 in test namespace - rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) // Get current registrations - const { cookie, registrations } = rServer.getRegistrations(testNamespace) + const { cookie, registrations } = await rServer.getRegistrations(testNamespace) expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) // Verify internal state - expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(1) - expect(rServer.cookieRegistrations.get(cookie)).to.exist() + expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(1) + expect(rServer.datastore.cookieRegistrations.get(cookie)).to.exist() await delay(800) - expect(rServer.nsRegistrations.get(testNamespace).size).to.eql(0) - expect(rServer.cookieRegistrations.get(cookie)).to.not.exist() + expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(0) + expect(rServer.datastore.cookieRegistrations.get(cookie)).to.not.exist() }) }) diff --git a/test/utils.js b/test/utils.js index 5b96c02..49245e6 100644 --- a/test/utils.js +++ b/test/utils.js @@ -6,7 +6,6 @@ const { NOISE: Crypto } = require('libp2p-noise') const PeerId = require('peer-id') const pTimes = require('p-times') -const pWaitFor = require('p-wait-for') const Libp2p = require('libp2p') const multiaddr = require('multiaddr') @@ -14,6 +13,7 @@ const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') +const Datastore = require('../src/server/datastores/memory') const Peers = require('./fixtures/peers') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') @@ -86,6 +86,7 @@ module.exports.createPeer = createPeer async function createRendezvousServer ({ config = {}, started = true } = {}) { const [peerId] = await createPeerId({ fixture: false }) + const datastore = new Datastore() const rendezvous = new RendezvousServer({ peerId: peerId, addresses: { @@ -93,7 +94,7 @@ async function createRendezvousServer ({ config = {}, started = true } = {}) { }, ...defaultConfig, ...config - }) + }, { datastore }) if (started) { await rendezvous.start() @@ -104,21 +105,6 @@ async function createRendezvousServer ({ config = {}, started = true } = {}) { module.exports.createRendezvousServer = createRendezvousServer -async function connectPeers (peer, otherPeer) { - // Connect to testing relay node - await peer.dial(relayAddr) - await otherPeer.dial(relayAddr) - - // Connect each other via relay node - const m = multiaddr(`${relayAddr}/p2p-circuit/p2p/${otherPeer.peerId.toB58String()}`) - await peer.dial(m) - - // Wait event propagation - await pWaitFor(() => peer.rendezvous._rendezvousPoints.size === 1) -} - -module.exports.connectPeers = connectPeers - async function createSignedPeerRecord (peerId, multiaddrs) { const pr = new PeerRecord({ peerId, diff --git a/tsconfig.json b/tsconfig.json new file mode 100644 index 0000000..5b9a618 --- /dev/null +++ b/tsconfig.json @@ -0,0 +1,9 @@ +{ + "extends": "./node_modules/aegir/src/config/tsconfig.aegir.json", + "compilerOptions": { + "outDir": "dist" + }, + "include": [ + "src" + ] +} \ No newline at end of file From a2d5f83deef556bee2bd2223e20940ac452411a4 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Sun, 13 Dec 2020 12:07:22 +0100 Subject: [PATCH 28/38] chore: fix build --- src/index.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/index.js b/src/index.js index be16fd4..cd31a96 100644 --- a/src/index.js +++ b/src/index.js @@ -116,7 +116,11 @@ class Rendezvous { const registerTasks = [] - const taskFn = async (/** @type {Multiaddr} **/ m) => { + /** + * @param {Multiaddr} m + * @returns {Promise} + */ + const taskFn = async (m) => { const connection = await this._libp2p.dial(m) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) @@ -183,7 +187,11 @@ class Rendezvous { }) const unregisterTasks = [] - const taskFn = async (/** @type {Multiaddr} **/ m) => { + /** + * @param {Multiaddr} m + * @returns {Promise} + */ + const taskFn = async (m) => { const connection = await this._libp2p.dial(m) const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) From 83cd4b7cfecbab60b7e2befcf79371346a7b4061 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 21 Dec 2020 17:50:02 +0000 Subject: [PATCH 29/38] chore: run with mysql --- .aegir.js | 37 +++++++ .github/workflows/main.yml | 62 ++++++++++++ .travis.yml | 42 -------- README.md | 67 ++++++++++-- mysql/Dockerfile | 10 -- mysql/docker-compose.yml | 16 +-- package.json | 11 +- sql | 27 +++++ src/index.js | 17 ++-- src/server/bin.js | 12 ++- src/server/datastores/memory.js | 7 +- src/server/datastores/mysql.js | 151 ++++++++++++++++++++++------ src/server/index.js | 16 +-- src/server/rpc/handlers/discover.js | 2 +- src/server/utils.js | 55 ++++++---- test/client/api.spec.js | 80 ++++++++++----- test/dos-attack-protection.spec.js | 8 +- test/protocol.spec.js | 24 ++--- test/server.spec.js | 77 +++++++------- test/utils.js | 23 ++++- 20 files changed, 520 insertions(+), 224 deletions(-) create mode 100644 .github/workflows/main.yml delete mode 100644 .travis.yml delete mode 100644 mysql/Dockerfile create mode 100644 sql diff --git a/.aegir.js b/.aegir.js index c3c7f90..ce11caa 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,7 +8,14 @@ const WebSockets = require('libp2p-websockets') const Muxer = require('libp2p-mplex') const { NOISE: Crypto } = require('libp2p-noise') +const { isNode } = require('ipfs-utils/src/env') +const delay = require('delay') +const execa = require('execa') +const pWaitFor = require('p-wait-for') +const isCI = require('is-ci') + let libp2p +let containerId const before = async () => { // Use the last peer @@ -36,10 +43,40 @@ const before = async () => { }) await libp2p.start() + + // CI runs datastore service + if (isCI || !isNode) { + return + } + + const procResult = execa.commandSync('docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=test-secret-pw -e MYSQL_DATABASE=libp2p_rendezvous_db -d mysql:8 --default-authentication-plugin=mysql_native_password', { + all: true + }) + containerId = procResult.stdout + + console.log(`wait for docker container ${containerId} to be ready`) + + await pWaitFor(() => { + const procCheck = execa.commandSync(`docker logs ${containerId}`) + const logs = procCheck.stdout + procCheck.stderr // Docker/MySQL sends to the stderr the ready for connections... + + return logs.includes('ready for connections') + }, { + interval: 5000 + }) + // Some more time waiting + await delay(10e3) } const after = async () => { await libp2p.stop() + + if (isCI || !isNode) { + return + } + + console.log('docker container is stopping') + execa.commandSync(`docker stop ${containerId}`) } module.exports = { diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml new file mode 100644 index 0000000..820e706 --- /dev/null +++ b/.github/workflows/main.yml @@ -0,0 +1,62 @@ +name: ci +on: + push: + branches: + - master + pull_request: + branches: + - '**' + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: yarn + - run: yarn lint + - uses: gozala/typescript-error-reporter-action@v1.0.8 + - run: yarn build + - run: yarn aegir dep-check + - uses: ipfs/aegir/actions/bundle-size@master + name: size + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + test-node: + needs: check + runs-on: ${{ matrix.os }} + services: + mysql: + image: mysql:5.7 + env: + MYSQL_ROOT_PASSWORD: test-secret-pw + MYSQL_DATABASE: libp2p_rendezvous_db + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + strategy: + matrix: + os: [ubuntu-latest] + node: [12, 14] + fail-fast: true + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v1 + with: + node-version: ${{ matrix.node }} + - run: yarn + - run: npx nyc --reporter=lcov aegir test -t node -- --bail + - uses: codecov/codecov-action@v1 + test-chrome: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: yarn + - run: npx aegir test -t browser -t webworker --bail + test-firefox: + needs: check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - run: yarn + - run: npx aegir test -t browser -t webworker --bail -- --browsers FirefoxHeadless diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index fdab469..0000000 --- a/.travis.yml +++ /dev/null @@ -1,42 +0,0 @@ -language: node_js -cache: npm -stages: - - check - - test - - cov - -node_js: - - '10' - - '12' - -os: - - linux - - osx - - windows - -script: npx nyc -s npm run test:node -- --bail -after_success: npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov - -jobs: - include: - - stage: check - script: - - npx aegir dep-check - - npm run lint - - - stage: test - name: chrome - addons: - chrome: stable - script: - - npx aegir test -t browser -t webworker - - - stage: test - name: firefox - addons: - firefox: latest - script: - - npx aegir test -t browser -t webworker -- --browsers FirefoxHeadless - -notifications: - email: false \ No newline at end of file diff --git a/README.md b/README.md index d0e55ef..b6b1c85 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,10 @@ [![](https://img.shields.io/badge/project-libp2p-yellow.svg?style=flat-square)](http://libp2p.io/) [![](https://img.shields.io/badge/freenode-%23libp2p-yellow.svg?style=flat-square)](http://webchat.freenode.net/?channels=%23libp2p) [![](https://img.shields.io/discourse/https/discuss.libp2p.io/posts.svg)](https://discuss.libp2p.io) +[![codecov](https://img.shields.io/codecov/c/github/libp2p/js-libp2p-rendezvous.svg?style=flat-square)](https://codecov.io/gh/libp2p/js-libp2p-rendezvous) +[![GitHub Workflow Status](https://img.shields.io/github/workflow/status/libp2p/js-libp2p-rendezvous/ci?label=ci&style=flat-square)](https://github.com/libp2p/js-libp2p-rendezvous/actions?query=branch%3Amaster+workflow%3Aci+) -> Javascript implementation of the rendezvous protocol for libp2p +> Javascript implementation of the rendezvous server protocol for libp2p ## Lead Maintainer @@ -23,7 +25,7 @@ ## Overview -Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. Any node implementing the rendezvous protocol can act as a rendezvous point, allowing the discovery of relevant peers in a decentralized fashion. +Libp2p rendezvous is a lightweight mechanism for generalized peer discovery. It can be used for bootstrap purposes, real time peer discovery, application specific routing, and so on. This module is the implementation of the rendezvous server protocol for libp2p. See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more details. @@ -35,24 +37,47 @@ See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more > npm install --global libp2p-rendezvous ``` -Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. +Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. Bear in mind that a MySQL database is required to run the rendezvous server. + +### Testing + +For running the tests in this module, you will need to have Docker installed. A docker container is used to run a MySQL database for testing purposes. ### CLI -After installing the rendezvous server, you can use its binary. It accepts several arguments: `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsPort` and `--disableMetrics` +After installing the rendezvous server, you can use its binary. It accepts several arguments: `--datastoreHost`, `--datastoreUser`, `--datastorePassword`, `--datastoreDatabase`, `--enableMemoryDatabase`, `--peerId`, `--listenMultiaddrs`, `--announceMultiaddrs`, `--metricsPort` and `--disableMetrics` ```sh -libp2p-rendezvous-server [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] +libp2p-rendezvous-server [--datastoreHost ] [--datastoreUser ] [datastorePassword ] [datastoreDatabase ] [--enableMemoryDatabase] [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] ``` -For further customization (e.g. swapping the muxer, using other transports) it is recommended to create a server via the API. +For further customization (e.g. swapping the muxer, using other transports, use other database) it is recommended to create a server via the API. + +#### Datastore + +A rendezvous server needs to leverage a MySQL database as a datastore for the registrations. This needs to be configured in order to run a rendezvous server. You can rely on docker to run a MySQL database using a command like: + +```sh +docker run -p 3306:3306 -e MYSQL_ROOT_PASSWORD=your-secret-pw -e MYSQL_DATABASE=libp2p_rendezvous_db -d mysql:8 --default-authentication-plugin=mysql_native_password +``` + +Once a MySQL database is running, you can run the rendezvous server by providing the datastore configuration options as follows: + +```sh +libp2p-rendezvous-server --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db' +``` + +⚠️ For testing purposes you can skip using MySQL and use a memory datastore. This must not be used in production! For this you just need to provide the `--enableMemoryDatabase` option. #### PeerId -You can create a [PeerId](https://github.com/libp2p/js-peer-id) via its [CLI](https://github.com/libp2p/js-peer-id#cli). +You can create a [PeerId](https://github.com/libp2p/js-peer-id) via its [CLI](https://github.com/libp2p/js-peer-id#cli) and use it in the rendezvous server. + +Once you have a generated PeerId json file, you can start the rendezvous with that PeerId by specifying its path via the `--peerId` flag: ```sh -libp2p-rendezvous-server --peerId id.json +peer-id --type=ed25519 > id.json +libp2p-rendezvous-server --peerId id.json --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db' ``` #### Multiaddrs @@ -60,10 +85,10 @@ libp2p-rendezvous-server --peerId id.json You can specify the libp2p rendezvous server listen and announce multiaddrs. This server is configured with [libp2p-tcp](https://github.com/libp2p/js-libp2p-tcp) and [libp2p-websockets](https://github.com/libp2p/js-libp2p-websockets) and addresses with this transports should be used. It can always be modified via the API. ```sh -libp2p-rendezvous-server --peerId id.json --listenMultiaddrs '/ip4/127.0.0.1/tcp/15002/ws' '/ip4/127.0.0.1/tcp/8000' --announceMultiaddrs '/dns4/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' '/dns6/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' +libp2p-rendezvous-server --peerId id.json --listenMultiaddrs '/ip4/127.0.0.1/tcp/15002/ws' '/ip4/127.0.0.1/tcp/8000' --announceMultiaddrs '/dns4/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' '/dns6/test.io/tcp/443/wss/p2p/12D3KooWAuEpJKhCAfNcHycKcZCv9Qy69utLAJ3MobjKpsoKbrGA' --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db' ``` -By default it listens on `/ip4/127.0.0.1/tcp/15002/ws` and has no announce multiaddrs specified. +By default it listens on `/ip4/127.0.0.1/tcp/8000` and `/ip4/127.0.0.1/tcp/15003/ws`. It has no announce multiaddrs specified. #### Metrics @@ -81,8 +106,30 @@ libp2p-rendezvous-server --disableMetrics ### Docker Setup +```yml +version: '3.1' +services: + db: + image: mysql + volumes: + - mysql-db:/var/lib/mysql + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + MYSQL_ROOT_PASSWORD: your-secret-pw + MYSQL_DATABASE: libp2p_rendezvous_db + ports: + - "3306:3306" +volumes: + mysql-db: +``` + +## Library + TODO +Datastores + ## Contribute Feel free to join in. All welcome. Open an [issue](https://github.com/libp2p/js-libp2p-rendezvous/issues)! diff --git a/mysql/Dockerfile b/mysql/Dockerfile deleted file mode 100644 index 44f5a21..0000000 --- a/mysql/Dockerfile +++ /dev/null @@ -1,10 +0,0 @@ -# Derived from official mysql image (our base image) -FROM mysql - -# Add env variables -ENV MYSQL_DATABASE libp2p-rendezvous -ENV MYSQL_ROOT_PASSWORD=my-secret-pw -ENV MYSQL_USER=vsantos -ENV MYSQL_PASSWORD=my-secret-pw - -EXPOSE 3306 diff --git a/mysql/docker-compose.yml b/mysql/docker-compose.yml index 45cba3f..79df281 100644 --- a/mysql/docker-compose.yml +++ b/mysql/docker-compose.yml @@ -7,14 +7,14 @@ services: command: --default-authentication-plugin=mysql_native_password restart: always environment: - # MYSQL_ROOT_PASSWORD: my-secret-pw - # MYSQL_USER: libp2p - # MYSQL_PASSWORD: my-secret-pw - # MYSQL_DATABASE: libp2p_rendezvous_db - MYSQL_DATABASE: ${DATABASE} - MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD} - MYSQL_USER: ${USER} - MYSQL_PASSWORD: ${PASSWORD} + MYSQL_ROOT_PASSWORD: my-secret-pw + MYSQL_USER: libp2p + MYSQL_PASSWORD: my-secret-pw + MYSQL_DATABASE: libp2p_rendezvous_db + # MYSQL_DATABASE: ${DATABASE} + # MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD} + # MYSQL_USER: ${USER} + # MYSQL_PASSWORD: ${PASSWORD} ports: - "3306:3306" volumes: diff --git a/package.json b/package.json index 4b9054f..addb180 100644 --- a/package.json +++ b/package.json @@ -30,6 +30,9 @@ "node": ">=12.0.0", "npm": ">=6.0.0" }, + "browser": { + "mysql": false + }, "scripts": { "lint": "aegir lint", "build": "aegir build", @@ -48,7 +51,7 @@ "it-buffer": "^0.1.2", "it-length-prefixed": "^3.1.0", "it-pipe": "^1.1.0", - "libp2p": "libp2p/js-libp2p#chore/add-typedfs-with-post-install", + "libp2p": "^0.30.0", "libp2p-mplex": "^0.10.0", "libp2p-noise": "^2.0.1", "libp2p-tcp": "^0.15.1", @@ -59,7 +62,8 @@ "mysql": "^2.18.1", "peer-id": "^0.14.1", "protons": "^2.0.0", - "streaming-iterables": "^5.0.2" + "streaming-iterables": "^5.0.2", + "uint8arrays": "^2.0.5" }, "devDependencies": { "aegir": "^29.2.2", @@ -67,6 +71,9 @@ "chai-as-promised": "^7.1.1", "delay": "^4.4.0", "dirty-chai": "^2.0.1", + "execa": "^5.0.0", + "ipfs-utils": "^5.0.1", + "is-ci": "^2.0.0", "p-defer": "^3.0.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", diff --git a/sql b/sql new file mode 100644 index 0000000..bbf90e1 --- /dev/null +++ b/sql @@ -0,0 +1,27 @@ +USE libp2p_rendezvous_db + +CREATE TABLE IF NOT EXISTS registration ( + id INT UNSIGNED NOT NULL AUTO_INCREMENT, + namespace varchar(255) NOT NULL, + peer_id varchar(255) NOT NULL, + PRIMARY KEY (id), + INDEX (namespace, peer_id) +); + +CREATE TABLE IF NOT EXISTS cookie ( + id varchar(21), + namespace varchar(255), + reg_id INT UNSIGNED, + peer_id varchar(255) NOT NULL, + created_at datetime DEFAULT CURRENT_TIMESTAMP, + PRIMARY KEY (id, namespace, reg_id), + INDEX (created_at) +); + +INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmW8rAgaaA6sRydK1k6vonShQME47aDxaFidbtMevWs73t'); + +SELECT * FROM registration + +SELECT * FROM cookie + +INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmZqCdSzgpsmB3Qweb9s4fojAoqELWzqku21UVrqtVSKi4'); diff --git a/src/index.js b/src/index.js index cd31a96..a090cea 100644 --- a/src/index.js +++ b/src/index.js @@ -117,7 +117,7 @@ class Rendezvous { const registerTasks = [] /** - * @param {Multiaddr} m + * @param {Multiaddr} m * @returns {Promise} */ const taskFn = async (m) => { @@ -280,16 +280,19 @@ class Rendezvous { // track registrations yield registrationTransformer(r) - // Store cookie - const nsCookies = this._cookies.get(ns) || new Map() - nsCookies.set(m.toString(), toString(recMessage.discoverResponse.cookie)) - this._cookies.set(ns, nsCookies) - limit-- if (limit === 0) { - return + break } } + + // Store cookie + const c = recMessage.discoverResponse.cookie + if (c && c.length) { + const nsCookies = this._cookies.get(ns) || new Map() + nsCookies.set(m.toString(), toString(c)) + this._cookies.set(ns, nsCookies) + } } } } diff --git a/src/server/bin.js b/src/server/bin.js index b21efab..3bfcce7 100644 --- a/src/server/bin.js +++ b/src/server/bin.js @@ -22,7 +22,7 @@ const { NOISE: Crypto } = require('libp2p-noise') const PeerId = require('peer-id') const RendezvousServer = require('./index') -const Datastore = require('./datastores/memory') +const Datastore = require('./datastores/mysql') const { getAnnounceAddresses, getListenAddresses } = require('./utils') async function main () { @@ -30,8 +30,6 @@ async function main () { let metricsServer const metrics = !(argv.disableMetrics || process.env.DISABLE_METRICS) const metricsPort = argv.metricsPort || argv.mp || process.env.METRICS_PORT || '8003' - // const metricsMa = multiaddr(argv.metricsMultiaddr || argv.ma || process.env.METRICSMA || '/ip4/127.0.0.1/tcp/8003') - // const metricsAddr = metricsMa.nodeAddress() // Multiaddrs const listenAddresses = getListenAddresses(argv) @@ -48,8 +46,14 @@ async function main () { log('If you want to keep the same address for the server you should provide a peerId with --peerId ') } + const datastore = new Datastore({ + host: 'localhost', + user: 'root', + password: 'test-secret-pw', + database: 'libp2p_rendezvous_db' + }) + // Create Rendezvous server - const datastore = new Datastore() const rendezvousServer = new RendezvousServer({ modules: { transport: [Websockets, TCP], diff --git a/src/server/datastores/memory.js b/src/server/datastores/memory.js index 3e49447..9fcad86 100644 --- a/src/server/datastores/memory.js +++ b/src/server/datastores/memory.js @@ -53,12 +53,11 @@ class Memory { return Promise.resolve() } - stop () { - this.nsRegistrations.clear() - this.cookieRegistrations.clear() - } + stop () {} reset () { + this.nsRegistrations.clear() + this.cookieRegistrations.clear() return Promise.resolve() } diff --git a/src/server/datastores/mysql.js b/src/server/datastores/mysql.js index 4efca90..445f487 100644 --- a/src/server/datastores/mysql.js +++ b/src/server/datastores/mysql.js @@ -4,6 +4,9 @@ const debug = require('debug') const log = debug('libp2p:rendezvous-server:mysql') log.error = debug('libp2p:rendezvous-server:mysql:error') +const errCode = require('err-code') +const { codes: errCodes } = require('../errors') + const mysql = require('mysql') /** @@ -40,6 +43,13 @@ class Mysql { insecureAuth, multipleStatements } + + /** + * Peer string identifier with current add operations. + * + * @type {Map>} + */ + this._registeringPeer = new Map() } /** @@ -60,8 +70,18 @@ class Mysql { this.conn.end() } - reset () { - return Promise.resolve() + async reset () { + await new Promise((resolve, reject) => { + this.conn.query(` + DROP TABLE IF EXISTS cookie; + DROP TABLE IF EXISTS registration; + `, (err) => { + if (err) { + return reject(err) + } + resolve() + }) + }) } /** @@ -74,14 +94,27 @@ class Mysql { * @returns {Promise} */ addRegistration (namespace, peerId, signedPeerRecord, ttl) { + const id = peerId.toB58String() + const opId = String(Math.random() + Date.now()) + const peerOps = this._registeringPeer.get(id) || new Set() + + peerOps.add(opId) + this._registeringPeer.set(id, peerOps) + return new Promise((resolve, reject) => { this.conn.query('INSERT INTO ?? SET ?', ['registration', { namespace, - peer_id: peerId, + peer_id: id, signed_peer_record: Buffer.from(signedPeerRecord), expiration: new Date(Date.now() + ttl) }], (err) => { + // Remove Operation + peerOps.delete(opId) + if (!peerOps.size) { + this._registeringPeer.delete(id) + } + if (err) { return reject(err) } @@ -101,23 +134,40 @@ class Mysql { * @returns {Promise<{ registrations: Array, cookie?: string }>} */ async getRegistrations (namespace, { limit = 10, cookie } = {}) { - // TODO: transaction + if (cookie) { + const cookieEntries = await new Promise((resolve, reject) => { + this.conn.query( + 'SELECT * FROM cookie WHERE id = ? LIMIT 1', + [cookie], + (err, results) => { + if (err) { + return reject(err) + } + resolve(results) + } + ) + }) + if (!cookieEntries.length) { + throw errCode(new Error('no registrations for the given cookie'), errCodes.INVALID_COOKIE) + } + } + const cookieWhereNotExists = () => { if (!cookie) return '' return ` AND NOT EXISTS ( SELECT null FROM cookie c - WHERE r.id = c.reg_id AND c.namespace = r.namespace + WHERE r.id = c.reg_id AND c.namespace = r.namespace AND c.id = ? )` } const results = await new Promise((resolve, reject) => { this.conn.query( - `SELECT id, namespace, signed_peer_record, expiration FROM registration r - WHERE namespace = ? AND expiration >= NOW()${cookieWhereNotExists()} + `SELECT id, namespace, peer_id, signed_peer_record, expiration FROM registration r + WHERE namespace = ? AND expiration >= NOW() ${cookieWhereNotExists()} ORDER BY expiration DESC LIMIT ?`, - [namespace, limit], + [namespace, cookie || limit, limit], (err, results) => { if (err) { return reject(err) @@ -139,8 +189,8 @@ class Mysql { // Store in cookies if results available await new Promise((resolve, reject) => { this.conn.query( - `INSERT INTO ?? (id, namespace, reg_id) VALUES ${results.map((entry) => - `("${this.conn.escape(cookie)}", "${this.conn.escape(entry.namespace)}", "${this.conn.escape(entry.id)}")` + `INSERT INTO ?? (id, namespace, reg_id, peer_id) VALUES ${results.map((entry) => + `(${this.conn.escape(cookie)}, ${this.conn.escape(entry.namespace)}, ${this.conn.escape(entry.id)}, ${this.conn.escape(entry.peer_id)})` )}`, ['cookie'] , (err) => { if (err) { @@ -177,12 +227,26 @@ class Mysql { if (err) { return reject(err) } - resolve(res[0]['COUNT(1)']) + // DoS attack defense check + const pendingReg = this._getNumberOfPendingRegistrationsFromPeer(peerId) + resolve(res[0]['COUNT(1)'] + pendingReg) } ) }) } + /** + * Get number of ongoing registrations for a peer. + * + * @param {PeerId} peerId + * @returns {number} + */ + _getNumberOfPendingRegistrationsFromPeer (peerId) { + const peerOps = this._registeringPeer.get(peerId.toB58String()) || new Set() + + return peerOps.size + } + /** * Remove registration of a given namespace to a peer * @@ -194,14 +258,16 @@ class Mysql { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query('DELETE FROM registration WHERE peer_id = ? AND namespace = ?', - [id, ns], - (err) => { - if (err) { - return reject(err) - } - resolve() - }) + this.conn.query(` + DELETE FROM cookie WHERE peer_id = ? AND namespace = ?; + DELETE FROM registration WHERE peer_id = ? AND namespace = ? + `, [id, ns, id, ns], + (err) => { + if (err) { + return reject(err) + } + resolve() + }) }) } @@ -215,14 +281,16 @@ class Mysql { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query('DELETE FROM registration WHERE peer_id = ?', - [id], - (err) => { - if (err) { - return reject(err) - } - resolve() - }) + this.conn.query(` + DELETE FROM cookie WHERE peer_id = ?; + DELETE FROM registration WHERE peer_id = ? + `, [id, id], + (err) => { + if (err) { + return reject(err) + } + resolve() + }) }) } @@ -248,9 +316,9 @@ class Mysql { id varchar(21), namespace varchar(255), reg_id INT UNSIGNED, + peer_id varchar(255) NOT NULL, created_at datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id, namespace, reg_id), - FOREIGN KEY (reg_id) REFERENCES registration(id), INDEX (created_at) ); `, (err) => { @@ -259,6 +327,33 @@ class Mysql { } resolve() }) + // this.conn.query(` + // CREATE TABLE IF NOT EXISTS registration ( + // id INT UNSIGNED NOT NULL AUTO_INCREMENT, + // namespace varchar(255) NOT NULL, + // peer_id varchar(255) NOT NULL, + // signed_peer_record blob NOT NULL, + // expiration timestamp NOT NULL, + // PRIMARY KEY (id), + // INDEX (namespace, expiration, peer_id) + // ); + + // CREATE TABLE IF NOT EXISTS cookie ( + // id varchar(21), + // namespace varchar(255), + // reg_id INT UNSIGNED, + // peer_id varchar(255) NOT NULL, + // created_at datetime DEFAULT CURRENT_TIMESTAMP, + // PRIMARY KEY (id, namespace, reg_id), + // FOREIGN KEY (reg_id) REFERENCES registration(id), + // INDEX (created_at) + // ); + // `, (err) => { + // if (err) { + // return reject(err) + // } + // resolve() + // }) }) } } diff --git a/src/server/index.js b/src/server/index.js index a4fbbe8..4d029ab 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -59,7 +59,7 @@ class RendezvousServer extends Libp2p { this._maxDiscoveryLimit = options.maxDiscoveryLimit || MAX_DISCOVER_LIMIT this._maxRegistrations = options.maxRegistrations || MAX_REGISTRATIONS - this.datastore = options.datastore + this.rendezvousDatastore = options.datastore // TODO: REMOVE! /** @@ -93,7 +93,7 @@ class RendezvousServer extends Libp2p { log('starting') - await this.datastore.start() + await this.rendezvousDatastore.start() // TODO: + use module // Garbage collection @@ -115,7 +115,7 @@ class RendezvousServer extends Libp2p { // clearTimeout(this._timeout) - this.datastore.stop() + this.rendezvousDatastore.stop() super.stop() log('stopped') @@ -177,7 +177,7 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async addRegistration (ns, peerId, signedPeerRecord, ttl) { - await this.datastore.addRegistration(ns, peerId, signedPeerRecord, ttl) + await this.rendezvousDatastore.addRegistration(ns, peerId, signedPeerRecord, ttl) log(`added registration for the namespace ${ns} with peer ${peerId.toB58String()}`) } @@ -189,7 +189,7 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async removeRegistration (ns, peerId) { - await this.datastore.removeRegistration(ns, peerId) + await this.rendezvousDatastore.removeRegistration(ns, peerId) log(`removed existing registrations for the namespace ${ns} - peer ${peerId.toB58String()} pair`) } @@ -200,7 +200,7 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async removePeerRegistrations (peerId) { - await this.datastore.removePeerRegistrations(peerId) + await this.rendezvousDatastore.removePeerRegistrations(peerId) log(`removed existing registrations for peer ${peerId.toB58String()}`) } @@ -214,7 +214,7 @@ class RendezvousServer extends Libp2p { * @returns {Promise<{ registrations: Array, cookie?: string }>} */ async getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { - return await this.datastore.getRegistrations(ns, { limit, cookie }) + return await this.rendezvousDatastore.getRegistrations(ns, { limit, cookie }) } /** @@ -224,7 +224,7 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async getNumberOfRegistrationsFromPeer (peerId) { - return await this.datastore.getNumberOfRegistrationsFromPeer(peerId) + return await this.rendezvousDatastore.getNumberOfRegistrationsFromPeer(peerId) } } diff --git a/src/server/rpc/handlers/discover.js b/src/server/rpc/handlers/discover.js index d4a301b..69a3c36 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/server/rpc/handlers/discover.js @@ -64,7 +64,7 @@ module.exports = (rendezvousPoint) => { return { type: MESSAGE_TYPE.DISCOVER_RESPONSE, discoverResponse: { - cookie: fromString(cookie), + cookie: cookie && fromString(cookie), registrations: registrations.map((r) => ({ ns: r.ns, signedPeerRecord: r.signedPeerRecord, diff --git a/src/server/utils.js b/src/server/utils.js index 97918ca..0d5dfbd 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -1,41 +1,52 @@ 'use strict' -const multiaddr = require('multiaddr') +function getExtraParams (alias1, alias2) { + const params = [] + + const flagIndex = process.argv.findIndex((e) => e === alias1 || e === alias2) + const tmpEndIndex = process.argv.slice(flagIndex + 1).findIndex((e) => e.startsWith('--')) + const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 + + for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { + params.push(process.argv[i + 1]) + } + + return params +} function getAnnounceAddresses (argv) { - const announceAddr = argv.announceMultiaddrs || argv.am - const announceAddresses = announceAddr ? [multiaddr(announceAddr).toString()] : [] + let announceAddresses = [] + const argvAddr = argv.announceMultiaddrs || argv.am - if (argv.announceMultiaddrs || argv.am) { - const flagIndex = process.argv.findIndex((e) => e === '--announceMultiaddrs' || e === '--am') - const tmpEndIndex = process.argv.slice(flagIndex + 1).findIndex((e) => e.startsWith('--')) - const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 + if (argvAddr) { + announceAddresses = [argvAddr] - for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { - announceAddresses.push(multiaddr(process.argv[i + 1]).toString()) - } + const extraParams = getExtraParams('--announceMultiaddrs', '--am') + extraParams.forEach((p) => announceAddresses.push(p)) + } else if (process.env.ANNOUNCE_MULTIADDRS) { + announceAddresses = process.env.ANNOUNCE_MULTIADDRS.split(',') } return announceAddresses } -module.exports.getAnnounceAddresses = getAnnounceAddresses - function getListenAddresses (argv) { - const listenAddr = argv.listenMultiaddrs || argv.lm || '/ip4/127.0.0.1/tcp/15002/ws' - const listenAddresses = [multiaddr(listenAddr).toString()] + let listenAddresses = ['/ip4/127.0.0.1/tcp/15003/ws', '/ip4/127.0.0.1/tcp/8000'] + const argvAddr = argv.listenMultiaddrs || argv.lm - if (argv.listenMultiaddrs || argv.lm) { - const flagIndex = process.argv.findIndex((e) => e === '--listenMultiaddrs' || e === '--lm') - const tmpEndIndex = process.argv.slice(flagIndex + 1).findIndex((e) => e.startsWith('--')) - const endIndex = tmpEndIndex !== -1 ? tmpEndIndex : process.argv.length - flagIndex - 1 + if (argvAddr) { + listenAddresses = [argvAddr] - for (let i = flagIndex + 1; i < flagIndex + endIndex; i++) { - listenAddresses.push(multiaddr(process.argv[i + 1]).toString()) - } + const extraParams = getExtraParams('--listenMultiaddrs', '--lm') + extraParams.forEach((p) => listenAddresses.push(p)) + } else if (process.env.LISTEN_MULTIADDRS) { + listenAddresses = process.env.LISTEN_MULTIADDRS.split(',') } return listenAddresses } -module.exports.getListenAddresses = getListenAddresses +module.exports = { + getAnnounceAddresses, + getListenAddresses +} diff --git a/test/client/api.spec.js b/test/client/api.spec.js index c7829d3..66bf19b 100644 --- a/test/client/api.spec.js +++ b/test/client/api.spec.js @@ -4,6 +4,7 @@ const { expect } = require('aegir/utils/chai') const sinon = require('sinon') +const delay = require('delay') const pWaitFor = require('p-wait-for') const multiaddr = require('multiaddr') @@ -80,7 +81,8 @@ describe('rendezvous api', () => { let clients // Create and start Libp2p - beforeEach(async () => { + beforeEach(async function () { + this.timeout(10e3) // Create Rendezvous Server rendezvousServer = await createRendezvousServer() await pWaitFor(() => rendezvousServer.multiaddrs.length > 0) @@ -95,14 +97,16 @@ describe('rendezvous api', () => { }) }) - afterEach(async () => { + afterEach(async function () { + this.timeout(10e3) sinon.restore() + await delay(500) // Await for datastore to be ready + await rendezvousServer.rendezvousDatastore.reset() await rendezvousServer.stop() - - for (const peer of clients) { + await Promise.all(clients.map(async (peer) => { await peer.rendezvous.stop() await peer.stop() - } + })) }) it('register throws error if a namespace is not provided', async () => { @@ -118,7 +122,10 @@ describe('rendezvous api', () => { .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_INVALID_NAMESPACE) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } }) it('register throws an error with an invalid ttl', async () => { @@ -128,7 +135,10 @@ describe('rendezvous api', () => { .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_INVALID_TTL) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } }) it('register throws an error with an invalid peerId', async () => { @@ -141,15 +151,29 @@ describe('rendezvous api', () => { .to.eventually.rejected() .and.have.property('code', RESPONSE_STATUS.E_NOT_AUTHORIZED) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } }) it('registers with an available rendezvous server node', async () => { - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + const registers = [] + + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + await clients[0].rendezvous.register(namespace) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(1) - expect(rendezvousServer.datastore.nsRegistrations.get(namespace)).to.exist() + // Peer2 discovers Peer0 registered in Peer1 + for await (const reg of clients[1].rendezvous.discover(namespace)) { + registers.push(reg) + } + + expect(registers).to.have.lengthOf(1) + expect(registers[0].ns).to.eql(namespace) }) it('unregister throws if a namespace is not provided', async () => { @@ -159,16 +183,21 @@ describe('rendezvous api', () => { }) it('unregisters with an available rendezvous server node', async () => { + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } + // Register - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) await clients[0].rendezvous.register(namespace) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(1) - expect(rendezvousServer.datastore.nsRegistrations.get(namespace)).to.exist() - // Unregister await clients[0].rendezvous.unregister(namespace) - expect(rendezvousServer.datastore.nsRegistrations.size).to.eql(0) + + // other client does not discovery any peer registered + for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + throw new Error('no registers should exist') + } }) it('unregister not fails if not registered', async () => { @@ -185,6 +214,7 @@ describe('rendezvous api', () => { expect(err.code).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) return } + throw new Error('discover should throw error if a namespace is not provided') }) @@ -222,7 +252,7 @@ describe('rendezvous api', () => { expect(rec.multiaddrs).to.eql(clients[0].multiaddrs) }) - it('discover finds registered peer for namespace once (cookie usage)', async () => { + it('discover finds registered peer for namespace once (cookie)', async () => { const registers = [] // Peer2 does not discovery any peer registered @@ -256,7 +286,8 @@ describe('rendezvous api', () => { let clients // Create and start Libp2p nodes - beforeEach(async () => { + beforeEach(async function () { + this.timeout(20e3) // Create Rendezvous Server rendezvousServers = await Promise.all([ createRendezvousServer(), @@ -278,8 +309,10 @@ describe('rendezvous api', () => { await Promise.all(rendezvousServers.map((libp2p) => libp2p.dial(relayAddr))) }) - afterEach(async () => { - await Promise.all(rendezvousServers.map((libp2p) => libp2p.stop())) + afterEach(async function () { + this.timeout(20e3) + await Promise.all(rendezvousServers.map((s) => s.rendezvousDatastore.reset())) + await Promise.all(rendezvousServers.map((s) => s.stop())) await Promise.all(clients.map((libp2p) => { libp2p.rendezvous.stop() return libp2p.stop() @@ -310,9 +343,10 @@ describe('rendezvous api', () => { await clients[0].rendezvous.unregister(namespace) // Peer2 does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } + // TODO: Cookies not available as they were removed + // for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line + // throw new Error('no registers should exist') + // } }) }) }) diff --git a/test/dos-attack-protection.spec.js b/test/dos-attack-protection.spec.js index 7259c27..f0b720d 100644 --- a/test/dos-attack-protection.spec.js +++ b/test/dos-attack-protection.spec.js @@ -12,7 +12,6 @@ const multiaddr = require('multiaddr') const Libp2p = require('libp2p') const RendezvousServer = require('../src/server') -const Datastore = require('../src/server/datastores/memory') const { PROTOCOL_MULTICODEC } = require('../src/server/constants') @@ -22,6 +21,7 @@ const RESPONSE_STATUS = Message.ResponseStatus const { createPeerId, + createDatastore, defaultLibp2pConfig } = require('./utils') @@ -42,7 +42,7 @@ describe('DoS attack protection', () => { beforeEach(async () => { [peerId] = await createPeerId() - datastore = new Datastore() + datastore = createDatastore() rServer = new RendezvousServer({ peerId: peerId, addresses: { @@ -64,6 +64,7 @@ describe('DoS attack protection', () => { }) afterEach(async () => { + await datastore.reset() await Promise.all([rServer, client].map((n) => n.stop())) }) @@ -103,6 +104,7 @@ describe('DoS attack protection', () => { expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_NOT_AUTHORIZED) // Only one record - expect(rServer.datastore.nsRegistrations.size).to.eql(1) + const { registrations } = await rServer.getRegistrations(ns) + expect(registrations).to.have.lengthOf(1) }) }) diff --git a/test/protocol.spec.js b/test/protocol.spec.js index 5606e8a..e22b774 100644 --- a/test/protocol.spec.js +++ b/test/protocol.spec.js @@ -13,7 +13,6 @@ const PeerId = require('peer-id') const Libp2p = require('libp2p') const RendezvousServer = require('../src/server') -const Datastore = require('../src/server/datastores/memory') const { PROTOCOL_MULTICODEC } = require('../src/server/constants') @@ -23,6 +22,7 @@ const RESPONSE_STATUS = Message.ResponseStatus const { createPeerId, + createDatastore, defaultLibp2pConfig } = require('./utils') @@ -44,8 +44,10 @@ describe('protocol', () => { }) // Create client and server and connect them - beforeEach(async () => { - datastore = new Datastore() + beforeEach(async function () { + this.timeout(10e3) + + datastore = createDatastore() rServer = new RendezvousServer({ peerId: peerIds[0], addresses: { @@ -65,7 +67,9 @@ describe('protocol', () => { await Promise.all([rServer, client].map((n) => n.start())) }) - afterEach(async () => { + afterEach(async function () { + this.timeout(10e3) + await datastore.reset() await Promise.all([rServer, client].map((n) => n.stop())) }) @@ -93,8 +97,6 @@ describe('protocol', () => { expect(recMessage).to.exist() expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(Message.ResponseStatus.OK) - - expect(rServer.datastore.nsRegistrations.size).to.eql(1) }) it('fails to register if invalid namespace', async () => { @@ -121,8 +123,6 @@ describe('protocol', () => { expect(recMessage).to.exist() expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) - - expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('fails to register if invalid ttl', async () => { @@ -149,8 +149,6 @@ describe('protocol', () => { expect(recMessage).to.exist() expect(recMessage.type).to.eql(MESSAGE_TYPE.REGISTER_RESPONSE) expect(recMessage.registerResponse.status).to.eql(RESPONSE_STATUS.E_INVALID_TTL) - - expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('fails to register if invalid signed peer record', async () => { @@ -199,13 +197,9 @@ describe('protocol', () => { for await (const _ of source) { } // eslint-disable-line } ) - - expect(rServer.datastore.nsRegistrations.size).to.eql(1) }) it('can unregister a namespace', async () => { - expect(rServer.datastore.nsRegistrations.size).to.eql(1) - const conn = await client.dial(multiaddrServer) const { stream } = await conn.newStream(PROTOCOL_MULTICODEC) @@ -223,8 +217,6 @@ describe('protocol', () => { for await (const _ of source) { } // eslint-disable-line } ) - - expect(rServer.datastore.nsRegistrations.size).to.eql(0) }) it('can discover a peer registered into a namespace', async () => { diff --git a/test/server.spec.js b/test/server.spec.js index b8cea64..8449372 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -9,11 +9,11 @@ const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') -const Datastore = require('../src/server/datastores/memory') const { codes: errCodes } = require('../src/server/errors') const { createPeerId, createSignedPeerRecord, + createDatastore, defaultLibp2pConfig } = require('./utils') @@ -35,11 +35,11 @@ describe('rendezvous server', () => { signedPeerRecords.push(spr.marshal()) } - datastore = new Datastore() + datastore = createDatastore() }) afterEach(async () => { - datastore = new Datastore() + await datastore.reset() rServer && await rServer.stop() }) @@ -59,6 +59,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -80,6 +81,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -100,6 +102,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -125,6 +128,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -154,6 +158,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() await rServer.removeRegistration(otherNamespace, peerIds[1]) }) @@ -163,6 +168,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -177,36 +183,12 @@ describe('rendezvous server', () => { expect(r.registrations).to.have.lengthOf(1) }) - it('gc expired records', async () => { - rServer = new RendezvousServer({ - ...defaultLibp2pConfig, - peerId: peerIds[0] - }, { datastore, gcInterval: 300 }) - - await rServer.start() - - // Add registration for peer 1 in test namespace - await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) - await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) - - let r = await rServer.getRegistrations(testNamespace) - expect(r.registrations).to.have.lengthOf(2) - - // wait for firt record to be removed - await delay(650) - r = await rServer.getRegistrations(testNamespace) - expect(r.registrations).to.have.lengthOf(1) - - await delay(400) - r = await rServer.getRegistrations(testNamespace) - expect(r.registrations).to.have.lengthOf(0) - }) - it('only new peers should be returned if cookie given', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -248,6 +230,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -271,6 +254,7 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() // Add registration for peer 1 in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) @@ -308,13 +292,38 @@ describe('rendezvous server', () => { ...defaultLibp2pConfig, peerId: peerIds[0] }, { datastore }) + await rServer.start() await expect(rServer.getRegistrations(testNamespace, { cookie: badCookie })) .to.eventually.be.rejectedWith(Error) .and.to.have.property('code', errCodes.INVALID_COOKIE) }) - it('garbage collector should remove cookies of discarded records', async () => { + it.skip('gc expired records', async () => { + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }, { datastore, gcInterval: 300 }) + await rServer.start() + + // Add registration for peer 1 in test namespace + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + + let r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(2) + + // wait for firt record to be removed + await delay(650) + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) + + await delay(400) + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(0) + }) + + it.skip('garbage collector should remove cookies of discarded records', async () => { rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] @@ -325,17 +334,17 @@ describe('rendezvous server', () => { await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) // Get current registrations - const { cookie, registrations } = await rServer.getRegistrations(testNamespace) + const { registrations } = await rServer.getRegistrations(testNamespace) expect(registrations).to.exist() expect(registrations).to.have.lengthOf(1) // Verify internal state - expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(1) - expect(rServer.datastore.cookieRegistrations.get(cookie)).to.exist() + // expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(1) + // expect(rServer.datastore.cookieRegistrations.get(cookie)).to.exist() await delay(800) - expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(0) - expect(rServer.datastore.cookieRegistrations.get(cookie)).to.not.exist() + // expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(0) + // expect(rServer.datastore.cookieRegistrations.get(cookie)).to.not.exist() }) }) diff --git a/test/utils.js b/test/utils.js index 49245e6..e73def6 100644 --- a/test/utils.js +++ b/test/utils.js @@ -6,6 +6,7 @@ const { NOISE: Crypto } = require('libp2p-noise') const PeerId = require('peer-id') const pTimes = require('p-times') +const { isNode } = require('ipfs-utils/src/env') const Libp2p = require('libp2p') const multiaddr = require('multiaddr') @@ -13,7 +14,6 @@ const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') const RendezvousServer = require('../src/server') -const Datastore = require('../src/server/datastores/memory') const Peers = require('./fixtures/peers') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') @@ -86,7 +86,7 @@ module.exports.createPeer = createPeer async function createRendezvousServer ({ config = {}, started = true } = {}) { const [peerId] = await createPeerId({ fixture: false }) - const datastore = new Datastore() + const datastore = createDatastore() const rendezvous = new RendezvousServer({ peerId: peerId, addresses: { @@ -117,3 +117,22 @@ async function createSignedPeerRecord (peerId, multiaddrs) { } module.exports.createSignedPeerRecord = createSignedPeerRecord + +function createDatastore () { + if (!isNode) { + const Memory = require('../src/server/datastores/memory') + return new Memory() + } + + const MySql = require('../src/server/datastores/mysql') + const datastore = new MySql({ + host: 'localhost', + user: 'root', + password: 'test-secret-pw', + database: 'libp2p_rendezvous_db' + }) + + return datastore +} + +module.exports.createDatastore = createDatastore From 9b294f52a25aeec7a2421449b21ac2ab54816356 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 24 Dec 2020 12:46:10 +0000 Subject: [PATCH 30/38] feat: gc --- package.json | 1 + sql | 5 +- src/server/constants.js | 7 +- src/server/datastores/interface.ts | 8 +- src/server/datastores/memory.js | 61 ++++++++++--- src/server/datastores/mysql.js | 84 ++++++++---------- src/server/index.js | 133 +++++++++++++++------------- src/server/rpc/handlers/register.js | 2 +- test/dos-attack-protection.spec.js | 2 +- 9 files changed, 173 insertions(+), 130 deletions(-) diff --git a/package.json b/package.json index addb180..4a71f77 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "mysql": "^2.18.1", "peer-id": "^0.14.1", "protons": "^2.0.0", + "set-delayed-interval": "^1.0.0", "streaming-iterables": "^5.0.2", "uint8arrays": "^2.0.5" }, diff --git a/sql b/sql index bbf90e1..fed9284 100644 --- a/sql +++ b/sql @@ -4,6 +4,7 @@ CREATE TABLE IF NOT EXISTS registration ( id INT UNSIGNED NOT NULL AUTO_INCREMENT, namespace varchar(255) NOT NULL, peer_id varchar(255) NOT NULL, + expiration timestamp DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id), INDEX (namespace, peer_id) ); @@ -20,8 +21,8 @@ CREATE TABLE IF NOT EXISTS cookie ( INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmW8rAgaaA6sRydK1k6vonShQME47aDxaFidbtMevWs73t'); -SELECT * FROM registration +SELECT * FROM registration; -SELECT * FROM cookie +SELECT * FROM cookie; INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmZqCdSzgpsmB3Qweb9s4fojAoqELWzqku21UVrqtVSKi4'); diff --git a/src/server/constants.js b/src/server/constants.js index fd8391a..cc70e50 100644 --- a/src/server/constants.js +++ b/src/server/constants.js @@ -2,7 +2,12 @@ exports.MAX_NS_LENGTH = 255 exports.MAX_DISCOVER_LIMIT = 1000 -exports.MAX_REGISTRATIONS = 1000 +exports.MAX_PEER_REGISTRATIONS = 1000 exports.MIN_TTL = 7.2e6 exports.MAX_TTL = 2.592e+8 exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' +exports.GC_BOOT_DELAY = 10e6 +exports.GC_INTERVAL = 7.2e6 +exports.GC_MIN_INTERVAL = 3e6 +exports.GC_MIN_REGISTRATIONS = 1000 +exports.GC_MAX_REGISTRATIONS = 10e6 diff --git a/src/server/datastores/interface.ts b/src/server/datastores/interface.ts index 8913214..b7be9cc 100644 --- a/src/server/datastores/interface.ts +++ b/src/server/datastores/interface.ts @@ -13,6 +13,10 @@ export interface Datastore { * Tear down datastore. */ stop (): void; + /** + * Run datastore garbage collector to remove expired records. + */ + gc (): Promise; /** * Add a rendezvous registrations. */ @@ -28,11 +32,11 @@ export interface Datastore { /** * Remove registration of a given namespace to a peer. */ - removeRegistration (ns: string, peerId: PeerId): Promise; + removeRegistration (ns: string, peerId: PeerId): Promise; /** * Remove all registrations of a given peer. */ - removePeerRegistrations (peerId: PeerId): Promise; + removePeerRegistrations (peerId: PeerId): Promise; /** * Reset content */ diff --git a/src/server/datastores/memory.js b/src/server/datastores/memory.js index 9fcad86..4c4fcfa 100644 --- a/src/server/datastores/memory.js +++ b/src/server/datastores/memory.js @@ -61,6 +61,43 @@ class Memory { return Promise.resolve() } + /** + * Run datastore garbage collector to remove expired records. + * + * @returns {Promise} + */ + gc () { + const now = Date.now() + const removedIds = [] + + // Iterate namespaces + this.nsRegistrations.forEach((nsEntry) => { + // Iterate registrations for namespaces + nsEntry.forEach((nsReg, idStr) => { + if (now >= nsReg.expiration) { + nsEntry.delete(idStr) + removedIds.push(nsReg.id) + + log(`gc removed namespace entry for ${idStr}`) + } + }) + }) + + // Remove outdated records references from cookies + for (const [key, idSet] of this.cookieRegistrations.entries()) { + const filteredIds = Array.from(idSet).filter((id) => !removedIds.includes(id)) + + if (filteredIds && filteredIds.length) { + this.cookieRegistrations.set(key, new Set(filteredIds)) + } else { + // Empty + this.cookieRegistrations.delete(key) + } + } + + return Promise.resolve(removedIds.length) + } + /** * Add an entry to the registration table. * @@ -168,13 +205,14 @@ class Memory { * * @param {string} ns * @param {PeerId} peerId - * @returns {Promise} + * @returns {Promise} */ removeRegistration (ns, peerId) { + let count = 0 const nsReg = this.nsRegistrations.get(ns) - if (nsReg) { - nsReg.delete(peerId.toB58String()) + if (nsReg && nsReg.delete(peerId.toB58String())) { + count += 1 // Remove registrations map to namespace if empty if (!nsReg.size) { @@ -183,27 +221,30 @@ class Memory { log('removed existing registrations for the namespace - peer pair:', ns, peerId.toB58String()) } - return Promise.resolve() + return Promise.resolve(count) } /** * Remove all registrations of a given peer * * @param {PeerId} peerId - * @returns {Promise} + * @returns {Promise} */ removePeerRegistrations (peerId) { + let count = 0 for (const [ns, nsReg] of this.nsRegistrations.entries()) { - nsReg.delete(peerId.toB58String()) + if (nsReg.delete(peerId.toB58String())) { + count += 1 - // Remove registrations map to namespace if empty - if (!nsReg.size) { - this.nsRegistrations.delete(ns) + // Remove registrations map to namespace if empty + if (!nsReg.size) { + this.nsRegistrations.delete(ns) + } } } log('removed existing registrations for peer', peerId.toB58String()) - return Promise.resolve() + return Promise.resolve(count) } } diff --git a/src/server/datastores/mysql.js b/src/server/datastores/mysql.js index 445f487..0dd08ea 100644 --- a/src/server/datastores/mysql.js +++ b/src/server/datastores/mysql.js @@ -84,6 +84,23 @@ class Mysql { }) } + /** + * Run datastore garbage collector to remove expired records. + * + * @returns {Promise} + */ + gc () { + return new Promise((resolve, reject) => { + this.conn.query('DELETE FROM registration WHERE expiration <= NOW()', + (err, res) => { + if (err) { + return reject(err) + } + resolve(res.affectedRows) + }) + }) + } + /** * Add an entry to the registration table. * @@ -252,22 +269,19 @@ class Mysql { * * @param {string} ns * @param {PeerId} peerId - * @returns {Promise} + * @returns {Promise} */ removeRegistration (ns, peerId) { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query(` - DELETE FROM cookie WHERE peer_id = ? AND namespace = ?; - DELETE FROM registration WHERE peer_id = ? AND namespace = ? - `, [id, ns, id, ns], - (err) => { - if (err) { - return reject(err) - } - resolve() - }) + this.conn.query('DELETE FROM registration WHERE peer_id = ? AND namespace = ?', [id, ns], + (err, res) => { + if (err) { + return reject(err) + } + resolve(res.affectedRows) + }) }) } @@ -275,22 +289,19 @@ class Mysql { * Remove all registrations of a given peer * * @param {PeerId} peerId - * @returns {Promise} + * @returns {Promise} */ removePeerRegistrations (peerId) { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query(` - DELETE FROM cookie WHERE peer_id = ?; - DELETE FROM registration WHERE peer_id = ? - `, [id, id], - (err) => { - if (err) { - return reject(err) - } - resolve() - }) + this.conn.query('DELETE FROM registration WHERE peer_id = ?', [id], + (err, res) => { + if (err) { + return reject(err) + } + resolve(res.affectedRows) + }) }) } @@ -300,6 +311,7 @@ class Mysql { * @returns {Promise} */ _initDB () { + // TODO: Do I need created at cookie? return new Promise((resolve, reject) => { this.conn.query(` CREATE TABLE IF NOT EXISTS registration ( @@ -319,6 +331,7 @@ class Mysql { peer_id varchar(255) NOT NULL, created_at datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id, namespace, reg_id), + FOREIGN KEY (reg_id) REFERENCES registration(id) ON DELETE CASCADE, INDEX (created_at) ); `, (err) => { @@ -327,33 +340,6 @@ class Mysql { } resolve() }) - // this.conn.query(` - // CREATE TABLE IF NOT EXISTS registration ( - // id INT UNSIGNED NOT NULL AUTO_INCREMENT, - // namespace varchar(255) NOT NULL, - // peer_id varchar(255) NOT NULL, - // signed_peer_record blob NOT NULL, - // expiration timestamp NOT NULL, - // PRIMARY KEY (id), - // INDEX (namespace, expiration, peer_id) - // ); - - // CREATE TABLE IF NOT EXISTS cookie ( - // id varchar(21), - // namespace varchar(255), - // reg_id INT UNSIGNED, - // peer_id varchar(255) NOT NULL, - // created_at datetime DEFAULT CURRENT_TIMESTAMP, - // PRIMARY KEY (id, namespace, reg_id), - // FOREIGN KEY (reg_id) REFERENCES registration(id), - // INDEX (created_at) - // ); - // `, (err) => { - // if (err) { - // return reject(err) - // } - // resolve() - // }) }) } } diff --git a/src/server/index.js b/src/server/index.js index 4d029ab..f9b8608 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -4,6 +4,10 @@ const debug = require('debug') const log = Object.assign(debug('libp2p:rendezvous-server'), { error: debug('libp2p:rendezvous-server:err') }) +const { + setDelayedInterval, + clearDelayedInterval +} = require('set-delayed-interval') const Libp2p = require('libp2p') const PeerId = require('peer-id') @@ -14,7 +18,12 @@ const { MAX_TTL, MAX_NS_LENGTH, MAX_DISCOVER_LIMIT, - MAX_REGISTRATIONS, + MAX_PEER_REGISTRATIONS, + GC_BOOT_DELAY, + GC_INTERVAL, + GC_MIN_INTERVAL, + GC_MIN_REGISTRATIONS, + GC_MAX_REGISTRATIONS, PROTOCOL_MULTICODEC } = require('./constants') @@ -36,7 +45,12 @@ const { * @property {number} [maxTtl = MAX_TTL] maxium acceptable ttl to store a registration * @property {number} [maxNsLength = MAX_NS_LENGTH] maxium acceptable namespace length * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maxium acceptable discover limit - * @property {number} [maxRegistrations = MAX_REGISTRATIONS] maxium acceptable registrations per peer + * @property {number} [maxPeerRegistrations = MAX_PEER_REGISTRATIONS] maxium acceptable registrations per peer + * @property {number} [gcBootDelay = GC_BOOT_DELAY] delay before starting garbage collector job + * @property {number} [gcMinInterval = GC_MIN_INTERVAL] minimum interval between each garbage collector job, in case maximum threshold reached + * @property {number} [gcInterval = GC_INTERVAL] interval between each garbage collector job + * @property {number} [gcMinRegistrations = GC_MIN_REGISTRATIONS] minimum number of registration for triggering garbage collector + * @property {number} [gcMaxRegistrations = GC_MAX_REGISTRATIONS] maximum number of registration for triggering garbage collector */ /** @@ -57,26 +71,18 @@ class RendezvousServer extends Libp2p { this._maxTtl = options.maxTtl || MAX_TTL this._maxNsLength = options.maxNsLength || MAX_NS_LENGTH this._maxDiscoveryLimit = options.maxDiscoveryLimit || MAX_DISCOVER_LIMIT - this._maxRegistrations = options.maxRegistrations || MAX_REGISTRATIONS + this._maxPeerRegistrations = options.maxPeerRegistrations || MAX_PEER_REGISTRATIONS this.rendezvousDatastore = options.datastore - // TODO: REMOVE! - /** - * Registrations per namespace, where a registration maps peer id strings to a namespace reg. - * - * @type {Map>} - */ - this.nsRegistrations = new Map() - - /** - * Registration ids per cookie. - * - * @type {Map>} - */ - this.cookieRegistrations = new Map() - - this._gc = this._gc.bind(this) + this._registrationsCount = 0 + this._lastGcTs = 0 + this._gcDelay = options.gcBootDelay || GC_BOOT_DELAY + this._gcInterval = options.gcInterval || GC_INTERVAL + this._gcMinInterval = options.gcMinInterval || GC_MIN_INTERVAL + this._gcMinRegistrations = options.gcMinRegistrations || GC_MIN_REGISTRATIONS + this._gcMaxRegistrations = options.gcMaxRegistrations || GC_MAX_REGISTRATIONS + this._gcJob = this._gcJob.bind(this) } /** @@ -87,21 +93,28 @@ class RendezvousServer extends Libp2p { async start () { super.start() - // if (this._interval) { - // return - // } + if (this._timeout) { + return + } log('starting') await this.rendezvousDatastore.start() - // TODO: + use module // Garbage collection - // this._timeout = setInterval(this._gc, this._gcDelay) + this._timeout = setDelayedInterval( + this._gcJob, this._gcInterval, this._gcDelay + ) // Incoming streams handling this.handle(PROTOCOL_MULTICODEC, rpc(this)) + // Remove peer records from memory as they are not needed + // TODO: This should be handled by PeerStore itself in the future + this.peerStore.on('peer', (peerId) => { + this.peerStore.delete(peerId) + }) + log('started') } @@ -112,8 +125,7 @@ class RendezvousServer extends Libp2p { */ stop () { this.unhandle(PROTOCOL_MULTICODEC) - - // clearTimeout(this._timeout) + clearDelayedInterval(this._timeout) this.rendezvousDatastore.stop() @@ -124,47 +136,29 @@ class RendezvousServer extends Libp2p { } /** - * Garbage collector to removed outdated registrations. + * Call garbage collector if enough registrations. * - * @returns {void} + * @returns {Promise} */ - _gc () { - log('gc starting') - // TODO: delete addressBook - - const now = Date.now() - const removedIds = [] - - // Iterate namespaces - this.nsRegistrations.forEach((nsEntry) => { - // Iterate registrations for namespaces - nsEntry.forEach((nsReg, idStr) => { - if (now >= nsReg.expiration) { - nsEntry.delete(idStr) - removedIds.push(nsReg.id) - - log(`gc removed namespace entry for ${idStr}`) - } - }) - }) - - // Remove outdated records references from cookies - for (const [key, idSet] of this.cookieRegistrations.entries()) { - const filteredIds = Array.from(idSet).filter((id) => !removedIds.includes(id)) - - if (filteredIds && filteredIds.length) { - this.cookieRegistrations.set(key, new Set(filteredIds)) - } else { - // Empty - this.cookieRegistrations.delete(key) - } + async _gcJob () { + if (this._registrationsCount > this._gcMinRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { + await this._gc() } + } - // if (!this._timeout) { - // return - // } + /** + * Run datastore garbage collector. + * + * @returns {Promise} + */ + async _gc () { + log('gc starting') - // this._timeout = setInterval(this._gc, this._gcInterval) + const count = await this.rendezvousDatastore.gc() + this._registrationsCount -= count + this._lastGcTs = Date.now() + + log('gc finished') } /** @@ -179,6 +173,13 @@ class RendezvousServer extends Libp2p { async addRegistration (ns, peerId, signedPeerRecord, ttl) { await this.rendezvousDatastore.addRegistration(ns, peerId, signedPeerRecord, ttl) log(`added registration for the namespace ${ns} with peer ${peerId.toB58String()}`) + + this._registrationsCount += 1 + // Manually trigger garbage collector if max registrations threshold reached + // and the minGc interval is finished + if (this._registrationsCount >= this._gcMaxRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { + this._gc() + } } /** @@ -189,8 +190,10 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async removeRegistration (ns, peerId) { - await this.rendezvousDatastore.removeRegistration(ns, peerId) + const count = await this.rendezvousDatastore.removeRegistration(ns, peerId) log(`removed existing registrations for the namespace ${ns} - peer ${peerId.toB58String()} pair`) + + this._registrationsCount -= count } /** @@ -200,8 +203,10 @@ class RendezvousServer extends Libp2p { * @returns {Promise} */ async removePeerRegistrations (peerId) { - await this.rendezvousDatastore.removePeerRegistrations(peerId) + const count = await this.rendezvousDatastore.removePeerRegistrations(peerId) log(`removed existing registrations for peer ${peerId.toB58String()}`) + + this._registrationsCount -= count } /** diff --git a/src/server/rpc/handlers/register.js b/src/server/rpc/handlers/register.js index 606a3dc..fbb8248 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/server/rpc/handlers/register.js @@ -64,7 +64,7 @@ module.exports = (rendezvousPoint) => { // simple limit to defend against trivial DoS attacks // example: a peer connects and keeps registering until it fills our memory const peerRegistrations = await rendezvousPoint.getNumberOfRegistrationsFromPeer(peerId) - if (peerRegistrations >= rendezvousPoint._maxRegistrations) { + if (peerRegistrations >= rendezvousPoint._maxPeerRegistrations) { log.error('unauthorized peer to register, too many registrations') return { diff --git a/test/dos-attack-protection.spec.js b/test/dos-attack-protection.spec.js index f0b720d..eaf3841 100644 --- a/test/dos-attack-protection.spec.js +++ b/test/dos-attack-protection.spec.js @@ -49,7 +49,7 @@ describe('DoS attack protection', () => { listen: [`${relayAddr}/p2p-circuit`] }, ...defaultLibp2pConfig - }, { maxRegistrations: 1, datastore }) // Maximum of one registration + }, { maxPeerRegistrations: 1, datastore }) // Maximum of one registration multiaddrServer = multiaddr(`${relayAddr}/p2p-circuit/p2p/${peerId.toB58String()}`) From 9bf5bcbc3484419bc1e0d166dff56e8f5bf8d8ae Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 24 Dec 2020 16:00:13 +0000 Subject: [PATCH 31/38] chore: add gc tests --- .aegir.js | 2 +- package.json | 1 + src/server/index.js | 23 ++++---- src/server/utils.js | 18 +++++- test/server.spec.js | 133 ++++++++++++++++++++++++++++++++++++-------- 5 files changed, 140 insertions(+), 37 deletions(-) diff --git a/.aegir.js b/.aegir.js index ce11caa..dd3f99b 100644 --- a/.aegir.js +++ b/.aegir.js @@ -65,7 +65,7 @@ const before = async () => { interval: 5000 }) // Some more time waiting - await delay(10e3) + await delay(12e3) } const after = async () => { diff --git a/package.json b/package.json index 4a71f77..3d4347e 100644 --- a/package.json +++ b/package.json @@ -76,6 +76,7 @@ "ipfs-utils": "^5.0.1", "is-ci": "^2.0.0", "p-defer": "^3.0.0", + "p-retry": "^4.2.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", "sinon": "^9.0.3" diff --git a/src/server/index.js b/src/server/index.js index f9b8608..2d4b46a 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -26,6 +26,7 @@ const { GC_MAX_REGISTRATIONS, PROTOCOL_MULTICODEC } = require('./constants') +const { fallbackNullish } = require('./utils') /** * @typedef {import('./datastores/interface').Datastore} Datastore @@ -65,23 +66,21 @@ class RendezvousServer extends Libp2p { constructor (libp2pOptions, options) { super(libp2pOptions) - this._gcDelay = options.gcDelay || 3e5 - this._gcInterval = options.gcInterval || 7.2e6 - this._minTtl = options.minTtl || MIN_TTL - this._maxTtl = options.maxTtl || MAX_TTL - this._maxNsLength = options.maxNsLength || MAX_NS_LENGTH - this._maxDiscoveryLimit = options.maxDiscoveryLimit || MAX_DISCOVER_LIMIT - this._maxPeerRegistrations = options.maxPeerRegistrations || MAX_PEER_REGISTRATIONS + this._minTtl = fallbackNullish(options.minTtl, MIN_TTL) + this._maxTtl = fallbackNullish(options.maxTtl, MAX_TTL) + this._maxNsLength = fallbackNullish(options.maxNsLength, MAX_NS_LENGTH) + this._maxDiscoveryLimit = fallbackNullish(options.maxDiscoveryLimit, MAX_DISCOVER_LIMIT) + this._maxPeerRegistrations = fallbackNullish(options.maxPeerRegistrations, MAX_PEER_REGISTRATIONS) this.rendezvousDatastore = options.datastore this._registrationsCount = 0 this._lastGcTs = 0 - this._gcDelay = options.gcBootDelay || GC_BOOT_DELAY - this._gcInterval = options.gcInterval || GC_INTERVAL - this._gcMinInterval = options.gcMinInterval || GC_MIN_INTERVAL - this._gcMinRegistrations = options.gcMinRegistrations || GC_MIN_REGISTRATIONS - this._gcMaxRegistrations = options.gcMaxRegistrations || GC_MAX_REGISTRATIONS + this._gcDelay = fallbackNullish(options.gcBootDelay, GC_BOOT_DELAY) + this._gcInterval = fallbackNullish(options.gcInterval, GC_INTERVAL) + this._gcMinInterval = fallbackNullish(options.gcMinInterval, GC_MIN_INTERVAL) + this._gcMinRegistrations = fallbackNullish(options.gcMinRegistrations, GC_MIN_REGISTRATIONS) + this._gcMaxRegistrations = fallbackNullish(options.gcMaxRegistrations, GC_MAX_REGISTRATIONS) this._gcJob = this._gcJob.bind(this) } diff --git a/src/server/utils.js b/src/server/utils.js index 0d5dfbd..becdf70 100644 --- a/src/server/utils.js +++ b/src/server/utils.js @@ -46,7 +46,23 @@ function getListenAddresses (argv) { return listenAddresses } +/** + * Nullish coalescing operator implementation + * + * @template T + * @param {any} value + * @param {T} d - default value + * @returns {T} + */ +function fallbackNullish (value, d) { + if (value === null || value === undefined) { + return d + } + return value +} + module.exports = { getAnnounceAddresses, - getListenAddresses + getListenAddresses, + fallbackNullish } diff --git a/test/server.spec.js b/test/server.spec.js index 8449372..c7acddd 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -3,6 +3,9 @@ const { expect } = require('aegir/utils/chai') const delay = require('delay') +const sinon = require('sinon') +const pRetry = require('p-retry') +const pWaitFor = require('p-wait-for') const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') @@ -41,6 +44,7 @@ describe('rendezvous server', () => { afterEach(async () => { await datastore.reset() rServer && await rServer.stop() + sinon.reset() }) it('can start a rendezvous server', async () => { @@ -299,52 +303,135 @@ describe('rendezvous server', () => { .and.to.have.property('code', errCodes.INVALID_COOKIE) }) - it.skip('gc expired records', async () => { + it('gc expired records on regular interval', async function () { + this.timeout(35e3) + rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }, { datastore, gcInterval: 300 }) + }, { + datastore, + gcInterval: 1000, + gcBootDelay: 1000, + gcMinInterval: 0, + gcMinRegistrations: 0 + }) + const spy = sinon.spy(rServer, '_gc') await rServer.start() - // Add registration for peer 1 in test namespace - await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) - await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 1000) + // Add registrations in test namespace + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1500) + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 3200) let r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(2) - // wait for firt record to be removed - await delay(650) + // wait for firt record to be removed (2nd gc) + await pWaitFor(() => spy.callCount >= 2) + r = await rServer.getRegistrations(testNamespace) expect(r.registrations).to.have.lengthOf(1) - await delay(400) - r = await rServer.getRegistrations(testNamespace) - expect(r.registrations).to.have.lengthOf(0) + // wait for second record to be removed + await pRetry(async () => { + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(0) + }) }) - it.skip('garbage collector should remove cookies of discarded records', async () => { + it('gc expired records when maximum threshold', async function () { + this.timeout(35e3) + rServer = new RendezvousServer({ ...defaultLibp2pConfig, peerId: peerIds[0] - }, { datastore, gcDelay: 300, gcInterval: 300 }) + }, { + datastore, + // gcMinInterval: 0, + gcMaxRegistrations: 2 + }) + const spy = sinon.spy(rServer, '_gc') await rServer.start() - // Add registration for peer 1 in test namespace + // Add registrations in test namespace await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) - // Get current registrations - const { registrations } = await rServer.getRegistrations(testNamespace) - expect(registrations).to.exist() - expect(registrations).to.have.lengthOf(1) + let r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) + + // Validate peer + let envelope = await Envelope.openAndCertify(r.registrations[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope.peerId.toString()).to.eql(peerIds[1].toString()) + + // Wait for previous record to be expired + await delay(500) + + // Add registrations in test namespace exceending the max number for gc trigger + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 3200) + + await pWaitFor(() => spy.callCount === 1) - // Verify internal state - // expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(1) - // expect(rServer.datastore.cookieRegistrations.get(cookie)).to.exist() + // retry as rServer._gc is async and it can be removing + await pRetry(async () => { + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) - await delay(800) + envelope = await Envelope.openAndCertify(r.registrations[0].signedPeerRecord, PeerRecord.DOMAIN) + expect(envelope.peerId.toString()).to.eql(peerIds[2].toString()) + }) + }) + + it('gc expired records when maximum threshold only if gc min interval', async function () { + this.timeout(45e3) + + rServer = new RendezvousServer({ + ...defaultLibp2pConfig, + peerId: peerIds[0] + }, { + datastore, + gcMaxRegistrations: 2 + }) + const spy = sinon.spy(rServer, '_gc') + await rServer.start() + + // Add registrations in test namespace + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 500) + + let r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) + + // Wait for previous record to be expired + await delay(500) + + // Add registrations in test namespace exceending the max number for gc trigger + await rServer.addRegistration(testNamespace, peerIds[2], signedPeerRecords[2], 3000) + + // Wait for gc + await pWaitFor(() => spy.callCount === 1) + + // retry as rServer._gc is async and it can take longer to finish + await pRetry(async () => { + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(1) + }) + + // Wait for second record to be expired + await delay(3000) + + // Add a new registration + await rServer.addRegistration(testNamespace, peerIds[1], signedPeerRecords[1], 1000) - // expect(rServer.datastore.nsRegistrations.get(testNamespace).size).to.eql(0) - // expect(rServer.datastore.cookieRegistrations.get(cookie)).to.not.exist() + await Promise.race([ + async () => { + // GC should not be triggered, even with max registrations as minInterval was not reached + await pWaitFor(() => spy.callCount === 2) + throw new Error('should not call gc') + }, + // It should return 0 records, even without gc, as expired records are not returned + await pRetry(async () => { + r = await rServer.getRegistrations(testNamespace) + expect(r.registrations).to.have.lengthOf(0) + }) + ]) }) }) From 7a569c7c55b6eac5ccd926fffe30a162a43bafb4 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Thu, 24 Dec 2020 17:54:21 +0000 Subject: [PATCH 32/38] chore: review docs and binary --- .aegir.js | 13 ++++++----- Dockerfile | 14 ++++-------- LIBP2P.md | 20 ++++++++++++----- README.md | 21 +++++++++++++----- {mysql => mysql-test}/docker-compose.yml | 4 ---- mysql/.env | 6 ----- package.json | 11 +++++++++- sql | 28 ------------------------ src/index.js | 11 +++++++--- src/server/bin.js | 21 +++++++++++++----- src/server/datastores/mysql.js | 4 +--- src/server/index.js | 6 ----- 12 files changed, 75 insertions(+), 84 deletions(-) rename {mysql => mysql-test}/docker-compose.yml (72%) delete mode 100644 mysql/.env delete mode 100644 sql diff --git a/.aegir.js b/.aegir.js index dd3f99b..605568d 100644 --- a/.aegir.js +++ b/.aegir.js @@ -8,7 +8,6 @@ const WebSockets = require('libp2p-websockets') const Muxer = require('libp2p-mplex') const { NOISE: Crypto } = require('libp2p-noise') -const { isNode } = require('ipfs-utils/src/env') const delay = require('delay') const execa = require('execa') const pWaitFor = require('p-wait-for') @@ -44,8 +43,10 @@ const before = async () => { await libp2p.start() - // CI runs datastore service - if (isCI || !isNode) { + // TODO: if not running test suite in Node, can also stop here + // https://github.com/ipfs/aegir/issues/707 + // CI runs own datastore service + if (isCI) { return } @@ -64,14 +65,14 @@ const before = async () => { }, { interval: 5000 }) - // Some more time waiting + // Some more time waiting to guarantee the container is really ready await delay(12e3) } const after = async () => { await libp2p.stop() - if (isCI || !isNode) { + if (isCI) { return } @@ -80,7 +81,7 @@ const after = async () => { } module.exports = { - bundlesize: { maxSize: '100kB' }, + bundlesize: { maxSize: '80kB' }, hooks: { pre: before, post: after diff --git a/Dockerfile b/Dockerfile index 8624b78..6e35f80 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ -FROM node:lts-buster +FROM node:lts-alpine # Install deps -RUN apt-get update && apt-get install -y +RUN apk add --update git build-base python3 # Get dumb-init to allow quit running interactively RUN wget -O /usr/local/bin/dumb-init https://github.com/Yelp/dumb-init/releases/download/v1.2.0/dumb-init_1.2.0_amd64 && chmod +x /usr/local/bin/dumb-init @@ -21,14 +21,8 @@ RUN npm install --production COPY --chown=node:node ./src ./src COPY --chown=node:node ./README.md ./ -# rendezvous defaults to 15002 -EXPOSE 15002 - -# metrics defaults to 8003 -EXPOSE 8003 +ENV DEBUG libp2p* # Available overrides (defaults shown): -# --disableMetrics=false -# Server logging can be enabled via the DEBUG environment variable: -# DEBUG=libp2p:rendezvous:* +# Server logging can be enabled via the DEBUG environment variable CMD [ "/usr/local/bin/dumb-init", "node", "src/server/bin.js"] \ No newline at end of file diff --git a/LIBP2P.md b/LIBP2P.md index 48c7d5b..99e35ca 100644 --- a/LIBP2P.md +++ b/LIBP2P.md @@ -13,14 +13,15 @@ const Libp2p = require('libp2p') const node = await Libp2p.create({ rendezvous: { - enabled: true + enabled: true, + rendezvousPoints: ['/dnsaddr/rendezvous.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJP'] } }) ``` ## Libp2p Flow -When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to a rendezvous server. The rendezvous server can be added to the bootstrap nodes or manually dialed. When a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undialable nodes can register themselves as reachable through that relay. +When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to the given rendezvous servers. When a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undialable nodes can register themselves as reachable through that relay. When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. @@ -34,16 +35,17 @@ This API allows users to register new rendezvous namespaces, unregister from pre |------|------|-------------| | options | `object` | rendezvous parameters | | options.enabled | `boolean` | is rendezvous enabled | +| options.rendezvousPoints | `Multiaddr[]` | list of multiaddrs of running rendezvous servers | ### rendezvous.start -Register the rendezvous protocol topology into libp2p. +Start the rendezvous client in the libp2p node. `rendezvous.start()` ### rendezvous.stop -Unregister the rendezvous protocol and the streams with other peers will be closed. +Clear the rendezvous state and unregister from namespaces. `rendezvous.stop()` @@ -117,7 +119,7 @@ Discovers peers registered under a given namespace. | Type | Description | |------|-------------| -| `AsyncIterable<{ signedPeerRecord: Envelope, ns: string, ttl: number }>` | Async Iterable registrations | +| `AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>` | Async Iterable registrations | #### Example @@ -128,4 +130,10 @@ await rendezvous.register(namespace) for await (const reg of rendezvous.discover(namespace)) { console.log(reg.signedPeerRecord, reg.ns, reg.ttl) } -``` \ No newline at end of file +``` + +## Future Work + +- Libp2p can handle re-registers when properly configured +- Rendezvous client should be able to register namespaces given in configuration on startup + - Not supported at the moment, as we would need to deal with re-register over time \ No newline at end of file diff --git a/README.md b/README.md index b6b1c85..63e36d1 100644 --- a/README.md +++ b/README.md @@ -18,8 +18,10 @@ - [Overview](#overview) - [Usage](#usage) - [Install](#install) + - [Testing](#testing) - [CLI](#cli) - [Docker Setup](#docker-setup) +- [Garbage Collector](#garbage-collector) - [Contribute](#contribute) - [License](#license) @@ -37,7 +39,7 @@ See the [SPEC](https://github.com/libp2p/specs/tree/master/rendezvous) for more > npm install --global libp2p-rendezvous ``` -Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. Bear in mind that a MySQL database is required to run the rendezvous server. +Now you can use the cli command `libp2p-rendezvous-server` to spawn a libp2p rendezvous server. Bear in mind that a MySQL database is required to run the rendezvous server. You can also use this module as a library and implement your own datastore to use a different database. A datastore `interface` is provided in this repository. ### Testing @@ -67,7 +69,7 @@ Once a MySQL database is running, you can run the rendezvous server by providing libp2p-rendezvous-server --datastoreHost 'localhost' --datastoreUser 'root' --datastorePassword 'your-secret-pw' --datastoreDatabase 'libp2p_rendezvous_db' ``` -⚠️ For testing purposes you can skip using MySQL and use a memory datastore. This must not be used in production! For this you just need to provide the `--enableMemoryDatabase` option. +⚠️ For testing purposes you can skip using MySQL and use a memory datastore. **This must not be used in production!**. For this you just need to provide the `--enableMemoryDatabase` option. #### PeerId @@ -106,6 +108,8 @@ libp2p-rendezvous-server --disableMetrics ### Docker Setup +TODO: Finish docker setup + ```yml version: '3.1' services: @@ -124,11 +128,18 @@ volumes: mysql-db: ``` -## Library +### Library + +TODO: How to use this module as a library +- Datastores + +## Garbage Collector + +The rendezvous server has a built in garbage collector (GC) that removes persisted data over time, as it is expired. -TODO +The GC job has two different triggers. It will run over time according to the configurable `gcBootDelay` and `gcInterval` options, and it will run if it reaches a configurable `gcMaxRegistrations` threshold. -Datastores +Taking into account the GC performance, two other factors are considered before the GC interacts with the Datastore. If a configurable number of minimum registrations `gcMinRegistrations` are not stored, the GC job will not act in this GC cycle. Moreover, to avoid multiple attempts of GC when the max threshold is reached, but no records are yet expired, a minimum interval between each job can also be configured with `gcMinInterval`. ## Contribute diff --git a/mysql/docker-compose.yml b/mysql-test/docker-compose.yml similarity index 72% rename from mysql/docker-compose.yml rename to mysql-test/docker-compose.yml index 79df281..dba509a 100644 --- a/mysql/docker-compose.yml +++ b/mysql-test/docker-compose.yml @@ -11,10 +11,6 @@ services: MYSQL_USER: libp2p MYSQL_PASSWORD: my-secret-pw MYSQL_DATABASE: libp2p_rendezvous_db - # MYSQL_DATABASE: ${DATABASE} - # MYSQL_ROOT_PASSWORD: ${ROOT_PASSWORD} - # MYSQL_USER: ${USER} - # MYSQL_PASSWORD: ${PASSWORD} ports: - "3306:3306" volumes: diff --git a/mysql/.env b/mysql/.env deleted file mode 100644 index b2276f3..0000000 --- a/mysql/.env +++ /dev/null @@ -1,6 +0,0 @@ -# MySQL -DATABASE=libp2p_rendezvous_db -ROOT_USER=root -ROOT_PASSWORD=my-secret-pw -USER=libp2p -PASSWORD=dev \ No newline at end of file diff --git a/package.json b/package.json index 3d4347e..0897406 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,18 @@ { "name": "libp2p-rendezvous", "version": "0.0.0", - "description": "Javascript implementation of the rendezvous protocol for libp2p", + "description": "Javascript implementation of the rendezvous protocol server for libp2p", "leadMaintainer": "Vasco Santos ", "main": "src/index.js", + "types": "dist/src/index.d.ts", + "typesVersions": { + "*": { + "src/*": [ + "dist/src/*", + "dist/src/*/index" + ] + } + }, "bin": { "libp2p-rendezvous-server": "src/server/bin.js" }, diff --git a/sql b/sql deleted file mode 100644 index fed9284..0000000 --- a/sql +++ /dev/null @@ -1,28 +0,0 @@ -USE libp2p_rendezvous_db - -CREATE TABLE IF NOT EXISTS registration ( - id INT UNSIGNED NOT NULL AUTO_INCREMENT, - namespace varchar(255) NOT NULL, - peer_id varchar(255) NOT NULL, - expiration timestamp DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (id), - INDEX (namespace, peer_id) -); - -CREATE TABLE IF NOT EXISTS cookie ( - id varchar(21), - namespace varchar(255), - reg_id INT UNSIGNED, - peer_id varchar(255) NOT NULL, - created_at datetime DEFAULT CURRENT_TIMESTAMP, - PRIMARY KEY (id, namespace, reg_id), - INDEX (created_at) -); - -INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmW8rAgaaA6sRydK1k6vonShQME47aDxaFidbtMevWs73t'); - -SELECT * FROM registration; - -SELECT * FROM cookie; - -INSERT INTO registration (namespace, peer_id) VALUES ('test-ns', 'QmZqCdSzgpsmB3Qweb9s4fojAoqELWzqku21UVrqtVSKi4'); diff --git a/src/index.js b/src/index.js index a090cea..acd3cca 100644 --- a/src/index.js +++ b/src/index.js @@ -59,7 +59,7 @@ class Rendezvous { } /** - * Register the rendezvous protocol in the libp2p node. + * Start the rendezvous client in the libp2p node. * * @returns {void} */ @@ -73,7 +73,7 @@ class Rendezvous { } /** - * Clear the rendezvous state and remove listeners. + * Clear the rendezvous state and unregister from namespaces. * * @returns {void} */ @@ -85,6 +85,8 @@ class Rendezvous { this._isStarted = false this._cookies.clear() log('stopped') + + // TODO: should unregister from the namespaces registered } /** @@ -156,7 +158,7 @@ class Rendezvous { } // Return first ttl - // TODO: consider pAny + // TODO: consider pAny instead of Promise.all? const [returnTtl] = await Promise.all(registerTasks) return returnTtl @@ -225,6 +227,9 @@ class Rendezvous { * @returns {AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>} */ async * discover (ns, limit = MAX_DISCOVER_LIMIT) { + // TODO: consider opening the envelope in the dicover + // This would store the addresses in the AddressBook + // Are there available rendezvous servers? if (!this._rendezvousPoints || !this._rendezvousPoints.length) { throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) diff --git a/src/server/bin.js b/src/server/bin.js index 3bfcce7..2e72be5 100644 --- a/src/server/bin.js +++ b/src/server/bin.js @@ -2,7 +2,8 @@ 'use strict' -// Usage: $0 [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] +// Usage: $0 [--datastoreHost ] [--datastoreUser ] [datastorePassword ] [datastoreDatabase ] [--enableMemoryDatabase] +// [--peerId ] [--listenMultiaddrs ... ] [--announceMultiaddrs ... ] [--metricsPort ] [--disableMetrics] /* eslint-disable no-console */ @@ -23,9 +24,17 @@ const PeerId = require('peer-id') const RendezvousServer = require('./index') const Datastore = require('./datastores/mysql') +const DatastoreMemory = require('./datastores/memory') const { getAnnounceAddresses, getListenAddresses } = require('./utils') async function main () { + // Datastore + const memoryDatabase = (argv.enableMemoryDatabase || argv.emd || process.env.DISABLE_METRICS) + const host = argv.datastoreHost || argv.dh || process.env.DATASTORE_HOST || 'localhost' + const user = argv.datastoreUser || argv.du || process.env.DATASTORE_USER || 'root' + const password = argv.datastorePassword || argv.dp || process.env.DATASTORE_PASSWORD || 'test-secret-pw' + const database = argv.datastoreDatabase || argv.dd || process.env.DATASTORE_DATABASE || 'libp2p_rendezvous_db' + // Metrics let metricsServer const metrics = !(argv.disableMetrics || process.env.DISABLE_METRICS) @@ -46,11 +55,11 @@ async function main () { log('If you want to keep the same address for the server you should provide a peerId with --peerId ') } - const datastore = new Datastore({ - host: 'localhost', - user: 'root', - password: 'test-secret-pw', - database: 'libp2p_rendezvous_db' + const datastore = memoryDatabase ? new DatastoreMemory() : new Datastore({ + host, + user, + password, + database }) // Create Rendezvous server diff --git a/src/server/datastores/mysql.js b/src/server/datastores/mysql.js index 0dd08ea..49ad0ca 100644 --- a/src/server/datastores/mysql.js +++ b/src/server/datastores/mysql.js @@ -329,10 +329,8 @@ class Mysql { namespace varchar(255), reg_id INT UNSIGNED, peer_id varchar(255) NOT NULL, - created_at datetime DEFAULT CURRENT_TIMESTAMP, PRIMARY KEY (id, namespace, reg_id), - FOREIGN KEY (reg_id) REFERENCES registration(id) ON DELETE CASCADE, - INDEX (created_at) + FOREIGN KEY (reg_id) REFERENCES registration(id) ON DELETE CASCADE ); `, (err) => { if (err) { diff --git a/src/server/index.js b/src/server/index.js index 2d4b46a..844723f 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -31,17 +31,11 @@ const { fallbackNullish } = require('./utils') /** * @typedef {import('./datastores/interface').Datastore} Datastore * @typedef {import('./datastores/interface').Registration} Registration - * - * @typedef {Object} NamespaceRegistration - * @property {string} id random generated id to map cookies - * @property {number} expiration */ /** * @typedef {Object} RendezvousServerOptions * @property {Datastore} datastore - * @property {number} [gcDelay = 3e5] garbage collector delay (default: 5 minutes) - * @property {number} [gcInterval = 7.2e6] garbage collector interval (default: 2 hours) * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration * @property {number} [maxTtl = MAX_TTL] maxium acceptable ttl to store a registration * @property {number} [maxNsLength = MAX_NS_LENGTH] maxium acceptable namespace length From c297156706bf515151d1cc7915c2f6ab4b804db9 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 28 Dec 2020 16:24:35 +0000 Subject: [PATCH 33/38] chore: add datastore docs and model picture --- img/db-model.png | Bin 0 -> 26920 bytes {mysql-test => mysql-local}/docker-compose.yml | 0 src/server/datastores/README.md | 13 +++++++++++++ src/server/datastores/mysql.js | 6 ++---- 4 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 img/db-model.png rename {mysql-test => mysql-local}/docker-compose.yml (100%) create mode 100644 src/server/datastores/README.md diff --git a/img/db-model.png b/img/db-model.png new file mode 100644 index 0000000000000000000000000000000000000000..4f1fb87e05752329cc527f134bd53b114d178f0b GIT binary patch literal 26920 zcmeEubx@Sy+pZu;sj%bh+fD#5JDbn32rGOwJB}hs) zoM-*jcjlb=&dmAyn_-;so%enEx$pbB?(5#@M-LTm5mFOgyLRoCvJyh;+BIzSwQE?q z_&30l#lRu#Yu6aADI;XGy-hcA@Vv=&W+LavweXwq@SB;Gw(&!QLmAX@ilSdy;F5Pm z2LG8Mk(VMug_6r4$SLF-*yLq?Hy@3tV#5nBCifP))Q5*y2Dd&xD<^8V8hCK@Kyb@L??p{PefmfNKy~QQ3}S{kPq&#dw*(Uugsu|{|H)*!^HFVPSeQ+cdYX1bfc4l z&q!YKHfWgevSO^kPeO2A^&b6_P-+_l?>Cpq8r;QP`MzLGli@f@ZGxbCdn@q}OzYxV z;Jz1>BxJIN7=Z@89Y5}sV_JDy0={*j@cEIBBc1_X3FAmGV8xgYA~0ggmpkO76=I;7 zS)*%%8rX1i5zuGj*ZYI2*ap&I61XHj1eiYmFkqASaF&;LoXB=5B0d{8WOeI(K6Y!7 zl+@Jog$CI-uc7Gi&TQ)3pwn+=Q5l2Y%xRNl=A4q@8eOyh)j^`o_d z)9>ubi42yuWmV5(`Q`WeJG~~8@#N{#o|#2ha<+uV&A|2#Ynt0=iwyIfqoc^)H7x91GwbLu&V1JA3A9CzWQeK*0-Y}j_7f2T)>{eiNFQfRPmowzP7Cu&%! z<$F7+F57#B8}tq)kLjM6X~>3N7gRZ3Je}P=oD8xn)8o+5|3TxQury4@nKbFrmv((# zt4_-`v)R8y?b1H1G~-Xj@!83MN#`3?C*vkG{nK>8VrrK9XE*Fb4qi~qSpBN{RAuMy zLco=1^xo%kz~i@2V6K{PRdYB+KZjJ=9Mh3-F{jj3y#j{A_Hk4pRDAG zYuF2MFL_Dd*9GA?%OiQ)Vy>nv{(Dhscl{Gyh|@Dl^nMz5l$w5(yZ_{|YUp)0C1d}l z4GCA%+|!?|_3lC{E#57$&xt?rIC{~guzc-+^R}6J(o1k?1bkE~c#bk}nOeX7&Y(Qw zW#d`b2?xLPq08=YOUjQu0&`_f;Ft_MdQ!k)Q);s$m@#!c4eUuQ+U+cDc=&h+E*N+>-dR>)ikB z?4i9ems$+f5SLzo1$<4RlzfP+baHj?SKaRXW70}3rWM{^r5@o)VrN$UXAbmsB>Jt z6mw4Tqt@0&uJ8TwPZdSnHcF7j;d`jtBp{9b?6~M2dOXrghaDlf9aY;{>v3{=+LteN zG>~h#Lb^P*nylbX+JXOy=7C4FlOzeZ%Pl942Vrh~U~J0cYehw7Glsr7l*0jxpIkU{ z3ROi?ox9q^<)7ba)PYWJ+`p%B<&tEsX1&6q zqfH{#B5HJbI%i4ECBN^s6B(dYD(xH{ADx;&b;+#B_d8*@W=^r+l=(I9bV-ZAy(hr* z4X%=0=HfUh|2!L&5L#Wl72Pq_Xq~JKi>zUOZ;aS^ebu0RL+ZVpgwJA63;gF-Y@H_a6Q<)sgXSI#XNq%d6DLK?C$K6Wy zS7T~EzuUzM{r1+P*+veUohSKx);jZ=UTs$2d@WWA%3SVe!nX}Vj?zFm}dQm;WtGF1gl^FlJegxhG`e^oo@OVzIn8PdC} zdBwax9=9f&XVMm)rm8BH?TYkb%YJ6CweCAN%5J9s8_Nu#qjmzZryj2*3bb?t&;iiLYd`c>UIRx$&j^? z)k>0pau09(+N_qSNLlfUioRVogeA8{nNxZextWr4__oU z_YU`ml{q+M()%3wQ8=)ik-p@euG?Zqm->K*x`?0w4_?+6!u*sZmlO+oM zI*p{ZREJS}J84>oANzZaFMI#|Hmg|t3*|k!J0@YRN8G?CZx?IWs-}D$HnZL9+*58d z0XA&5h~@6?d<-6CgJn?mG0JmYcEs)Y#9k0GJINIpP!3UKA+5L$W!t;bZ+4_v=qFR}TSzq{Ie;DP1hL>(tz)jOpBZd49@M+CkzaS(mB?EZC6 zqr6(>waAyyu}e=uxkj$ z{l3gl6?buOl2ka?>N@zpNU4=%#rF!W4CeRD(#ZYk;#zH9$sjJ|qOvdpXs=V{|M+;X zW`#%h%^%3<6a7pj1~FjBt5eFm9sQ=S4_ff~P!oK_1!FbR4x6#+n9XgarO`{{3oz=h ztWIBU(KIXiY2}m^eBf4@ghfVRMFq~SXOhQ4!`5M&TRxk3D$e&@a5Xne`ikzj*R8y- zjGcT(1ZmKe-4#4arE9p{yuR2n!DR}%Z3OFG9a~B^=~0K5MkYW14*v5HOxM{=p8M0w zuPZLGS=LLbQHCm)-(^T$HzS=Spxit`m1?j8XMADKq1Ik6)7PgSHo5{ACKk>d(MzEm&!22qT zy>|DYJiJDi17~&=FZbC>@P2(s+E(GgkYcRWr}%<}DdyQ4Uwa?FM@G$u5vcWKDZPqc zizO6B)?;4lO9@9eejmOSOUO+fli#(M)OC`C4{MmdD7xvxw+-;Gd#7()^gK6*vp&~Q zayn^W(&Ded>}VY6wiw=)2NiKOWkxbZw`Y6QizE6o0?U^DyM!_;t*X&}A8S?UJV!$x zefEEqjfo3y=e58e2VIKI&J7zk7)0<1OPZbs1dz!;J9!;;>Y|8zA=y-diA&@Unq4Lb zF1)_890|UCE9NC}`a?GJ!oo|!_TU3k)p8p)kjjkI<2R@|4u=r{5Xv4fjmhyWs$G&=wNleVM#znvRNQ17mu4P~` zVg0c1?Cj{*dmW^^8q*u@ROX;<4Mucq&ry;;qYQgYM4wGFt7PhX(+0ERa0-=k2{?I& zsP(9VFg9sl;Ko!2fnhF9ifs4_R0W+FM!aK^{#ngn!@8XB)%bBwH4Uec7 zEl_rvqufL)I=ga6#|@2#6Gnl@+IaSc&CeC!ww$vswze7FNfeakMg`JwXXjQY6lzE( z-O;?C1L~u~DL!*#ALb=16HO?a2e*1k$f;E>J1#mP7uFYWTc#p?IWMXjUnXrABl*WZIF_=}dbYpQd? zwc6NvPabf3+>3PRa-AHl<_|n{;a~6GI&CZpqEDRq=>2=7BKdN5DRY<&on{!@{4k%l z0U7Z4{}^5eibmc4+&L%0sjxb|eBk=x$~*YN3W(22$R48iXC^xN_fyRFgT~JSK$K;r zohP=ubb?=hlxnc+l-@J;e)Cf_kBLGM^ITluBi0Ylb>6^pHAQ>v+NGeAkN)bX%?D*M z4!d*n)Z1M89Mx%0D!XD>zr|ua{(5%DYGi5$rw_)DOzM^ug&I&OeCISPXLzS0sSO;- z%h}#|0xi8VcL;linEvHAuY28E!gnrys38**W(Fz7{qKtTG55cd-;J=7%y;gxqY`E5 z;!Tpa0s9~z*Wcay2x@ZkMP0QJXuhmON`5{EqL2*7a$&(-u<0VO7)89QrhtB0t<`aS zXHKr5I^~^w>+dEev!k8$*@u!Rfm`=|^40JjG-C9>m}rg;C_O!orlw|SI=xx2v;MoG zWP9w&%F0krS`rfTQ)2h!7VaIPkq!dHh$qWRUxzc z>l24giLOw99|jzo24F{wmXxGNJj3u|1m52~iYbudPk#oh<%$~^*4Q!9in=T5KzBMj zqlj(wKKPiB_Gw`UT^ytvxA+?vesIck{vs|aYWV9@HY+0|<5;bu-nh?8-YkV*Us(Yp zZ}2s^&kB|!Cpg{YjUl9`%&2Z)_TG5f@$xl7ihD}44Ib9>9T9HJ z1D}fIRGzajmKoOar&19Xup65O@kWfUmt^49C;?IdV(gh~BM(jSQWkNVdi3GR42ygW zd6IzbXFJaw7X@f_^)K(uQuxm)<~X~>BqqiE-OJ0%(49C;2Pt5w`cL*&y77#1Q@J?Q zvxS9HZ~fnU_*Ke#n@(JRAf1N?%rgy_h_Ww+f}>D1otwO8qg2pTY>=Rk1zn_?o+h^S zQEp~GnvAvZVU8%Zz58p`)8n1RQ0FHu>VbiQqBjJ2c^~aA52kOXpu}^SB1Xp@6HkUg zjMB3KTkrD7q3FpHa#WdX39$bD(Usq6inrc%F|vgXigeG{@-F_{i+YyeD32WK{?@l@ z#gT~;?RN{$d42uDA*1@GFR!upWIPflA|z+~xqG_*oJI>OQS>@PLyrt_D_PzwTPvl! zA6({4dtGl)3o5awXTfcUb14uuZ0OA3VY%;-(heO{r3M0sp@m~fMD+!ZZvu#*vZ3t*wY%Ik5kcx5Pi zg5i6|^CTCkr1FOit|xpdUU@$mW`XrY5txWbozHk_G`fVHY0*ZjGm7nu4RnYHn-Lkt{o<#pkv4-HGms^ygl>dI)%q zu|R3<|7nmDO~q|I;=9Z*b|vH@8Pt4O@HzB>GUyiGwsAj)xOC&?fMfDCCUif6Ia)yL zm2AY6I26N@hqwd(H(L?%Z=Ykng3lDeXI=ekZ_-g}jWZ(!L_80%hvLUi+6HbHH?a`# zcJ$#}_T*u649^`Od(W!lnMrP({wslx44gyf<5GjE-4whK!&pXJWEc}AS z9s9Zcs$hXeh6! zxL*0SL~k;u{3eo9Ge7)=WPqUvXIt9aDQXIW6$b3+?vqL^TMx!!DQLl(hTU1;omH)1rR!iQr9T+zSNSTY&AMI-$D$;=1kB$)itP^N&d z&pGgg7YW&OL02a0cl^Fjmgs4;TwVG_qfm*So6`iaQGHG+1_tT6Z(&diWe%rxF_#}r zi2Edc!cV4G)7~geH4~1Zz5x4ua<<*7!;*4U?y)`*x>LiUS59tbW@ch`04&nIH_d)W z6M2@KI!r=OrnTh-s-rtH479k#<2zPmrD(r4XknkR7ZQHDlsu-TRi4|;M*}Y&Jaa#H7|xYAP{lcHYTf)Wv$Q%|5ZWiWRPp%D z!{`qE5?!u1IPheqxYe=|gkv9FEl>7_rF0JeSmTV!;WOojeaaR|2c~kcKZRB2QKg7@ zc&*d7Drz4Df{ftkYB51-pZfl5>%lbYTEP@Hw2fW;s{TJIX$FgRgR&R<$-7ME%O~ql zB;v^IznIibi=lKL`iM~ljqLhoD}>V!n718G zy|{;OC~*ZF3K^3>3NMk86+29pm!+7eaT>tD>#DNfHKB|-srPjFO_<0P#`~(SaDc_y z7^c{k#{QuUGE=|>@pnwM1o+Xm3fqk^WD43RGc1Is-)}5z-aAP$d^b^M^u}a3QHHZF zJo?e9uR;xdy1L-raw&zpzH(euH**2| zq=aDS0Pp5*;b>5t+E&)Q=qfEDouU+aSyeS1B$Qm#v&D8{3u@sN{OZClPos` z(nWtWwk<;*A;FFou?LsQWw8mHU@9J@3_4azd0!vCibkWqEI%HhjWZ}#gI~i=6hXmR zx~*7{oUmz91j%U65Pax)&xT!fiXnf4aATnW%$+E37=Mu~__NV;8m5^H+77!JVsfIb zS`2>Guf;%ea+`j~zjp)R^$YK6a{~~TWJK^*uW6C@l*0Pgy{WO>ix`lj271hdOqih* zYtU>j_Wy0b@1G4Y<5cu<_H3Bjlm53av7x!!UwD&of6qx4$!jft64sQxf=QboO$>a@ zktHv|Wl*lP=yuOjF5L{T&8r9`BNax>)!3(O2}D!;s7xc9u*J zD|3uQ%KLJje|v8xb@9N)IN;)7(rsgkZ-4E7uHl2G~iuI|nSLPYR*M;mx zr0Xg>rmZq=c`px8^k)jZ`Jj9r*SF}>dcm?XoHtqf>FI6C+U_)cJ4bzds!d9wFImcH zUVh-d3NkfrCe(6(sGeq`i2@n^{L_mEkD1@g$5P%y=?~1;ujD=^$Of*)*)X1JyVRgs z`?w-+85kxHvwjdgt`XgBBsFi>C_M=GD6GJ{2me!6_hU>bzk=~}Yqs$K_hNwKec$i# zH`dcN_G7IfUeRqe~GWNJS@+4Im`0HN61jYT2?O75O%^F;ZYJC?J&HBzOj zcZo z$%Yi1G9{$oUi5sWCjj5>ompNA(8|GU`LX(U$$7SS7JD6WZY0qntVrt5HmZjU4Qm~! zmqU0Df2;FNhWtwlFs^t-A3JP8^GdTo;oaT9{ZU0~`EY&N`<0GZ`SUMIP;8$Ed0I;+ zgA3=2;-yw2)pAKv5rTW**_S7=PoMNL!!l7f=OZr1RC#{eC#s!;z$Ii_CtAsB_gq?V zXCdg|t6s`?!V{JCORpb3Z)hf7df5_q5LCW5sD6E0PI}gIhW(u8APOZ+I(h_deqIt3RE@)~h40sD)Ixq<}kren-H@PMciz>F+V-IB}%0 zjp0$06@PthFyy4O9CuYxL4>a%L(>cZvN4`JP}Nuz(xKz{lgONszJA7S32E6G8Iy(B z_(e{8IVfKmg&B$@0&ygmpPBfOu;7`;urd$(iGk0hs2-7cHja4MU!>nZaGi+fdrdeb z|3g@t9$$$Bn?fox1f7p1pgMtEJp_C8w}M+6$*v zi){O#xSHI9ph9!Vg_HEo2hs^PfXa6a5Wn5m3Hg6j}{7pQty}6gtw$`q&!Y^WAnkSo=z@ zK*W^*OhwIzlXr~z-vEE14Bt|ryG>}PLm{2cw8p#M@<{22cjR)_umYvyZdtUyi#bNM1UI_jW zac2Z|?X}QD5>3S#n!72P;j2tN+|--I3%{o*1)jw%^*+7H)XOXSA$r66($GF#JPPM~ z95@}b4hTdquCFhDxa(41Qm!fauerg0dRv~W#A@&VWVe1WpjTtM!tvBp%wbqwfW|2V z0rl(&*+5f0hLCUk5-um%BmzXZwsUaeB*D8y%-`!JxYC8|Ml7J|Ig37NwF-|4NGN=8r<#Et#dk7`M+eg zLL&I~O13Is7>vPU~gq)-0_ zBp|MF{?TMm@r%=wql0cu%yT1=+oYkdzbsQfD$-fehO!x*QT1UX?vcVNp+h`I4c1qI`;!Y&!zjE=8N++_G=o<;ovfJdHX$ zDwfx)tOpC7XB#l&&aA_;&FR`KJg2f~%Zg4aPaEImRK5MCn%#)?MPsM04FCboc2v8} zwcKO-z1;b2IA7$&6$tIuZ);<$2G`S3$_g%L5H!1I8}s(sP)_w>)7&+uG5)fLL0Xo# zHUcAV5sP|lm1?1UJ6}`W|KNMDruACw&g?hB)=p=`HyU|q;o&&!dgU(!9lj_*7Vd7u zM3~f7U@3gc5-I{XuF7>$VOITCWAo;0qaL-O-OI^Ar+i+UwC6+#1-uO;!Y#Rw;bg??%F^U9P8Y9Yj1+OL}neEdK@6fS+4*n_@PV3DQU!cY(9IcCX7 zK|%so8b~(_%yf{HJpOroMoRVCwX3Yg=Y&9p89_ibj>nwLH?m zB_vO5+-gt-1YCL_G4{9D>NF=iz4QmRZ;6$dv;g`zP$gJ{oopM{IhTnck?BGkHK}4= z&h5^IiQ_+-{nm#Q=%sSK=}S{?jt>1_=f4=$Hn2BIqR>>q=!AmF;9v~r9ZCO>DicrY zW?%EH*yaW3;*LzI()P-nGZ+V)?rZj+pYILp>@&nA-g1g2nD30l2V~_~^A|>@8mXd2<-xzPS_5+eh)=Pj$9VR7U-p6j{O7w7u$wmCOw;~d;v=f3yH)518IsD>5!K4!1e-P=Y zC8B_j;D)A?fB#dp_zwKvg0McD0Y#*s+@O4~7SseMtBw&1a&hP~8D6Qa1QqLcPYF(4 zc@IH$dAC!U`SV5d<~vpb(sm4BRG;;-I(Idg)exF@VcD$4PBV@TrV@^CX=PO?{gc5S zB`$l7PuCfQzNc4a{yPydjNuT{(7Z(`{*C-p3;`ry2AvajUwss(0uzm{Oj=Fr@nGn9 zw|sq(lTlm^2DWaz+_ukLTl;xDqx_^me%xCD1U3;Bcd6_a8yYoYp^ruCy*PO7<3jB9 z@IPRSPhHTl?l1Ni*wFLy^S*Z@X@08WHPpIlC?ab_8?ylGRyG4j$6K-OoJqloe`z3E zn&Q+I7vByP9~vqZX~$H!Gap5-J1?8gyA{Fx8x2E~^aNg<_KVE4~Zr0ND zP)jBpdO(n=rV8D$#`1`02xFsO)MY$FdFHETb+9dvk&&hRycPEq%!+2<6Mte`8A$5_ zf53rWmF+3Kp>(Gql~4HAA|YcmFjXHu=k^>6!G}yE@zFFQ4-4c;6x3uxu7Q>3-T)*L zi2@>ygre)C931U}m}JdYTD=hjXc^L2tarh@`9@?{3VO^v0MklecK zFC>RQENwrQM_Ymv87#TpqMI;P6@jyZC-9!n2BN7!xTu~*uHbdC#A^osiE+8)3bqk^ z0-tCWCuQa2a_F!_n2kteWXVbR&EW_xS7HQxoNyc)6`Ytq8lM?v2smYp}CE|kr6owAx$!H#*Iwn<4^5!}b96>l~>iB4vRLpVN(d1hOY@taPl;=km;?N1F`VFz<^p-|xQ;jDl<1Kwkm zg<`&+nDgdD_Y`#KQZ<(sN@KzZb?HQ4-+{Ejs-e|k-wH9TC}9DsRFKKRgZThJ9jrz^ zX&Z;3^}o^iJe16!wgb7QskZO;Rml}5b#G51)%s_~a{=_{Ys${~8?YUgiC0vZb4;{% zbi6wju~9@;>JJUbb)2eB>5)(Nd^wM&6?3Sgx6%3+vNvg>imkCI@*aB8>1&!%bp@ih zK2@IP=Pc};SRpz175CX+Z&>!<7@F!7HO<&c5NZXvht5bFK2?CMMm*XvV7|En1LiLP zm@nn}5OC(1gPXx)COP|o&?ptDOk5*0RElWqB!%0F+$SbxG$!_&kcC3ZdYq(d*f}4e zC46^8Un|DOrhKkBN+Cr;XB)lNW2w%PWY1q~V?0uVukRjZ zl91^ZpkAD;9$FDRA=>v zQX_377k`bW|9snAVqY?|>Y7lc^PLtaY4OTgiY=nbt~nB&QnN_M`P0OqfmDtXM3yqx zTX2kqSR;FOQ(Z^6&fr0hPU6eK`QVZJqH4iq_{*1^G|E9X*Z!fbya}2{oNpVjX{cpPRdcJAH zu0e6ae0~1>l%aTZ-Y)7sn9egfdyk~D>tOVf5Mb&CtAEU8zc-R~3Z&V=U6}3K|^1|-n@5ps{+Vs%9i;qw|3;8!T zuF=6~`q0&}a(zZAFC=EV@(JPQZ;T5SJM1tem?3TyE5e537C+^RKoSWH+E6v z7ES3pRgvNg%W+>9LE5VfB4bGr^dgGw)`uW|U|=JKKerRjqTB#xs7QK{$;`AHWkNI( zfOfWU=VQ`BH9vz$bmi8fps!ibQ)D8XoN7@T49M?X`D0TJ1FmT`jA+2jjv*;fviW}0 zqiRa55u^1nONHsr&w(N;rvx`w9XnSBbGr|$mP4%3$S#0a7iE`7c#!VO7z8c~!5W+~ zFW57vwz01h%9owert?#?@MnteRXxD~-b_IbbimnR>ZGZkdc-U6)-G^A@1!xQ9n5I` zE2R-M1aeRYKWMf0>ImZ#y+p|k{*bK|ItgF9B@9VwIVvA#CtOIjNH$s)=b4qo3tK?# zB%xID2Wb|mDp@&N3e7iQf8nf35!jwjb=0{?Jzy(EN_|Q?@c9r{#3o^@gk4UWYN0Yz zb09Qi0de@mn8nShmoj03$tZ3~w2T00z6g{k(pkYtIPV*`y(MiAU?iDiaLN;!l`z3& z91EWh6_$TJxL*1SaI{`mIu7SVymmf<-~k|$KeWGn%U19HTg&1(jhPa|^F(<#f_=Mb z?;-72qA?WmF7mm>YpI5!BobseI{%5~-KDj!CG2R*EGoeE*uDDphddjgvj$|h5k=3x zVCfJQOXbf92RK@+bC@v&G#W+Jkqi^Wnz)i7CW<^uT39{9Oqlzls6L$Pi*Krtq z=)RK(RX@K4P-_Nz0uN6HRvf;$;UW&i<0b*Ys3m_2LyBF3f>?z2>j@iyXIzWW_`gZ4 zLkp0&eBHdGPx?pu@1i~6Am(&emV+wB-_2ZRmJK9%pvJ`*gW7Raga3T@*ZUC`yb1Kp zbkV~MI@yzD+gctvdJsi$BLk-m0mSn@7(WHFb$fYKboOR{+w|iA=cn5|O{t9uf64&L zcL;qa9fD1#saM_y2XyeuAwO@}-_uV`um4YhLPa&N1(61tsk@NrbvDlNxwR@o$d|u< z!}?Fi@;~(zVnj^2LK!2vuViuGmzh;#6la`#bS_kXJGK9};>_F@i72utQREB>Psd8} zQ-oHkT+^j*TO(J4o0GH+K=iS2OVnQmSm^<*MPbHPX|ePDx+c1|8^%9EGbu)30N>xi~$Vm z4Okl&*iB;MN!$wo-}(EuzJKBYIiyBp=Z(|{01^gFKz{rD%Cud|@%oX3Nn=EVn99@# zu;w0K@QudH@6U#RdzL0hp1yUCj`qa=t;&JKV|FTvE;Ez5Jhn7xqnqrdEYDarEl{|X zDH$LdG5Tpn&(n(7s7RKYM#!O1PIUt0pzLRA9Z4)kUoAGbeuh0QXg0FEU3FwQr z3`l>);C#;mtcgACt42Z|N~i5;sUD)<3rBuOH~kiWpK&VwaLCvZAYmN~-&l0VG^C%t zR?l|d-UA??AdPRj!K1{J7PfHT<%hUIYhZJY!`FctUYP7jKWknePwP1huylz45f6f7 zs5(;6#4k5U)}aG>_bz9SnAZdf28+e$KQLL*#zL2gy1#{ID|*dmw;K*`eBiuJ00-H1 ztL&$6^JSt&Ms(tRcQKmLwl94cga%F>%hgfP6{}q2O-;t(9I-lkWT|O$14A(;2N8$8 z*no24@2~>`-z?s=dpatuN*HLF#$#5#m;%wnuuYps1juM*hFBnrsm3o%H}1LfZ?vdv ze)A42Cg^-8N&(78LkqhDg}ycVc(^@h)(f-PFaW%W8WB@Z9+ydjHh9Q`kx)>XddotJdLGtki=YRkzR7t%zZ8BS;$79g!TYEXP0ab7hz~I>iaTP!>33i~sK3_~%gV=-@ zwYq9o59AXbcHx;oVDUGr$}#uUli$0N8A~3XFl%0!0`hJ7Qy5POI@B`5Ys?!4N;(U|fbkUhw+2r9k z-@zhW0&B!R@wZ^Vr@$z$*4J>4cq^(QXAFKTlfj~4wL7>Li-APxf!ua_U zFl+H-Fl(j~t{MKE*IZk&+a{{*?y>3?CCZj?oERBH6DoSOBMqse2vnwLjZI`bT z8EL#Kx$YQd?PXZ+R+2}~XT1Dk22*7qn73upy(nCh@inL8Si1Vp&xp6&Qt%CJl zJ`c$IKTOv#T*<~&Lkg-6H6{!Xti1P1vG~2AfC*THa#M*pOD{Z5_w6gI{6Y)_$$M&| zHKC8IlvR&!(@U0iOuf+nnvQS8j0s*@(z=pCIiGLxNMk!Az{d)yNUQEOFfasC?WT>? zSCHEU@(u>?6K>Nc{er;N4x<{oBqu;%_I`TKZDQ!}@4r;uxHTr!RFT0sfzx?{NuoIG z#)p?hw47~;&011AMj(ahJz1oW_7bEn->S-(Z+~m@*&8@mpB#?^5lua?(;yXe^*K+f zwOrn2=(3Y8qxIr2I~a?yx|>th-3X=uz1adFi$opBCWv0UD}Yn^=3W}fF%X3;6{VZ1 zR4Y_|X#;lM!xqPwOg4Gqf?*mp6iRO+5DK|bXxYQ0JJ749uynnerZOwg;Y9jDg}+Rf!i;L8e}f_(VkGmB!}w_)OEBYxq)fejK``8M_MFyk&czJf8`G zWhHJm%3v-To-@j@{*i@ajDd@!L%*CYmP2sg0a7h%i!PXocWEt2k#4lvJPy?Iz$lZu zZ)2fE%S{?}nG#4?Iq%1^ADT3{k6tK^)}n>FpGv~b5U<;r=&VPvjClz;{_rU zvs-6vq1OjOYfBku<4@W?n2@okJ#r>_)+Y|tgr*QOAu2(R^xlG9=W%i{4^t#>K38ba z@<6JG0{ar=$#pY32K2j}g!N2-oCIo`uJk#E&9>ikkR}JOLj9}F`eswJ%e;I;nivW3 zqdUs6r*7kGN^LNh<1YdW;L%tN61Wbu7BuWmkDNj6Lqs2=3JTY@GPVt1VC^b~=Oki% zV|eub3_whz9o;!?CgHn3vK|09$Wt$yaL8nDEVUa^WyisrhdITQowo^;5+i{^@b~3g z)Lq%TK*Lko8upQKJ5e7+eP|{BZA_J<2OK-cZ;vR^>}s2VJ%JTBh!OQ|lt(uN@L~CJ zN__$!PGxJx`RL7Wgb2fOrv}MJ0kNos3_qVI!W$!&I)6#bR3Q`&F{5UPAwj?qNnV@N zMIP6XYMcIjO_>M`fp?YwYIuTZtHURNM=5h1oTvu6Ck7!P3@O@Db!c+0B)Zn{@D>sg zUJ4X$4a_8@IE2`CK(gW9e7q2QFOVb;sECT#wo^1D--S-qKeOGuROsNojv}>&qZ3)> z3P(Pnf`~yVfcq0<2b8xWs?1ly>wfrh$kUBT#J^ntNfE2_NI2 zbFR+juH0QF{7-h7O{sR{@|aptp^S_jF+0(#Rh~n5A2vfrEU56{PftK*nGJorHvCj! zM#f8mG(QTEc+@(=V8U)7aDQWFl{k_Mxuck^zJ3nU3f5DUcZq?zv}5@I4;23RN{~|~ z;)FvK>YQ+Z;YWQrIXhdYcb{i8ydK?tEG7pX?MCxiV6q8rG@&;5ixtSueQzLK8D;)x zJ4|;+)cx({U6WuE_>fb;)x}XQ5#&fFl=qb*9}@-^F5wa+Jwj|g4M41p$U7rY4A0Ve z%qF<45DfI*KSL0%(q4R4q>J~;2p4BuT65Mecpu(A@jKt?trfeY)d+a4P17EI4SeRF z8Ajxsu6t)#l%-@F{fnoZq_FHnRR!qPS>LORS7%eJvXLWlUG3u}JwRl-enmq>X=%*X zBOpsSE-KZGBzPC_6L32xP^;INszQbt{HX?2>e(C;d9+cNwiFTa{c(+hz&iv?Jr~G)qMjly z{R-*5Ta4jW$QKKi2@)(aGBW&*6P%kL=mhN&rME~n<@Dfih~-VjmPE0V4wUn+nk`#* zJwjB+j^6K5y2mZcAlM zLFFE%ppwxukxH#gdg)em0;4Gq0t3PevCd#kB8Y(PVE1t9^wETK`Gk+63V|w2qlkcD zJo>|S8ucugspKlx-gIr$;ISW2UJc5%jG*Uri01Hd* zB#{9Z6P$Z$%7zQfFyFEd<7F!Hl5%G(FuJoFDlY3VA{4_N^qp8ru~@4phEPQKnb#0r zesZE}-)&Mjp7l%K*W?N)5z=W+7NHQbo}f@i1S0%)4sZU}TcV-M-l@lO(3CtE&qu-$ zqkHt1oLHDT;0#7{MI@*4J&-?(z5Vzs4o$4bgKrNMdx) zz8m}s&-@mF8rPkJV64e_OO-fF0Z}O?Gl^{cFut6$`Z(v2!|M&7cw2H9Fn^-`3fIo;p-S`Y9=x#cIZ zq~J+F>ieqeR54(Nl*kUSjv8`72K*-;yy8TRd1}rBA}7DEBC4Qj`yYjlK9Ij<%~0?> z;oUu(=Q!V^z<2`ILLV?b~N2-mAZL^RN<~Y{RiK!V#wZu51R%6Pz*iuo9t63OB9I%X zdbaW~Q@}R9>GE_vQ!)>)A-)X&nB}GZ6s=}ok8iC(t))Qb>3g)5Q+DUR=SD_XZ?Ed_ zv7+h&NgJMrAm&8(reK!C7HD8?G%uLp@IH{gEKYewSb}+#tYad*6JRg-}o! z0Q##U;D7$v6VPQX=LKCR0nh#j#6~QM7Znv1PM_L~yo3~re9)fH+9FSD07*%6vXy_8~l}P|D^<&b)t=Xa;#Wg@LA5SmE4Rnd= zWULSS-aUR(Xk72M3k0J0rZ4IJ|AdI2|LMFJh{4SROgM`micAKj1pS~sVBhfA2V{YC za3lftkB*5M!zV2?-&yG9_x@uZcW<7WM%Za2XwGxy2_Utpo;(4qKL@6F0C>j(YE0z& z-L;oCcb0Tk{0O7}ZitSJO%eCm{aYIW)F)%Wp5;{P<10 zcDb>^RU8jKiN+Uu&|U2h|A2s6u2O z+V~q#5z<5U_u>{{?D%`*dYDVoPUM$TgU4oTK(3YqLZ`lG!xF3@|Cd4^cr23}cxoTe zzc2+dwf#WQU%P$`Vty_5<6qe>`|>|P)^;;F@bH`MfxJY(sXIpchf#%R3EIOjaFd`d zmVPoqTlyMArla}B?YnFsb*a9Qe7RXSCn)7Tn$BsU)pEMV@lK~CO@(7C%yIF&_3F|H z7iTgG%4d;K%Vvp+jKNx+c80Fl?VQpjaAlh>PjPZkQ*|?cL%9+2>ST%e_(4H-oMp>Ap) zkY)tvkTY-t+^$n8-xb4th7W#`Kr9{^&MP}clA{2Kva}e67EXrbs6If~n33}uV~r)? z@O26K9jq0>@ZN=qpPrpLHg;0s_Hx@RF~EtXqQV!5dFm^KZb?z zuqA3Es4)`eBN=UonMERr&@BQ2O-7-=nr%RsVt!)7AN&S|LX&vilbN)9O2!~>cDY?} z6IgVLwUdB(`Rxic3j2QhtFNwtK%u~9Ih~Nh^O$T1@&%t|B${5zG(pA+KN+F-~iVQ)QL3Tup&K*t*l$iVK%lh6f zW<(g{{wl18Dg_7;1~8gTe3{)Z0M|ZILh z`*OCRO3)Jib5H@mX+eJ9<3C6W!x|sN_~`2-43lSOIx;Z?CoH&QD4zD$^5g}VML;a6 zjyAqAfr@~ySqME^7$0hYVIXWL0!1RT3$)36mWjgWt&EANa~TYEmkNhqI)w2EHqA810@bqLG>|B$MF5e-Ml&a#9apnYR#C^K+d}Zg%XU^1F}TdDij?H zm4drbLE~KlGpmtYF^@GXLRiEoIX9i!U$3CiHqr4B0BPEkDIkN*`4hVJpvjnzj$~B2 z>F4MH`t42A#MMK2FQzW6j$bP)Z${JduW{ur=s0;5c!i-%6+nOC#_IH0ArQnRaB{9Dl3HgwYau zenW~gnNE9W7Wff|28UP1uiox~;}n!Y7YAF;cL%$;?cJGWk9jFc2`uk0CT4_RBS8GG z_O3Iksij>@Q$VE(0#ZUJl+dIjy>}@B3IqfMEFegiencq&5?TOhQU#==QX)Zm5fM;0 zAc8^yMruM!Ai&+6?Bh6V>fR ze40^$&^3H7t2kE`onRF5n<*+Z;$BG8nS?q1>T^Rq?YRI69oM;F?g02iQq}Zz3UrSF z8&)~pUB&?`*MHl>?@wf*03s6f+|>m>>hsF@pG6p(b;UW{xxpb@!xaYXhs>OPyZ>Xk z0fx?wmbr@$l_{U+1B3YJg>3*1>ns&ii+l!%BDAg4lttQ$R+O4;U)NsQgJlAj^T{qj zj44$&lUeS4;61ux3oHAa*!v1EZgB>szWJW>Ndy*5TP4pT(q4S=J7asL|Akp97J(^m zrPP5+q_o}$vw1IvbBl3{rvwmL6VEmF7ZMo*qL8mThCG$I9l-&ps<A17)|{ULXCyw*x*aAXfn7FxyIS3}k6A0YrgW++%{LSZ@r#OFJLJ-Rq3MX9K0@t;r~r zzwlK)0C2E_VY;U~&n1JgdfjY*hs02S0I=b2R+0&X`(OMkyU_n3cae)AtS+YS@hQgp z`-tCrGv;=ydgwLROf)25OwuH5c*Artt6F zoQ3^VMj&Uvjwmvng4t7s%^XV%MJ&8A9DKY`gm50H^E#;O?CfN@(N;|)J(ovoMI-j! z_L+`_Yida;9{r!{5TlU`?Gr;m>Ig+I+_{5XUiAMudDE-fVjYlE*PX# zV*sY4h)6t097A^}ENITixu|~e$Q-{KQCJK(5AM_c-H_0C0ZO)W58eh=3IW*|WewZ5A{TZ5=XbR7F47}F7zk&w?26#sNl%}3~9QCQ2%s3(i zJb%FFaJ24EFoTYuuTL6r+mFjawWGuQtc<~qVpiFHsqjjTYvMMA5aH39mEa^1%%k4R z1D?2LTGATMdg7%C;_7xX&pDaE=;v-Z>6 zlO7)Q5fy6DiXmTZM`0ND9@Pn`6W3m!i}2{S)@)-JBpe_4TbDcZ%P%*Vv;soK;=y~r zQHYiZtq@y3wOQ~~CMrXrjgtD@f${1cI}g3b(L1U#$U>hN^p|(Hu?|hrD zA>i(64k6@=?1PM>-)^L5LRdhrM`$X=hjY6&*`CcMu#qCsX&43;AFFo#ttOBvF7|v(T#jv1z_z|hq_7J;c8H&9mK{N{o-|ms zeVs5f(cH9r#^qZl%i3ir4Z0JlEGjn`>|#218+|e~wd^+bLm9G_NX=&BPIfhL=eVwj z>BHymB)Fa9Mm*`IyIPgvW_CiRUN7m>%ZK&v_6fSjHNtc2b@MZ3{~UTwI*WwW?{3~T z0;j$sZGWyW!rR$s>Le34OV)4HBgpDNY&ZfF^tk8!#$66%!6&g-FmJUC{CVu=rtj{n=GAk#|Bx7N8u=I~>K+X?m!}_5u zpGuF-#MQ;kRYAsYj#`B{TfL2_py<2|7ahtY%kEYMu-cGp#r{-oRxgFnRAa=!;Vcb})b-$&2 z#dEQk#Tum*liztsa4b4XN1Tlj{_HDz`sGe#$a4lK>}I(V2s#L}^y#5jT3u^-L$J^%slwy21mNlsG4P#75g)v_8%KSk3W z!ke7_iF_QewQ~^+l?3%k;`Vm8hnvjpZHtqzjPR{wC6E0^F`ztjp4Unz&-R_ll8UN` zcIWX?9O~rACoz$=tv0X$GlY{k*W)*sl0%K(W>tBtqUfJ}HPufw;%(u{$Cz|rjt%#? z)6yw2Qh%cFWsQI-jx|`K_ze`-&>{s_uo?c5Nunk!Pu8ad$97zjtpuCUbjgzE z_WOI*HTN4b#aYB^-sr!i7937k%?5LU_|(}e$P*@p(h1~++tDhQ!Gtd{4s=`SFd-rT zUtF4PmxQgQu4=~}=`$4VBbpHcA$I0)ij*ZG%L%REJ9*}dvj6P}1E$VF6mHA0RBq7* zBJJSltLOy<9MN@};(j0Jg`Y`d;EP$S2~bdbuKf5(t_k~Yn=!9Y{8FTJeKvjE{FuS(c!j{Mquq5 zaVo5q5lJTWZm@O)h;a@9-ME)&Daye%yd)jJy=;a18ve80R-E~z0Bm(bjZ=6(RRv^T z6F>NbLb%+O_e9T)8lO9c{IbN~!;03F$m#t9guAS3YWiA6PdL)!eY+`PO6iAOwY`5> zgJ+aYcj&LC!3NTCI}K)O)X(BctWjYK6UnWnp4`-9&wHOqno+f}NI8D|V5QLOHxrEk zQ{IX9-)tO_BOdVt?A$IU0%?UbR2?9XCXnjx<`Og`nX&QmA2xTd;0XtFHiXg5%bxE? zuw@3h`Nkh-0=DwhxAxqsuB*xbvecXVea+<=u>Bt5Kqo8XNUGqpr)|9zZdM2#vDM>4 zTCnuOrbG#s-I^>52;8#GyigFX!|>FwQUYdB0Ws(wj!#fdOiY>f&tUpof5SObMR3@7 zo$y}nQ6EiMB_xyyN`HwsW$`e85p*4Po%`8Fr&R4Errsm^Hfd z#I|K8Vzs!Xroga&U&76VV*$D$F%o<@93z6CXuA=SzAiAPmjH9bh^`1kVpd}tIj5p; zp0nbRDqTwa4T@Y-wSILcKyaDXO@~`F;2Sb}pRSQ{B^h$jHZ8q}%S5GaZx=3k378k8 z@0Y+_Ldk=}m(>Ob+^hn3FFxJKmeYS!#?+rkWjcIqmrxNy*n3-vO%`6k0O8QsAqIWS zkY(CXyE8&=yKN%i7OqT(i;Y#fr-B=~5Modv0YldJ4d>fk_8hGLBnc6o=MchKPJZ~!sfJ7ilgxN zb1!AsesoGjSh5#nY3{pBCWuXS>xCR;-Sf9D{V@8nrWDAD_TtrNFkgD;Li8RqBWjZ; z+e?iN5xY(=d`#}D5)1S0cJsL*g)|GrXd=uCe*kfY96iLHX{GqBwvNTJsHWV>Y2Sp1 z)?v#VYkLT2(F>Wp!(PIvXXo^4rJ+tg?~d#4B#_`q)-QUabz<-L_4kdeP97ut?4OOy z$&QwV=c1Z!e--9b-z`h<`#$RFDVN8P|0)7VIt1h4R>kI!U+-7nKWWXQ#u$mia%7$S z1|d9N$cReh$*R3h`_+%gxV6@{%9a{V);qlD%7zQM0Zum_bO^bk@`V~yHW+9^f=>klqQ*P}7Uj}-^Eh&%u zK7Q~$nT_Ikmg~fUjs*K;vUmS^a?~SNX2Z<+Ns-O%RiSCe>q!#;$8hhZ6mfT81bc1{ z7%v4Fi|XhbB%9s*AsW55kAN0d$jry3UGp>k4fdC5g^G64nfN5yj_Qp&yH?ZTTMHUv z9A85;CM*}p+a~Xw1pe5Rf*j4p4}Dyk#;$LU4OWyk#>M-RKgHevJ{Ukda{7&cNd9vt zbUHe5X}$frB9SM%Lni`=T(iIGIQ;)I@3m>>nQX- zG=q84!W&uNsNf0&58BuD4WFO17|iau-b6@AN?%!sM==-lDt7_<^6<3u+@*f(Edh@$ zL&HEP?CQJZ)mB|W?Nq*~lLl^OwgRx*1@uGRJamJ-XcH_*epuN%Lc1h7OveFdvvjL#BoVOZg-vWc@Yd z+`l-2;i6i1vxjr_mSvv$%}hziZKu)V@w=8yT|sx^nH{D!V3{i#nh@DWI>MjV;>&DG zQ2;cQ(0tG;nJ}nLIhnUJ@y3F8{cbH2mly|ff2iaUVc;X}H`efpQsE$Ew!4>~_uGAh zyzE4qzYFeLQ;db!Ov8S41nTFjHRW(;!RJ17f+B^m{3xBM1zvueLa^Xy^*7ij$njwk zC@-qi5B1qyXZNCS@;aaQ3!zw7aa&B6t9zSmT?+w&|eAO2P&nuGX%C_4S{U=_bcFW(YJjJuC%Y>T#y$w8OF^asW z^=dBvEF+)fuZZUT8BHOpVRl%S{f0DuN)2WjA71s~V(TH%u5JHhFUik!LrA`QlvXgH z*>zo*v^5|v{!B@ppM?b@y1t6dV!I_!X8VJiuosiw-tu9%mRZhYST?NMojLbpKv>0J zo7oGiO+)?IVi}cEe7b&?gn=QX@ zQq!`jei%bdLp1rE66(BHX^S}H*n~^EaI}_VpxSsJj!@)7K^W1|M$;8*(+6nTtaCl3 zWedH66J<9cVKpbC#?lK9_Ohy?i6;060qK_HEc$pqAlAv0j3L|c<<3jrpgg&l!;2Op z;5MV0mdeuf`p8PRUjQ)M_w^E{@E09egq%aV1ZYG)EjIo`|EVcz9vkakINHW!J|QZD z^W|^y-p#f=8zNS&_FTx4sh?pfBpGVuQVGP_^~tH;3xR&wJ>SR_{G|cpVhw&R=oa1kyKBS#MKbxG~dFHjEJ&wetwpoG5&GpU|s=AB_kVNJc>i-B)BZ28a@S?jzq zlID4(l~Kw5<(KTw#!B1hp3>&K3Gr-x=4jahXl7xm+N`70Yo&5iY>q-=y(o{Aq-o`z z=d-SvCs$Q*-go=Q?^;)bkqinVuhp^{F!5I@-TA}wlh07nf?{IM$Hq~*e||!1SY>Pd zZ9$N}lQd`~1FdN9){RH)a(Q@0h5*ZhC1t(5*VjYXfp$L1uYa|Dnqo zj)%M~o_VC|p0boumlJN`>y)vw=kjI=4^Y`~a)3ckspk5u(SUIkgsX{fE08B;wwBW8 z`FSReFdZhXGZfTJV(D57rN+6cPknC%yPD19rHEWTYdzwKl!MZXve|+zAy{eaXbiS# zHn&F4cAy`DmA3hfZu@+(Fn<2FIX;Tk+9aF0-eLDK0MZ&}jJN1MDnJbo5f#dTRq zUZ$?-jidRy-PfVZ_I_Tr?<$piXI2*3m0NmgXko&~XFf$z1HFh{q|2F&tZ3c&L$qkU zu { this.conn.query( - `INSERT INTO ?? (id, namespace, reg_id, peer_id) VALUES ${results.map((entry) => - `(${this.conn.escape(cookie)}, ${this.conn.escape(entry.namespace)}, ${this.conn.escape(entry.id)}, ${this.conn.escape(entry.peer_id)})` + `INSERT INTO ?? (id, namespace, reg_id) VALUES ${results.map((entry) => + `(${this.conn.escape(cookie)}, ${this.conn.escape(entry.namespace)}, ${this.conn.escape(entry.id)})` )}`, ['cookie'] , (err) => { if (err) { @@ -311,7 +311,6 @@ class Mysql { * @returns {Promise} */ _initDB () { - // TODO: Do I need created at cookie? return new Promise((resolve, reject) => { this.conn.query(` CREATE TABLE IF NOT EXISTS registration ( @@ -328,7 +327,6 @@ class Mysql { id varchar(21), namespace varchar(255), reg_id INT UNSIGNED, - peer_id varchar(255) NOT NULL, PRIMARY KEY (id, namespace, reg_id), FOREIGN KEY (reg_id) REFERENCES registration(id) ON DELETE CASCADE ); From e1cd224164ae1aaf1271d6f496ad72ea065c6f14 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 28 Dec 2020 18:02:27 +0000 Subject: [PATCH 34/38] chore: add library docs --- README.md | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 63e36d1..d3825d7 100644 --- a/README.md +++ b/README.md @@ -130,8 +130,36 @@ volumes: ### Library -TODO: How to use this module as a library -- Datastores +The rendezvous server can be used as a library, in order to spawn your custom server. This is useful if you want to customize libp2p's transports or use a different database as a datastore. + +```js +const RendezvousServer = require('libp2p-rendezvous') + +const server = new RendezvousServer({ + libp2pOptions, + rendezvousServerOptions +}) +``` + +`libp2pOptions` contains the libp2p [node options](https://github.com/libp2p/js-libp2p/blob/master/doc/API.md#create) to create a libp2p node. + +#### rendezvousServerOptions + +The `rendezvousServerOptions` customizes the rendezvous server. Only the `datastore` is required. + +| Name | Type | Description | +|------|------|-------------| +| datastore | `object` | [datastore implementation](./src/server/datastores/README.md) | +| [minTtl] | `number` | minimum acceptable ttl to store a registration | +| [maxTtl] | `number` | maximum acceptable ttl to store a registration | +| [maxNsLength] | `number` | maximum acceptable namespace length | +| [maxDiscoveryLimit] | `number` | maximum acceptable discover limit | +| [maxPeerRegistrations] | `number` | maximum acceptable registrations per peer | +| [gcBootDelay] | `number` | delay before starting garbage collector job | +| [gcMinInterval] | `number` | minimum interval between each garbage collector job, in case maximum threshold reached | +| [gcInterval] | `number` | interval between each garbage collector job | +| [gcMinRegistrations] | `number` | minimum number of registration for triggering garbage collector | +| [gcMaxRegistrations] | `number` | maximum number of registration for triggering garbage collector | ## Garbage Collector From 264ce2abfeb5fff917a3b2975671937fd51a024c Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 28 Dec 2020 20:27:12 +0000 Subject: [PATCH 35/38] chore: add docker setup docks --- README.md | 65 ++++++++++++++++++++++++++++++---- mysql-local/docker-compose.yml | 34 ++++++++++++++---- package.json | 2 +- src/server/bin.js | 4 +-- src/server/datastores/mysql.js | 10 ++++-- src/server/index.js | 8 ++--- 6 files changed, 100 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index d3825d7..65d4070 100644 --- a/README.md +++ b/README.md @@ -108,22 +108,75 @@ libp2p-rendezvous-server --disableMetrics ### Docker Setup -TODO: Finish docker setup +When running the rendezvous server in Docker, you can configure the same parameters via environment variables, as follows: + +```sh +PEER_ID='/etc/opt/rendezvous/id.json' +LISTEN_MULTIADDRS='/ip4/127.0.0.1/tcp/15002/ws,/ip4/127.0.0.1/tcp/8001' +ANNOUNCE_MULTIADDRS='/dns4/test.io/tcp/443/wss,/dns6/test.io/tcp/443/wss' +DATASTORE_HOST='localhost' +DATASTORE_USER='root' +DATASTORE_PASSWORD='your-secret-pw' +DATASTORE_DATABASE='libp2p_rendezvous_db' +``` + +Please note that you should expose the listening ports with the docker run command. The default ports used are `8003` for the metrics, `8000` for the tcp listener and `150003` for the websockets listener. + +Example: + +```sh +peer-id --type=ed25519 > id.json +docker build . -t libp2p-rendezvous +docker run -p 8003:8003 -p 15002:15002 -p 8000:8000 \ +-e LISTEN_MULTIADDRS='/ip4/127.0.0.1/tcp/8000,/ip4/127.0.0.1/tcp/15002/ws' \ +-e ANNOUNCE_MULTIADDRS='/dns4/localhost/tcp/8000,/dns4/localhost/tcp/15002/ws' \ +-e DATASTORE_USER='root' \ +-e DATASTORE_PASSWORD='your-secret-pw' \ +-e DATASTORE_DATABASE='libp2p_rendezvous_db' \ +-e PEER_ID='/etc/opt/rendezvous/id.json' \ +-v $PWD/id.json:/etc/opt/rendezvous/id.json \ +-d libp2p-rendezvous +``` + +### Docker compose setup with mysql + +Here follows an example on how you can setup a rendezvous server with a mysql database. ```yml -version: '3.1' +version: '3.2' services: db: - image: mysql + image: mysql:8 volumes: - - mysql-db:/var/lib/mysql + - mysql-db:/var/lib/mysql command: --default-authentication-plugin=mysql_native_password restart: always environment: - MYSQL_ROOT_PASSWORD: your-secret-pw - MYSQL_DATABASE: libp2p_rendezvous_db + - MYSQL_ROOT_PASSWORD=my-secret-pw + - MYSQL_DATABASE=libp2p_rendezvous_db ports: - "3306:3306" + healthcheck: + test: ["CMD-SHELL", 'mysqladmin ping'] + interval: 10s + timeout: 2s + retries: 10 + server: + image: libp2p/js-libp2p-rendezvous + volumes: + - ./id.json:/etc/opt/rendezvous/id.json + ports: + - "8000:8000" + - "8003:8003" + - "15003:15003" + restart: always + environment: + - DATASTORE_PASSWORD=my-secret-pw + - DATASTORE_DATABASE=libp2p_rendezvous_db + - DATASTORE_HOST=db + depends_on: + db: + condition: service_healthy volumes: mysql-db: ``` diff --git a/mysql-local/docker-compose.yml b/mysql-local/docker-compose.yml index dba509a..e98c262 100644 --- a/mysql-local/docker-compose.yml +++ b/mysql-local/docker-compose.yml @@ -1,17 +1,37 @@ -version: '3.1' +version: '3.2' services: db: - image: mysql + image: mysql:8 volumes: - mysql-db:/var/lib/mysql command: --default-authentication-plugin=mysql_native_password restart: always environment: - MYSQL_ROOT_PASSWORD: my-secret-pw - MYSQL_USER: libp2p - MYSQL_PASSWORD: my-secret-pw - MYSQL_DATABASE: libp2p_rendezvous_db + - MYSQL_ROOT_PASSWORD=my-secret-pw + - MYSQL_DATABASE=libp2p_rendezvous_db ports: - "3306:3306" + healthcheck: + test: ["CMD-SHELL", 'mysqladmin ping'] + interval: 10s + timeout: 2s + retries: 10 + server: + image: libp2p-rendezvous + volumes: + - ./id.json:/etc/opt/rendezvous/id.json + ports: + - "8000:8000" + - "8003:8003" + - "15003:15003" + restart: always + environment: + - DATASTORE_PASSWORD=my-secret-pw + - DATASTORE_DATABASE=libp2p_rendezvous_db + - DATASTORE_HOST=db + - PEER_ID=/etc/opt/rendezvous/id.json + depends_on: + db: + condition: service_healthy volumes: - mysql-db: + mysql-db: \ No newline at end of file diff --git a/package.json b/package.json index 0897406..38b9bb5 100644 --- a/package.json +++ b/package.json @@ -69,6 +69,7 @@ "minimist": "^1.2.5", "multiaddr": "^8.0.0", "mysql": "^2.18.1", + "p-retry": "^4.2.0", "peer-id": "^0.14.1", "protons": "^2.0.0", "set-delayed-interval": "^1.0.0", @@ -85,7 +86,6 @@ "ipfs-utils": "^5.0.1", "is-ci": "^2.0.0", "p-defer": "^3.0.0", - "p-retry": "^4.2.0", "p-times": "^3.0.0", "p-wait-for": "^3.1.0", "sinon": "^9.0.3" diff --git a/src/server/bin.js b/src/server/bin.js index 2e72be5..d18432b 100644 --- a/src/server/bin.js +++ b/src/server/bin.js @@ -46,8 +46,8 @@ async function main () { // PeerId let peerId - if (argv.peerId) { - const peerData = fs.readFileSync(argv.peerId) + if (argv.peerId || process.env.PEER_ID) { + const peerData = fs.readFileSync(argv.peerId || process.env.PEER_ID) peerId = await PeerId.createFromJSON(JSON.parse(peerData.toString())) } else { peerId = await PeerId.create() diff --git a/src/server/datastores/mysql.js b/src/server/datastores/mysql.js index f4c5f20..ea80c6d 100644 --- a/src/server/datastores/mysql.js +++ b/src/server/datastores/mysql.js @@ -8,6 +8,7 @@ const errCode = require('err-code') const { codes: errCodes } = require('../errors') const mysql = require('mysql') +const pRetry = require('p-retry') /** * @typedef {import('peer-id')} PeerId @@ -58,9 +59,8 @@ class Mysql { * @returns {Promise} */ async start () { - this.conn = mysql.createConnection(this.options) - - await this._initDB() + // Retry starting the Database in case it is still booting + await pRetry(() => this._initDB()) } /** @@ -311,6 +311,8 @@ class Mysql { * @returns {Promise} */ _initDB () { + this.conn = mysql.createConnection(this.options) + return new Promise((resolve, reject) => { this.conn.query(` CREATE TABLE IF NOT EXISTS registration ( @@ -332,8 +334,10 @@ class Mysql { ); `, (err) => { if (err) { + log.error(err) return reject(err) } + log('db is initialized') resolve() }) }) diff --git a/src/server/index.js b/src/server/index.js index 844723f..5d45e04 100644 --- a/src/server/index.js +++ b/src/server/index.js @@ -37,10 +37,10 @@ const { fallbackNullish } = require('./utils') * @typedef {Object} RendezvousServerOptions * @property {Datastore} datastore * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration - * @property {number} [maxTtl = MAX_TTL] maxium acceptable ttl to store a registration - * @property {number} [maxNsLength = MAX_NS_LENGTH] maxium acceptable namespace length - * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maxium acceptable discover limit - * @property {number} [maxPeerRegistrations = MAX_PEER_REGISTRATIONS] maxium acceptable registrations per peer + * @property {number} [maxTtl = MAX_TTL] maximum acceptable ttl to store a registration + * @property {number} [maxNsLength = MAX_NS_LENGTH] maximum acceptable namespace length + * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maximum acceptable discover limit + * @property {number} [maxPeerRegistrations = MAX_PEER_REGISTRATIONS] maximum acceptable registrations per peer * @property {number} [gcBootDelay = GC_BOOT_DELAY] delay before starting garbage collector job * @property {number} [gcMinInterval = GC_MIN_INTERVAL] minimum interval between each garbage collector job, in case maximum threshold reached * @property {number} [gcInterval = GC_INTERVAL] interval between each garbage collector job From 01ec7bc8135af0f704df9b098ae5a4d437b7db3a Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 4 Jan 2021 15:45:08 +0000 Subject: [PATCH 36/38] chore: remove client code and move server into src --- .aegir.js | 2 +- Dockerfile | 2 +- LIBP2P.md | 139 ------- package.json | 2 +- src/{server => }/bin.js | 0 src/constants.js | 11 +- src/{server => }/datastores/README.md | 0 src/{server => }/datastores/interface.ts | 0 src/{server => }/datastores/memory.js | 0 src/{server => }/datastores/mysql.js | 0 src/errors.js | 3 +- src/index.js | 396 ++++++++------------ src/{server => }/rpc/handlers/discover.js | 2 +- src/{server => }/rpc/handlers/index.js | 2 +- src/{server => }/rpc/handlers/register.js | 2 +- src/{server => }/rpc/handlers/unregister.js | 0 src/{server => }/rpc/index.js | 2 +- src/server/constants.js | 13 - src/server/errors.js | 5 - src/server/index.js | 229 ----------- src/{server => }/utils.js | 0 test/client/api.spec.js | 352 ----------------- test/dos-attack-protection.spec.js | 4 +- test/protocol.spec.js | 4 +- test/server.spec.js | 4 +- test/utils.js | 6 +- 26 files changed, 187 insertions(+), 993 deletions(-) delete mode 100644 LIBP2P.md rename src/{server => }/bin.js (100%) rename src/{server => }/datastores/README.md (100%) rename src/{server => }/datastores/interface.ts (100%) rename src/{server => }/datastores/memory.js (100%) rename src/{server => }/datastores/mysql.js (100%) rename src/{server => }/rpc/handlers/discover.js (98%) rename src/{server => }/rpc/handlers/index.js (92%) rename src/{server => }/rpc/handlers/register.js (98%) rename src/{server => }/rpc/handlers/unregister.js (100%) rename src/{server => }/rpc/index.js (97%) delete mode 100644 src/server/constants.js delete mode 100644 src/server/errors.js delete mode 100644 src/server/index.js rename src/{server => }/utils.js (100%) delete mode 100644 test/client/api.spec.js diff --git a/.aegir.js b/.aegir.js index 605568d..707f042 100644 --- a/.aegir.js +++ b/.aegir.js @@ -81,7 +81,7 @@ const after = async () => { } module.exports = { - bundlesize: { maxSize: '80kB' }, + bundlesize: { maxSize: '250kB' }, hooks: { pre: before, post: after diff --git a/Dockerfile b/Dockerfile index 6e35f80..63f3bad 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,4 +25,4 @@ ENV DEBUG libp2p* # Available overrides (defaults shown): # Server logging can be enabled via the DEBUG environment variable -CMD [ "/usr/local/bin/dumb-init", "node", "src/server/bin.js"] \ No newline at end of file +CMD [ "/usr/local/bin/dumb-init", "node", "src/bin.js"] \ No newline at end of file diff --git a/LIBP2P.md b/LIBP2P.md deleted file mode 100644 index 99e35ca..0000000 --- a/LIBP2P.md +++ /dev/null @@ -1,139 +0,0 @@ -# Rendezvous Protocol in js-libp2p - -The rendezvous protocol can be used in different contexts across libp2p. For using it, the libp2p network needs to have well known libp2p nodes acting as rendezvous servers. These nodes will have an extra role in the network. They will collect and maintain a list of registrations per rendezvous namespace. Other peers in the network will act as rendezvous clients and will register themselves on given namespaces by messaging a rendezvous server node. Taking into account these registrations, a rendezvous client is able to discover other peers in a given namespace by querying a server. A registration should have a `ttl`, in order to avoid having invalid registrations. - -## Usage - -`js-libp2p` supports the usage of the rendezvous protocol through its configuration. It allows the rendezvous protocol to be enabled and customized. - -You can configure it through libp2p as follows: - -```js -const Libp2p = require('libp2p') - -const node = await Libp2p.create({ - rendezvous: { - enabled: true, - rendezvousPoints: ['/dnsaddr/rendezvous.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJP'] - } -}) -``` - -## Libp2p Flow - -When a libp2p node with the rendezvous protocol enabled starts, it should start by connecting to the given rendezvous servers. When a rendezvous server is connected, the node can ask for nodes in given namespaces. An example of a namespace could be a relay namespace, so that undialable nodes can register themselves as reachable through that relay. - -When a libp2p node running the rendezvous protocol is stopping, it will unregister from all the namespaces previously registered. - -## API - -This API allows users to register new rendezvous namespaces, unregister from previously registered namespaces and to discover peers on a given namespace. - -### Options - -| Name | Type | Description | -|------|------|-------------| -| options | `object` | rendezvous parameters | -| options.enabled | `boolean` | is rendezvous enabled | -| options.rendezvousPoints | `Multiaddr[]` | list of multiaddrs of running rendezvous servers | - -### rendezvous.start - -Start the rendezvous client in the libp2p node. - -`rendezvous.start()` - -### rendezvous.stop - -Clear the rendezvous state and unregister from namespaces. - -`rendezvous.stop()` - -### rendezvous.register - -Registers the peer in a given namespace. - -`rendezvous.register(namespace, [options])` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to register | -| [options] | `Object` | rendezvous registrations options | -| [options.ttl=7.2e6] | `number` | registration ttl in ms | - -#### Returns - -| Type | Description | -|------|-------------| -| `Promise` | Remaining ttl value | - -#### Example - -```js -// ... -const ttl = await rendezvous.register(namespace) -``` - -### rendezvous.unregister - -Unregisters the peer from a given namespace. - -`rendezvous.unregister(namespace)` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to unregister | - -#### Returns - -| Type | Description | -|------|-------------| -| `Promise` | Operation resolved | - -#### Example - -```js -// ... -await rendezvous.register(namespace) -await rendezvous.unregister(namespace) -``` - -### rendezvous.discover - -Discovers peers registered under a given namespace. - -`rendezvous.discover(namespace, [limit])` - -#### Parameters - -| Name | Type | Description | -|------|------|-------------| -| namespace | `string` | namespace to discover | -| limit | `number` | limit of peers to discover | - -#### Returns - -| Type | Description | -|------|-------------| -| `AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>` | Async Iterable registrations | - -#### Example - -```js -// ... -await rendezvous.register(namespace) - -for await (const reg of rendezvous.discover(namespace)) { - console.log(reg.signedPeerRecord, reg.ns, reg.ttl) -} -``` - -## Future Work - -- Libp2p can handle re-registers when properly configured -- Rendezvous client should be able to register namespaces given in configuration on startup - - Not supported at the moment, as we would need to deal with re-register over time \ No newline at end of file diff --git a/package.json b/package.json index 38b9bb5..7883b8f 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ } }, "bin": { - "libp2p-rendezvous-server": "src/server/bin.js" + "libp2p-rendezvous-server": "src/bin.js" }, "files": [ "dist", diff --git a/src/server/bin.js b/src/bin.js similarity index 100% rename from src/server/bin.js rename to src/bin.js diff --git a/src/constants.js b/src/constants.js index 4a2b937..cc70e50 100644 --- a/src/constants.js +++ b/src/constants.js @@ -1,4 +1,13 @@ 'use strict' +exports.MAX_NS_LENGTH = 255 +exports.MAX_DISCOVER_LIMIT = 1000 +exports.MAX_PEER_REGISTRATIONS = 1000 +exports.MIN_TTL = 7.2e6 +exports.MAX_TTL = 2.592e+8 exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' -exports.MAX_DISCOVER_LIMIT = 50 +exports.GC_BOOT_DELAY = 10e6 +exports.GC_INTERVAL = 7.2e6 +exports.GC_MIN_INTERVAL = 3e6 +exports.GC_MIN_REGISTRATIONS = 1000 +exports.GC_MAX_REGISTRATIONS = 10e6 diff --git a/src/server/datastores/README.md b/src/datastores/README.md similarity index 100% rename from src/server/datastores/README.md rename to src/datastores/README.md diff --git a/src/server/datastores/interface.ts b/src/datastores/interface.ts similarity index 100% rename from src/server/datastores/interface.ts rename to src/datastores/interface.ts diff --git a/src/server/datastores/memory.js b/src/datastores/memory.js similarity index 100% rename from src/server/datastores/memory.js rename to src/datastores/memory.js diff --git a/src/server/datastores/mysql.js b/src/datastores/mysql.js similarity index 100% rename from src/server/datastores/mysql.js rename to src/datastores/mysql.js diff --git a/src/errors.js b/src/errors.js index 9b0e8e3..efb6560 100644 --- a/src/errors.js +++ b/src/errors.js @@ -1,6 +1,5 @@ 'use strict' exports.codes = { - INVALID_NAMESPACE: 'ERR_INVALID_NAMESPACE', - NO_CONNECTED_RENDEZVOUS_SERVERS: 'ERR_NO_CONNECTED_RENDEZVOUS_SERVERS' + INVALID_COOKIE: 'ERR_INVALID_COOKIE' } diff --git a/src/index.js b/src/index.js index acd3cca..5d45e04 100644 --- a/src/index.js +++ b/src/index.js @@ -1,305 +1,229 @@ 'use strict' const debug = require('debug') -const log = Object.assign(debug('libp2p:rendezvous'), { - error: debug('libp2p:rendezvous:err') +const log = Object.assign(debug('libp2p:rendezvous-server'), { + error: debug('libp2p:rendezvous-server:err') }) +const { + setDelayedInterval, + clearDelayedInterval +} = require('set-delayed-interval') -const errCode = require('err-code') -const { pipe } = require('it-pipe') -const lp = require('it-length-prefixed') -const { collect } = require('streaming-iterables') -const { toBuffer } = require('it-buffer') -const fromString = require('uint8arrays/from-string') -const toString = require('uint8arrays/to-string') +const Libp2p = require('libp2p') +const PeerId = require('peer-id') -const { codes: errCodes } = require('./errors') +const rpc = require('./rpc') const { + MIN_TTL, + MAX_TTL, + MAX_NS_LENGTH, MAX_DISCOVER_LIMIT, + MAX_PEER_REGISTRATIONS, + GC_BOOT_DELAY, + GC_INTERVAL, + GC_MIN_INTERVAL, + GC_MIN_REGISTRATIONS, + GC_MAX_REGISTRATIONS, PROTOCOL_MULTICODEC } = require('./constants') -const { Message } = require('./proto') -const MESSAGE_TYPE = Message.MessageType +const { fallbackNullish } = require('./utils') /** - * @typedef {import('libp2p')} Libp2p - * @typedef {import('multiaddr')} Multiaddr + * @typedef {import('./datastores/interface').Datastore} Datastore + * @typedef {import('./datastores/interface').Registration} Registration */ /** - * @typedef {Object} RendezvousProperties - * @property {Libp2p} libp2p - * - * @typedef {Object} RendezvousOptions - * @property {Multiaddr[]} rendezvousPoints + * @typedef {Object} RendezvousServerOptions + * @property {Datastore} datastore + * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration + * @property {number} [maxTtl = MAX_TTL] maximum acceptable ttl to store a registration + * @property {number} [maxNsLength = MAX_NS_LENGTH] maximum acceptable namespace length + * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maximum acceptable discover limit + * @property {number} [maxPeerRegistrations = MAX_PEER_REGISTRATIONS] maximum acceptable registrations per peer + * @property {number} [gcBootDelay = GC_BOOT_DELAY] delay before starting garbage collector job + * @property {number} [gcMinInterval = GC_MIN_INTERVAL] minimum interval between each garbage collector job, in case maximum threshold reached + * @property {number} [gcInterval = GC_INTERVAL] interval between each garbage collector job + * @property {number} [gcMinRegistrations = GC_MIN_REGISTRATIONS] minimum number of registration for triggering garbage collector + * @property {number} [gcMaxRegistrations = GC_MAX_REGISTRATIONS] maximum number of registration for triggering garbage collector */ -class Rendezvous { +/** + * Libp2p rendezvous server. + */ +class RendezvousServer extends Libp2p { /** - * Libp2p Rendezvous. A lightweight mechanism for generalized peer discovery. - * * @class - * @param {RendezvousProperties & RendezvousOptions} params + * @param {import('libp2p').Libp2pOptions} libp2pOptions + * @param {RendezvousServerOptions} options */ - constructor ({ libp2p, rendezvousPoints }) { - this._libp2p = libp2p - this._peerId = libp2p.peerId - this._peerStore = libp2p.peerStore - this._connectionManager = libp2p.connectionManager - this._rendezvousPoints = rendezvousPoints - - this._isStarted = false - - /** - * Map namespaces to a map of rendezvous point identifier to cookie. - * - * @type {Map>} - */ - this._cookies = new Map() + constructor (libp2pOptions, options) { + super(libp2pOptions) + + this._minTtl = fallbackNullish(options.minTtl, MIN_TTL) + this._maxTtl = fallbackNullish(options.maxTtl, MAX_TTL) + this._maxNsLength = fallbackNullish(options.maxNsLength, MAX_NS_LENGTH) + this._maxDiscoveryLimit = fallbackNullish(options.maxDiscoveryLimit, MAX_DISCOVER_LIMIT) + this._maxPeerRegistrations = fallbackNullish(options.maxPeerRegistrations, MAX_PEER_REGISTRATIONS) + + this.rendezvousDatastore = options.datastore + + this._registrationsCount = 0 + this._lastGcTs = 0 + this._gcDelay = fallbackNullish(options.gcBootDelay, GC_BOOT_DELAY) + this._gcInterval = fallbackNullish(options.gcInterval, GC_INTERVAL) + this._gcMinInterval = fallbackNullish(options.gcMinInterval, GC_MIN_INTERVAL) + this._gcMinRegistrations = fallbackNullish(options.gcMinRegistrations, GC_MIN_REGISTRATIONS) + this._gcMaxRegistrations = fallbackNullish(options.gcMaxRegistrations, GC_MAX_REGISTRATIONS) + this._gcJob = this._gcJob.bind(this) } /** - * Start the rendezvous client in the libp2p node. + * Start rendezvous server for handling rendezvous streams and gc. * - * @returns {void} + * @returns {Promise} */ - start () { - if (this._isStarted) { + async start () { + super.start() + + if (this._timeout) { return } - this._isStarted = true + log('starting') + + await this.rendezvousDatastore.start() + + // Garbage collection + this._timeout = setDelayedInterval( + this._gcJob, this._gcInterval, this._gcDelay + ) + + // Incoming streams handling + this.handle(PROTOCOL_MULTICODEC, rpc(this)) + + // Remove peer records from memory as they are not needed + // TODO: This should be handled by PeerStore itself in the future + this.peerStore.on('peer', (peerId) => { + this.peerStore.delete(peerId) + }) + log('started') } /** - * Clear the rendezvous state and unregister from namespaces. + * Stops rendezvous server gc and clears registrations * - * @returns {void} + * @returns {Promise} */ stop () { - if (!this._isStarted) { - return - } + this.unhandle(PROTOCOL_MULTICODEC) + clearDelayedInterval(this._timeout) - this._isStarted = false - this._cookies.clear() + this.rendezvousDatastore.stop() + + super.stop() log('stopped') - // TODO: should unregister from the namespaces registered + return Promise.resolve() } /** - * Register the peer in a given namespace + * Call garbage collector if enough registrations. * - * @param {string} ns - * @param {object} [options] - * @param {number} [options.ttl = 7.2e6] - registration ttl in ms - * @returns {Promise} rendezvous register ttl. + * @returns {Promise} */ - async register (ns, { ttl = 7.2e6 } = {}) { - if (!ns) { - throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) - } - - // Are there available rendezvous servers? - if (!this._rendezvousPoints || !this._rendezvousPoints.length) { - throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - } - - const message = Message.encode({ - type: MESSAGE_TYPE.REGISTER, - register: { - signedPeerRecord: this._libp2p.peerStore.addressBook.getRawEnvelope(this._peerId), - ns, - ttl: ttl * 1e-3 // Convert to seconds - } - }) - - const registerTasks = [] - - /** - * @param {Multiaddr} m - * @returns {Promise} - */ - const taskFn = async (m) => { - const connection = await this._libp2p.dial(m) - const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) - - const [response] = await pipe( - [message], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - // Close connection if not any other open streams - if (!connection.streams.length) { - await connection.close() - } - - const recMessage = Message.decode(response) - - if (!recMessage.type === MESSAGE_TYPE.REGISTER_RESPONSE) { - throw new Error('unexpected message received') - } - - if (recMessage.registerResponse.status !== Message.ResponseStatus.OK) { - throw errCode(new Error(recMessage.registerResponse.statusText), recMessage.registerResponse.status) - } - - return recMessage.registerResponse.ttl * 1e3 // convert to ms + async _gcJob () { + if (this._registrationsCount > this._gcMinRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { + await this._gc() } + } - for (const m of this._rendezvousPoints) { - registerTasks.push(taskFn(m)) - } + /** + * Run datastore garbage collector. + * + * @returns {Promise} + */ + async _gc () { + log('gc starting') - // Return first ttl - // TODO: consider pAny instead of Promise.all? - const [returnTtl] = await Promise.all(registerTasks) + const count = await this.rendezvousDatastore.gc() + this._registrationsCount -= count + this._lastGcTs = Date.now() - return returnTtl + log('gc finished') } /** - * Unregister peer from the nampesapce. + * Add a peer registration to a namespace. * * @param {string} ns + * @param {PeerId} peerId + * @param {Uint8Array} signedPeerRecord + * @param {number} ttl * @returns {Promise} */ - async unregister (ns) { - if (!ns) { - throw errCode(new Error('a namespace must be provided'), errCodes.INVALID_NAMESPACE) - } - - // Are there available rendezvous servers? - if (!this._rendezvousPoints || !this._rendezvousPoints.length) { - throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) + async addRegistration (ns, peerId, signedPeerRecord, ttl) { + await this.rendezvousDatastore.addRegistration(ns, peerId, signedPeerRecord, ttl) + log(`added registration for the namespace ${ns} with peer ${peerId.toB58String()}`) + + this._registrationsCount += 1 + // Manually trigger garbage collector if max registrations threshold reached + // and the minGc interval is finished + if (this._registrationsCount >= this._gcMaxRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { + this._gc() } + } - const message = Message.encode({ - type: MESSAGE_TYPE.UNREGISTER, - unregister: { - id: this._peerId.toBytes(), - ns - } - }) + /** + * Remove registration of a given namespace to a peer + * + * @param {string} ns + * @param {PeerId} peerId + * @returns {Promise} + */ + async removeRegistration (ns, peerId) { + const count = await this.rendezvousDatastore.removeRegistration(ns, peerId) + log(`removed existing registrations for the namespace ${ns} - peer ${peerId.toB58String()} pair`) - const unregisterTasks = [] - /** - * @param {Multiaddr} m - * @returns {Promise} - */ - const taskFn = async (m) => { - const connection = await this._libp2p.dial(m) - const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) - - await pipe( - [message], - lp.encode(), - stream, - async (source) => { - for await (const _ of source) { } // eslint-disable-line - } - ) - - // Close connection if not any other open streams - if (!connection.streams.length) { - await connection.close() - } - } + this._registrationsCount -= count + } - for (const m of this._rendezvousPoints) { - unregisterTasks.push(taskFn(m)) - } + /** + * Remove all registrations of a given peer + * + * @param {PeerId} peerId + * @returns {Promise} + */ + async removePeerRegistrations (peerId) { + const count = await this.rendezvousDatastore.removePeerRegistrations(peerId) + log(`removed existing registrations for peer ${peerId.toB58String()}`) - await Promise.all(unregisterTasks) + this._registrationsCount -= count } /** - * Discover peers registered under a given namespace + * Get registrations for a namespace * * @param {string} ns - * @param {number} [limit = MAX_DISCOVER_LIMIT] - * @returns {AsyncIterable<{ signedPeerRecord: Uint8Array, ns: string, ttl: number }>} + * @param {object} [options] + * @param {number} [options.limit] + * @param {string} [options.cookie] + * @returns {Promise<{ registrations: Array, cookie?: string }>} */ - async * discover (ns, limit = MAX_DISCOVER_LIMIT) { - // TODO: consider opening the envelope in the dicover - // This would store the addresses in the AddressBook - - // Are there available rendezvous servers? - if (!this._rendezvousPoints || !this._rendezvousPoints.length) { - throw errCode(new Error('no rendezvous servers connected'), errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - } - - const registrationTransformer = (r) => ({ - signedPeerRecord: r.signedPeerRecord, - ns: r.ns, - ttl: r.ttl * 1e3 // convert to ms - }) + async getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { + return await this.rendezvousDatastore.getRegistrations(ns, { limit, cookie }) + } - // Iterate over all rendezvous points - for (const m of this._rendezvousPoints) { - const namespaseCookies = this._cookies.get(ns) || new Map() - - // Check if we have a cookie and encode discover message - const cookie = namespaseCookies.get(m.toString()) - const message = Message.encode({ - type: MESSAGE_TYPE.DISCOVER, - discover: { - ns, - limit, - cookie: cookie ? fromString(cookie) : undefined - } - }) - - // Send discover message and wait for response - const connection = await this._libp2p.dial(m) - const { stream } = await connection.newStream(PROTOCOL_MULTICODEC) - const [response] = await pipe( - [message], - lp.encode(), - stream, - lp.decode(), - toBuffer, - collect - ) - - if (!connection.streams.length) { - await connection.close() - } - - const recMessage = Message.decode(response) - - if (!recMessage.type === MESSAGE_TYPE.DISCOVER_RESPONSE) { - throw new Error('unexpected message received') - } else if (recMessage.discoverResponse.status !== Message.ResponseStatus.OK) { - throw errCode(new Error(recMessage.discoverResponse.statusText), recMessage.discoverResponse.status) - } - - // Iterate over registrations response - for (const r of recMessage.discoverResponse.registrations) { - // track registrations - yield registrationTransformer(r) - - limit-- - if (limit === 0) { - break - } - } - - // Store cookie - const c = recMessage.discoverResponse.cookie - if (c && c.length) { - const nsCookies = this._cookies.get(ns) || new Map() - nsCookies.set(m.toString(), toString(c)) - this._cookies.set(ns, nsCookies) - } - } + /** + * Get number of registrations of a given peer. + * + * @param {PeerId} peerId + * @returns {Promise} + */ + async getNumberOfRegistrationsFromPeer (peerId) { + return await this.rendezvousDatastore.getNumberOfRegistrationsFromPeer(peerId) } } -module.exports = Rendezvous +module.exports = RendezvousServer diff --git a/src/server/rpc/handlers/discover.js b/src/rpc/handlers/discover.js similarity index 98% rename from src/server/rpc/handlers/discover.js rename to src/rpc/handlers/discover.js index 69a3c36..d64d628 100644 --- a/src/server/rpc/handlers/discover.js +++ b/src/rpc/handlers/discover.js @@ -9,7 +9,7 @@ const log = Object.assign(debug('libp2p:rendezvous-server:rpc:discover'), { const fromString = require('uint8arrays/from-string') const toString = require('uint8arrays/to-string') -const { Message } = require('../../../proto') +const { Message } = require('../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus diff --git a/src/server/rpc/handlers/index.js b/src/rpc/handlers/index.js similarity index 92% rename from src/server/rpc/handlers/index.js rename to src/rpc/handlers/index.js index 9daaf91..f8d37f3 100644 --- a/src/server/rpc/handlers/index.js +++ b/src/rpc/handlers/index.js @@ -1,6 +1,6 @@ 'use strict' -const { Message } = require('../../../proto') +const { Message } = require('../../proto') const MESSAGE_TYPE = Message.MessageType module.exports = (server) => { diff --git a/src/server/rpc/handlers/register.js b/src/rpc/handlers/register.js similarity index 98% rename from src/server/rpc/handlers/register.js rename to src/rpc/handlers/register.js index fbb8248..77bbd32 100644 --- a/src/server/rpc/handlers/register.js +++ b/src/rpc/handlers/register.js @@ -9,7 +9,7 @@ const log = Object.assign(debug('libp2p:rendezvous-server:rpc:register'), { const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') -const { Message } = require('../../../proto') +const { Message } = require('../../proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus diff --git a/src/server/rpc/handlers/unregister.js b/src/rpc/handlers/unregister.js similarity index 100% rename from src/server/rpc/handlers/unregister.js rename to src/rpc/handlers/unregister.js diff --git a/src/server/rpc/index.js b/src/rpc/index.js similarity index 97% rename from src/server/rpc/index.js rename to src/rpc/index.js index 8c86143..84df8e2 100644 --- a/src/server/rpc/index.js +++ b/src/rpc/index.js @@ -10,7 +10,7 @@ const lp = require('it-length-prefixed') const { toBuffer } = require('it-buffer') const handlers = require('./handlers') -const { Message } = require('../../proto') +const { Message } = require('../proto') module.exports = (rendezvous) => { const getMessageHandler = handlers(rendezvous) diff --git a/src/server/constants.js b/src/server/constants.js deleted file mode 100644 index cc70e50..0000000 --- a/src/server/constants.js +++ /dev/null @@ -1,13 +0,0 @@ -'use strict' - -exports.MAX_NS_LENGTH = 255 -exports.MAX_DISCOVER_LIMIT = 1000 -exports.MAX_PEER_REGISTRATIONS = 1000 -exports.MIN_TTL = 7.2e6 -exports.MAX_TTL = 2.592e+8 -exports.PROTOCOL_MULTICODEC = '/rendezvous/1.0.0' -exports.GC_BOOT_DELAY = 10e6 -exports.GC_INTERVAL = 7.2e6 -exports.GC_MIN_INTERVAL = 3e6 -exports.GC_MIN_REGISTRATIONS = 1000 -exports.GC_MAX_REGISTRATIONS = 10e6 diff --git a/src/server/errors.js b/src/server/errors.js deleted file mode 100644 index efb6560..0000000 --- a/src/server/errors.js +++ /dev/null @@ -1,5 +0,0 @@ -'use strict' - -exports.codes = { - INVALID_COOKIE: 'ERR_INVALID_COOKIE' -} diff --git a/src/server/index.js b/src/server/index.js deleted file mode 100644 index 5d45e04..0000000 --- a/src/server/index.js +++ /dev/null @@ -1,229 +0,0 @@ -'use strict' - -const debug = require('debug') -const log = Object.assign(debug('libp2p:rendezvous-server'), { - error: debug('libp2p:rendezvous-server:err') -}) -const { - setDelayedInterval, - clearDelayedInterval -} = require('set-delayed-interval') - -const Libp2p = require('libp2p') -const PeerId = require('peer-id') - -const rpc = require('./rpc') -const { - MIN_TTL, - MAX_TTL, - MAX_NS_LENGTH, - MAX_DISCOVER_LIMIT, - MAX_PEER_REGISTRATIONS, - GC_BOOT_DELAY, - GC_INTERVAL, - GC_MIN_INTERVAL, - GC_MIN_REGISTRATIONS, - GC_MAX_REGISTRATIONS, - PROTOCOL_MULTICODEC -} = require('./constants') -const { fallbackNullish } = require('./utils') - -/** - * @typedef {import('./datastores/interface').Datastore} Datastore - * @typedef {import('./datastores/interface').Registration} Registration - */ - -/** - * @typedef {Object} RendezvousServerOptions - * @property {Datastore} datastore - * @property {number} [minTtl = MIN_TTL] minimum acceptable ttl to store a registration - * @property {number} [maxTtl = MAX_TTL] maximum acceptable ttl to store a registration - * @property {number} [maxNsLength = MAX_NS_LENGTH] maximum acceptable namespace length - * @property {number} [maxDiscoveryLimit = MAX_DISCOVER_LIMIT] maximum acceptable discover limit - * @property {number} [maxPeerRegistrations = MAX_PEER_REGISTRATIONS] maximum acceptable registrations per peer - * @property {number} [gcBootDelay = GC_BOOT_DELAY] delay before starting garbage collector job - * @property {number} [gcMinInterval = GC_MIN_INTERVAL] minimum interval between each garbage collector job, in case maximum threshold reached - * @property {number} [gcInterval = GC_INTERVAL] interval between each garbage collector job - * @property {number} [gcMinRegistrations = GC_MIN_REGISTRATIONS] minimum number of registration for triggering garbage collector - * @property {number} [gcMaxRegistrations = GC_MAX_REGISTRATIONS] maximum number of registration for triggering garbage collector - */ - -/** - * Libp2p rendezvous server. - */ -class RendezvousServer extends Libp2p { - /** - * @class - * @param {import('libp2p').Libp2pOptions} libp2pOptions - * @param {RendezvousServerOptions} options - */ - constructor (libp2pOptions, options) { - super(libp2pOptions) - - this._minTtl = fallbackNullish(options.minTtl, MIN_TTL) - this._maxTtl = fallbackNullish(options.maxTtl, MAX_TTL) - this._maxNsLength = fallbackNullish(options.maxNsLength, MAX_NS_LENGTH) - this._maxDiscoveryLimit = fallbackNullish(options.maxDiscoveryLimit, MAX_DISCOVER_LIMIT) - this._maxPeerRegistrations = fallbackNullish(options.maxPeerRegistrations, MAX_PEER_REGISTRATIONS) - - this.rendezvousDatastore = options.datastore - - this._registrationsCount = 0 - this._lastGcTs = 0 - this._gcDelay = fallbackNullish(options.gcBootDelay, GC_BOOT_DELAY) - this._gcInterval = fallbackNullish(options.gcInterval, GC_INTERVAL) - this._gcMinInterval = fallbackNullish(options.gcMinInterval, GC_MIN_INTERVAL) - this._gcMinRegistrations = fallbackNullish(options.gcMinRegistrations, GC_MIN_REGISTRATIONS) - this._gcMaxRegistrations = fallbackNullish(options.gcMaxRegistrations, GC_MAX_REGISTRATIONS) - this._gcJob = this._gcJob.bind(this) - } - - /** - * Start rendezvous server for handling rendezvous streams and gc. - * - * @returns {Promise} - */ - async start () { - super.start() - - if (this._timeout) { - return - } - - log('starting') - - await this.rendezvousDatastore.start() - - // Garbage collection - this._timeout = setDelayedInterval( - this._gcJob, this._gcInterval, this._gcDelay - ) - - // Incoming streams handling - this.handle(PROTOCOL_MULTICODEC, rpc(this)) - - // Remove peer records from memory as they are not needed - // TODO: This should be handled by PeerStore itself in the future - this.peerStore.on('peer', (peerId) => { - this.peerStore.delete(peerId) - }) - - log('started') - } - - /** - * Stops rendezvous server gc and clears registrations - * - * @returns {Promise} - */ - stop () { - this.unhandle(PROTOCOL_MULTICODEC) - clearDelayedInterval(this._timeout) - - this.rendezvousDatastore.stop() - - super.stop() - log('stopped') - - return Promise.resolve() - } - - /** - * Call garbage collector if enough registrations. - * - * @returns {Promise} - */ - async _gcJob () { - if (this._registrationsCount > this._gcMinRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { - await this._gc() - } - } - - /** - * Run datastore garbage collector. - * - * @returns {Promise} - */ - async _gc () { - log('gc starting') - - const count = await this.rendezvousDatastore.gc() - this._registrationsCount -= count - this._lastGcTs = Date.now() - - log('gc finished') - } - - /** - * Add a peer registration to a namespace. - * - * @param {string} ns - * @param {PeerId} peerId - * @param {Uint8Array} signedPeerRecord - * @param {number} ttl - * @returns {Promise} - */ - async addRegistration (ns, peerId, signedPeerRecord, ttl) { - await this.rendezvousDatastore.addRegistration(ns, peerId, signedPeerRecord, ttl) - log(`added registration for the namespace ${ns} with peer ${peerId.toB58String()}`) - - this._registrationsCount += 1 - // Manually trigger garbage collector if max registrations threshold reached - // and the minGc interval is finished - if (this._registrationsCount >= this._gcMaxRegistrations && Date.now() > this._gcMinInterval + this._lastGcTs) { - this._gc() - } - } - - /** - * Remove registration of a given namespace to a peer - * - * @param {string} ns - * @param {PeerId} peerId - * @returns {Promise} - */ - async removeRegistration (ns, peerId) { - const count = await this.rendezvousDatastore.removeRegistration(ns, peerId) - log(`removed existing registrations for the namespace ${ns} - peer ${peerId.toB58String()} pair`) - - this._registrationsCount -= count - } - - /** - * Remove all registrations of a given peer - * - * @param {PeerId} peerId - * @returns {Promise} - */ - async removePeerRegistrations (peerId) { - const count = await this.rendezvousDatastore.removePeerRegistrations(peerId) - log(`removed existing registrations for peer ${peerId.toB58String()}`) - - this._registrationsCount -= count - } - - /** - * Get registrations for a namespace - * - * @param {string} ns - * @param {object} [options] - * @param {number} [options.limit] - * @param {string} [options.cookie] - * @returns {Promise<{ registrations: Array, cookie?: string }>} - */ - async getRegistrations (ns, { limit = MAX_DISCOVER_LIMIT, cookie } = {}) { - return await this.rendezvousDatastore.getRegistrations(ns, { limit, cookie }) - } - - /** - * Get number of registrations of a given peer. - * - * @param {PeerId} peerId - * @returns {Promise} - */ - async getNumberOfRegistrationsFromPeer (peerId) { - return await this.rendezvousDatastore.getNumberOfRegistrationsFromPeer(peerId) - } -} - -module.exports = RendezvousServer diff --git a/src/server/utils.js b/src/utils.js similarity index 100% rename from src/server/utils.js rename to src/utils.js diff --git a/test/client/api.spec.js b/test/client/api.spec.js deleted file mode 100644 index 66bf19b..0000000 --- a/test/client/api.spec.js +++ /dev/null @@ -1,352 +0,0 @@ -'use strict' -/* eslint-env mocha */ - -const { expect } = require('aegir/utils/chai') -const sinon = require('sinon') - -const delay = require('delay') -const pWaitFor = require('p-wait-for') - -const multiaddr = require('multiaddr') -const Envelope = require('libp2p/src/record/envelope') -const PeerRecord = require('libp2p/src/record/peer-record') - -const Rendezvous = require('../../src') -const { codes: errCodes } = require('../../src/errors') - -const { Message } = require('../../src/proto') -const RESPONSE_STATUS = Message.ResponseStatus - -const { - createPeer, - createRendezvousServer, - createSignedPeerRecord -} = require('../utils') -const { MULTIADDRS_WEBSOCKETS } = require('../fixtures/browser') -const relayAddr = MULTIADDRS_WEBSOCKETS[0] - -const namespace = 'ns' - -describe('rendezvous api', () => { - describe('no rendezvous server', () => { - let clients - - // Create and start Libp2p nodes - beforeEach(async () => { - clients = await createPeer({ number: 2 }) - - // Create 2 rendezvous clients - clients.forEach((peer) => { - const rendezvous = new Rendezvous({ libp2p: peer }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - }) - - afterEach(async () => { - sinon.restore() - - for (const peer of clients) { - await peer.rendezvous.stop() - await peer.stop() - } - }) - - it('register throws error if no rendezvous servers', async () => { - await expect(clients[0].rendezvous.register(namespace)) - .to.eventually.rejected() - .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - }) - - it('unregister throws error if no rendezvous servers', async () => { - await expect(clients[0].rendezvous.unregister(namespace)) - .to.eventually.rejected() - .and.have.property('code', errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - }) - - it('discover throws error if no rendezvous servers', async () => { - try { - for await (const _ of clients[0].rendezvous.discover()) { } // eslint-disable-line - } catch (err) { - expect(err).to.exist() - expect(err.code).to.eql(errCodes.NO_CONNECTED_RENDEZVOUS_SERVERS) - return - } - throw new Error('discover should throw error if no rendezvous servers') - }) - }) - - describe('one rendezvous server', () => { - let rendezvousServer - let clients - - // Create and start Libp2p - beforeEach(async function () { - this.timeout(10e3) - // Create Rendezvous Server - rendezvousServer = await createRendezvousServer() - await pWaitFor(() => rendezvousServer.multiaddrs.length > 0) - const rendezvousServerMultiaddr = `${rendezvousServer.multiaddrs[0]}/p2p/${rendezvousServer.peerId.toB58String()}` - - // Create 2 rendezvous clients - clients = await createPeer({ number: 2 }) - clients.forEach((peer) => { - const rendezvous = new Rendezvous({ libp2p: peer, rendezvousPoints: [rendezvousServerMultiaddr] }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - }) - - afterEach(async function () { - this.timeout(10e3) - sinon.restore() - await delay(500) // Await for datastore to be ready - await rendezvousServer.rendezvousDatastore.reset() - await rendezvousServer.stop() - await Promise.all(clients.map(async (peer) => { - await peer.rendezvous.stop() - await peer.stop() - })) - }) - - it('register throws error if a namespace is not provided', async () => { - await expect(clients[0].rendezvous.register()) - .to.eventually.rejected() - .and.have.property('code', errCodes.INVALID_NAMESPACE) - }) - - it('register throws an error with an invalid namespace', async () => { - const badNamespace = 'x'.repeat(300) - - await expect(clients[0].rendezvous.register(badNamespace)) - .to.eventually.rejected() - .and.have.property('code', RESPONSE_STATUS.E_INVALID_NAMESPACE) - - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('register throws an error with an invalid ttl', async () => { - const badTtl = 5e10 - - await expect(clients[0].rendezvous.register(namespace, { ttl: badTtl })) - .to.eventually.rejected() - .and.have.property('code', RESPONSE_STATUS.E_INVALID_TTL) - - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('register throws an error with an invalid peerId', async () => { - const badSignedPeerRecord = await createSignedPeerRecord(clients[1].peerId, [multiaddr('/ip4/127.0.0.1/tcp/100')]) - - const stub = sinon.stub(clients[0].peerStore.addressBook, 'getRawEnvelope') - stub.onCall(0).returns(badSignedPeerRecord.marshal()) - - await expect(clients[0].rendezvous.register(namespace)) - .to.eventually.rejected() - .and.have.property('code', RESPONSE_STATUS.E_NOT_AUTHORIZED) - - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('registers with an available rendezvous server node', async () => { - const registers = [] - - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - await clients[0].rendezvous.register(namespace) - - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of clients[1].rendezvous.discover(namespace)) { - registers.push(reg) - } - - expect(registers).to.have.lengthOf(1) - expect(registers[0].ns).to.eql(namespace) - }) - - it('unregister throws if a namespace is not provided', async () => { - await expect(clients[0].rendezvous.unregister()) - .to.eventually.rejected() - .and.have.property('code', errCodes.INVALID_NAMESPACE) - }) - - it('unregisters with an available rendezvous server node', async () => { - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - // Register - await clients[0].rendezvous.register(namespace) - - // Unregister - await clients[0].rendezvous.unregister(namespace) - - // other client does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('unregister not fails if not registered', async () => { - await clients[0].rendezvous.unregister(namespace) - }) - - it('discover throws error if a namespace is invalid', async () => { - const badNamespace = 'x'.repeat(300) - - try { - for await (const _ of clients[0].rendezvous.discover(badNamespace)) { } // eslint-disable-line - } catch (err) { - expect(err).to.exist() - expect(err.code).to.eql(RESPONSE_STATUS.E_INVALID_NAMESPACE) - return - } - - throw new Error('discover should throw error if a namespace is not provided') - }) - - it('discover does not find any register if there is none', async () => { - for await (const reg of clients[0].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - }) - - it('discover finds registered peer for namespace', async () => { - const registers = [] - - // Peer2 does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - // Peer0 register itself on namespace (connected to Peer1) - await clients[0].rendezvous.register(namespace) - - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of clients[1].rendezvous.discover(namespace)) { - registers.push(reg) - } - - expect(registers).to.have.lengthOf(1) - expect(registers[0].signedPeerRecord).to.exist() - expect(registers[0].ns).to.eql(namespace) - expect(registers[0].ttl).to.exist() - - // Validate envelope - const envelope = await Envelope.openAndCertify(registers[0].signedPeerRecord, PeerRecord.DOMAIN) - const rec = PeerRecord.createFromProtobuf(envelope.payload) - - expect(rec.multiaddrs).to.eql(clients[0].multiaddrs) - }) - - it('discover finds registered peer for namespace once (cookie)', async () => { - const registers = [] - - // Peer2 does not discovery any peer registered - for await (const _ of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - // Peer0 register itself on namespace (connected to Peer1) - await clients[0].rendezvous.register(namespace) - - // Peer2 discovers Peer0 registered in Peer1 - for await (const reg of clients[1].rendezvous.discover(namespace)) { - registers.push(reg) - } - - expect(registers).to.have.lengthOf(1) - expect(registers[0].signedPeerRecord).to.exist() - expect(registers[0].ns).to.eql(namespace) - expect(registers[0].ttl).to.exist() - - for await (const reg of clients[1].rendezvous.discover(namespace)) { - registers.push(reg) - } - - expect(registers).to.have.lengthOf(1) - }) - }) - - describe('multiple rendezvous servers available', () => { - let rendezvousServers = [] - let clients - - // Create and start Libp2p nodes - beforeEach(async function () { - this.timeout(20e3) - // Create Rendezvous Server - rendezvousServers = await Promise.all([ - createRendezvousServer(), - createRendezvousServer() - ]) - await pWaitFor(() => rendezvousServers[0].multiaddrs.length > 0 && rendezvousServers[1].multiaddrs.length > 0) - const rendezvousServerMultiaddrs = rendezvousServers.map((rendezvousServer) => `${rendezvousServer.multiaddrs[0]}/p2p/${rendezvousServer.peerId.toB58String()}`) - - // Create 2 rendezvous clients - clients = await createPeer({ number: 2 }) - clients.forEach((peer) => { - const rendezvous = new Rendezvous({ libp2p: peer, rendezvousPoints: rendezvousServerMultiaddrs }) - rendezvous.start() - peer.rendezvous = rendezvous - }) - - // Connect to testing relay node - await Promise.all(clients.map((libp2p) => libp2p.dial(relayAddr))) - await Promise.all(rendezvousServers.map((libp2p) => libp2p.dial(relayAddr))) - }) - - afterEach(async function () { - this.timeout(20e3) - await Promise.all(rendezvousServers.map((s) => s.rendezvousDatastore.reset())) - await Promise.all(rendezvousServers.map((s) => s.stop())) - await Promise.all(clients.map((libp2p) => { - libp2p.rendezvous.stop() - return libp2p.stop() - })) - }) - - it('discover find registered peer for namespace only when registered ', async () => { - const registers = [] - - // Client 1 does not discovery any peer registered - for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - throw new Error('no registers should exist') - } - - // Client 0 register itself on namespace (connected to Peer1) - await clients[0].rendezvous.register(namespace) - - // Client1 discovers Client0 - for await (const reg of clients[1].rendezvous.discover(namespace)) { - registers.push(reg) - } - - expect(registers[0].signedPeerRecord).to.exist() - expect(registers[0].ns).to.eql(namespace) - expect(registers[0].ttl).to.exist() - - // Client0 unregister itself on namespace - await clients[0].rendezvous.unregister(namespace) - - // Peer2 does not discovery any peer registered - // TODO: Cookies not available as they were removed - // for await (const reg of clients[1].rendezvous.discover(namespace)) { // eslint-disable-line - // throw new Error('no registers should exist') - // } - }) - }) -}) diff --git a/test/dos-attack-protection.spec.js b/test/dos-attack-protection.spec.js index eaf3841..abf25c6 100644 --- a/test/dos-attack-protection.spec.js +++ b/test/dos-attack-protection.spec.js @@ -11,10 +11,10 @@ const { toBuffer } = require('it-buffer') const multiaddr = require('multiaddr') const Libp2p = require('libp2p') -const RendezvousServer = require('../src/server') +const RendezvousServer = require('../src') const { PROTOCOL_MULTICODEC -} = require('../src/server/constants') +} = require('../src/constants') const { Message } = require('../src/proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus diff --git a/test/protocol.spec.js b/test/protocol.spec.js index e22b774..95b10ed 100644 --- a/test/protocol.spec.js +++ b/test/protocol.spec.js @@ -12,10 +12,10 @@ const multiaddr = require('multiaddr') const PeerId = require('peer-id') const Libp2p = require('libp2p') -const RendezvousServer = require('../src/server') +const RendezvousServer = require('../src') const { PROTOCOL_MULTICODEC -} = require('../src/server/constants') +} = require('../src/constants') const { Message } = require('../src/proto') const MESSAGE_TYPE = Message.MessageType const RESPONSE_STATUS = Message.ResponseStatus diff --git a/test/server.spec.js b/test/server.spec.js index c7acddd..b328cf9 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -11,8 +11,8 @@ const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') -const RendezvousServer = require('../src/server') -const { codes: errCodes } = require('../src/server/errors') +const RendezvousServer = require('../src') +const { codes: errCodes } = require('../src/errors') const { createPeerId, createSignedPeerRecord, diff --git a/test/utils.js b/test/utils.js index e73def6..d548211 100644 --- a/test/utils.js +++ b/test/utils.js @@ -13,7 +13,7 @@ const multiaddr = require('multiaddr') const Envelope = require('libp2p/src/record/envelope') const PeerRecord = require('libp2p/src/record/peer-record') -const RendezvousServer = require('../src/server') +const RendezvousServer = require('../src') const Peers = require('./fixtures/peers') const { MULTIADDRS_WEBSOCKETS } = require('./fixtures/browser') @@ -120,11 +120,11 @@ module.exports.createSignedPeerRecord = createSignedPeerRecord function createDatastore () { if (!isNode) { - const Memory = require('../src/server/datastores/memory') + const Memory = require('../src/datastores/memory') return new Memory() } - const MySql = require('../src/server/datastores/mysql') + const MySql = require('../src/datastores/mysql') const datastore = new MySql({ host: 'localhost', user: 'root', From 5f025d3b198aa8f37e0cf2f65f3149b7297f72f1 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Mon, 11 Jan 2021 19:36:45 +0100 Subject: [PATCH 37/38] chore: use connection pool --- src/datastores/mysql.js | 37 ++++++++++++++++++++----------------- test/server.spec.js | 3 --- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/datastores/mysql.js b/src/datastores/mysql.js index ea80c6d..8421013 100644 --- a/src/datastores/mysql.js +++ b/src/datastores/mysql.js @@ -22,6 +22,7 @@ const pRetry = require('p-retry') * @param {string} user * @param {string} password * @param {string} database + * @param {number} [connectionLimit = 20] * @param {boolean} [insecureAuth = true] * @param {boolean} [multipleStatements = true] */ @@ -35,12 +36,13 @@ class Mysql { * * @param {MySqlOptions} options */ - constructor ({ host, user, password, database, insecureAuth = true, multipleStatements = true }) { + constructor ({ host, user, password, database, connectionLimit = 20, insecureAuth = true, multipleStatements = true }) { this.options = { host, user, password, database, + connectionLimit, insecureAuth, multipleStatements } @@ -67,12 +69,12 @@ class Mysql { * Closes Database connection */ stop () { - this.conn.end() + this.pool.end() } async reset () { await new Promise((resolve, reject) => { - this.conn.query(` + this.pool.query(` DROP TABLE IF EXISTS cookie; DROP TABLE IF EXISTS registration; `, (err) => { @@ -91,7 +93,7 @@ class Mysql { */ gc () { return new Promise((resolve, reject) => { - this.conn.query('DELETE FROM registration WHERE expiration <= NOW()', + this.pool.query('DELETE FROM registration WHERE expiration <= UNIX_TIMESTAMP(NOW())', (err, res) => { if (err) { return reject(err) @@ -119,12 +121,12 @@ class Mysql { this._registeringPeer.set(id, peerOps) return new Promise((resolve, reject) => { - this.conn.query('INSERT INTO ?? SET ?', + this.pool.query('INSERT INTO ?? SET ?', ['registration', { namespace, peer_id: id, signed_peer_record: Buffer.from(signedPeerRecord), - expiration: new Date(Date.now() + ttl) + expiration: (Date.now() + ttl) / 1000 // Epoch in seconds like MySQL }], (err) => { // Remove Operation peerOps.delete(opId) @@ -153,7 +155,7 @@ class Mysql { async getRegistrations (namespace, { limit = 10, cookie } = {}) { if (cookie) { const cookieEntries = await new Promise((resolve, reject) => { - this.conn.query( + this.pool.query( 'SELECT * FROM cookie WHERE id = ? LIMIT 1', [cookie], (err, results) => { @@ -179,9 +181,9 @@ class Mysql { } const results = await new Promise((resolve, reject) => { - this.conn.query( + this.pool.query( `SELECT id, namespace, peer_id, signed_peer_record, expiration FROM registration r - WHERE namespace = ? AND expiration >= NOW() ${cookieWhereNotExists()} + WHERE namespace = ? AND expiration >= UNIX_TIMESTAMP(NOW()) ${cookieWhereNotExists()} ORDER BY expiration DESC LIMIT ?`, [namespace, cookie || limit, limit], @@ -205,14 +207,15 @@ class Mysql { // Store in cookies if results available await new Promise((resolve, reject) => { - this.conn.query( + this.pool.query( `INSERT INTO ?? (id, namespace, reg_id) VALUES ${results.map((entry) => - `(${this.conn.escape(cookie)}, ${this.conn.escape(entry.namespace)}, ${this.conn.escape(entry.id)})` + `(${this.pool.escape(cookie)}, ${this.pool.escape(entry.namespace)}, ${this.pool.escape(entry.id)})` )}`, ['cookie'] , (err) => { if (err) { return reject(err) } + // @ts-ignore resolve() }) }) @@ -238,7 +241,7 @@ class Mysql { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query('SELECT COUNT(1) FROM registration WHERE peer_id = ?', + this.pool.query('SELECT COUNT(1) FROM registration WHERE peer_id = ?', [id], (err, res) => { if (err) { @@ -275,7 +278,7 @@ class Mysql { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query('DELETE FROM registration WHERE peer_id = ? AND namespace = ?', [id, ns], + this.pool.query('DELETE FROM registration WHERE peer_id = ? AND namespace = ?', [id, ns], (err, res) => { if (err) { return reject(err) @@ -295,7 +298,7 @@ class Mysql { const id = peerId.toB58String() return new Promise((resolve, reject) => { - this.conn.query('DELETE FROM registration WHERE peer_id = ?', [id], + this.pool.query('DELETE FROM registration WHERE peer_id = ?', [id], (err, res) => { if (err) { return reject(err) @@ -311,16 +314,16 @@ class Mysql { * @returns {Promise} */ _initDB () { - this.conn = mysql.createConnection(this.options) + this.pool = mysql.createPool(this.options) return new Promise((resolve, reject) => { - this.conn.query(` + this.pool.query(` CREATE TABLE IF NOT EXISTS registration ( id INT UNSIGNED NOT NULL AUTO_INCREMENT, namespace varchar(255) NOT NULL, peer_id varchar(255) NOT NULL, signed_peer_record blob NOT NULL, - expiration timestamp NOT NULL, + expiration BIGINT NOT NULL, PRIMARY KEY (id), INDEX (namespace, expiration, peer_id) ); diff --git a/test/server.spec.js b/test/server.spec.js index b328cf9..680856b 100644 --- a/test/server.spec.js +++ b/test/server.spec.js @@ -329,9 +329,6 @@ describe('rendezvous server', () => { // wait for firt record to be removed (2nd gc) await pWaitFor(() => spy.callCount >= 2) - r = await rServer.getRegistrations(testNamespace) - expect(r.registrations).to.have.lengthOf(1) - // wait for second record to be removed await pRetry(async () => { r = await rServer.getRegistrations(testNamespace) From 28402510b91a8098e222e71d6f8c151f830d4f50 Mon Sep 17 00:00:00 2001 From: Vasco Santos Date: Fri, 15 Jan 2021 16:09:31 +0100 Subject: [PATCH 38/38] fix: bin stdout addresses and ports correctly --- src/bin.js | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/bin.js b/src/bin.js index d18432b..966f0e5 100644 --- a/src/bin.js +++ b/src/bin.js @@ -77,8 +77,13 @@ async function main () { }, { datastore }) await rendezvousServer.start() - console.log('Rendezvous server listening on:') - rendezvousServer.multiaddrs.forEach((m) => console.log(m)) + + rendezvousServer.peerStore.on('change:multiaddrs', ({ peerId, multiaddrs }) => { + console.log('Rendezvous server listening on:') + if (peerId.equals(rendezvousServer.peerId)) { + multiaddrs.forEach((m) => console.log(`${m}/p2p/${peerId.toB58String()}`)) + } + }) if (metrics) { log('enabling metrics') @@ -92,7 +97,7 @@ async function main () { menoetius.instrument(metricsServer) metricsServer.listen(metricsPort, '0.0.0.0', () => { - console.log(`metrics server listening on ${metricsPort.port}`) + console.log(`metrics server listening on ${metricsPort}`) }) }