From a38c875ee45baa5362764e1d980b6978eee74d75 Mon Sep 17 00:00:00 2001 From: Mykhailo Marynenko <0x77dev@protonmail.com> Date: Sat, 26 Mar 2022 06:59:59 +0100 Subject: [PATCH] feat(relay, lib): add stack to relay and store/shard improvements (#86) * feat(relay, lib): add stack to relay and store/shard improvements #26; #64 * chore(relay): move wrtc to dependencies --- packages/explorer/src/main.tsx | 5 +- packages/ipfs/package.json | 2 +- packages/ipfs/src/bootstrap.ts | 7 +- packages/ipfs/src/index.ts | 9 +- packages/ipfs/tsconfig.json | 2 +- packages/lib/src/stack.ts | 37 +++-- packages/lib/src/storage.ts | 12 +- packages/lib/src/store.ts | 22 ++- packages/relay/package.json | 4 + .../relay/src/services/signaling/index.ts | 11 +- packages/relay/src/services/stack.ts | 88 ++++++++++++ packages/relay/src/types.ts | 17 ++- yarn.lock | 132 ++++++++++++++++-- 13 files changed, 311 insertions(+), 37 deletions(-) create mode 100644 packages/relay/src/services/stack.ts diff --git a/packages/explorer/src/main.tsx b/packages/explorer/src/main.tsx index 0bb0cea..03129be 100644 --- a/packages/explorer/src/main.tsx +++ b/packages/explorer/src/main.tsx @@ -20,7 +20,10 @@ export const App: React.FC = () => { let stop = () => {} const run = async () => { - const stack = await Stack.create({ namespace }) + const stack = await Stack.create({ + namespace, + relay: localStorage['relay'] + }) window.stack = stack window.Shard = Shard window.ShardKind = ShardKind diff --git a/packages/ipfs/package.json b/packages/ipfs/package.json index 1ea92ed..9959579 100644 --- a/packages/ipfs/package.json +++ b/packages/ipfs/package.json @@ -9,6 +9,6 @@ "license": "GPL-3.0", "name": "@dstack-js/ipfs", "repository": "https://github.com/dstack-js/dstack.git", - "type": "module", + "type": "commonjs", "version": "0.2.47" } diff --git a/packages/ipfs/src/bootstrap.ts b/packages/ipfs/src/bootstrap.ts index 9edd5e3..faaf3c6 100644 --- a/packages/ipfs/src/bootstrap.ts +++ b/packages/ipfs/src/bootstrap.ts @@ -9,7 +9,12 @@ export const bootstrap = async ( const query = gql` query Bootstrap($protocol: Protocol!, $hostname: String!, $port: Int!) { listen(protocol: $protocol, hostname: $hostname, port: $port) - peers(randomize: true) + peers( + randomize: true + protocol: $protocol + hostname: $hostname + port: $port + ) } ` diff --git a/packages/ipfs/src/index.ts b/packages/ipfs/src/index.ts index 4c23ff1..1831455 100644 --- a/packages/ipfs/src/index.ts +++ b/packages/ipfs/src/index.ts @@ -3,12 +3,14 @@ import { bootstrap } from './bootstrap' // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-ignore import WebRTCStar from '@dstack-js/transport' +import type { IPFSOptions } from 'ipfs-core/src/components/storage' export interface Options { namespace: string; relay?: string; wrtc?: any; privateKey?: string; + repo?: IPFSOptions['repo']; } export { CID, PeerId } @@ -17,12 +19,14 @@ export const create = async ({ namespace, relay, wrtc, - privateKey + privateKey, + repo }: Options): Promise => { const { listen, peers } = await bootstrap(namespace, relay) return IPFSCreate({ init: { privateKey }, + repo, config: { Discovery: { webRTCStar: { Enabled: true } @@ -57,7 +61,8 @@ export const create = async ({ relay: { enabled: true, hop: { - enabled: true + enabled: true, + active: true } } }) diff --git a/packages/ipfs/tsconfig.json b/packages/ipfs/tsconfig.json index 37114dc..e471db8 100644 --- a/packages/ipfs/tsconfig.json +++ b/packages/ipfs/tsconfig.json @@ -1,7 +1,7 @@ { "extends": "../../tsconfig.base.json", "compilerOptions": { - "module": "ESNext", + "module": "CommonJS", "forceConsistentCasingInFileNames": true, "strict": true, "noImplicitOverride": true, diff --git a/packages/lib/src/stack.ts b/packages/lib/src/stack.ts index fd47cc0..b731dc5 100644 --- a/packages/lib/src/stack.ts +++ b/packages/lib/src/stack.ts @@ -4,8 +4,8 @@ import all from 'it-all' import type { CID } from 'ipfs-core' import { PeerUnreachableError, Store } from '.' import { PubSub } from './pubsub' -import { Storage } from './storage' -import { create } from '@dstack-js/ipfs' +import { InMemoryStorage, Storage } from './storage' +import { create, Options } from '@dstack-js/ipfs' export interface Peer { id: string; @@ -33,19 +33,29 @@ export interface StackOptions { * * No need to provide it unless you want to use DStack in non browser environment */ - wrtc?: any; + wrtc?: Options['wrtc']; /** * Relay GraphQL Endpoint * * Defaults to DStack Cloud */ - relay?: string; + relay?: Options['relay']; /** * Storage implementation * * No need to provide it unless you want custom storage implementation to be used */ storage?: Storage; + /** + * A path to store IPFS repo + * + * No need to provide it unless you to create a more than one Stack instance + */ + repo?: Options['repo']; + /** + * Preload shard on store replication + */ + loadOnReplicate?: boolean; } export class Stack { @@ -55,9 +65,14 @@ export class Stack { private announceInterval?: ReturnType public announce = true - constructor(public namespace: CID, public ipfs: IPFS, storage: Storage) { + private constructor( + public namespace: CID, + public ipfs: IPFS, + storage: Storage, + loadOnReplicate?: boolean + ) { this.pubsub = new PubSub(ipfs, namespace.toString()) - this.store = new Store(this, storage) + this.store = new Store(this, storage, loadOnReplicate) } /** @@ -100,16 +115,18 @@ export class Stack { ipfs, storage, relay, - wrtc + wrtc, + repo, + loadOnReplicate }: StackOptions) { if (!ipfs) { - ipfs = await create({ namespace, relay, wrtc }) + ipfs = await create({ namespace, relay, wrtc, repo }) } const cid = await ipfs.dag.put({ namespace }) - storage = storage || new Storage(namespace) + storage = storage || new InMemoryStorage(namespace) - const stack = new Stack(cid, ipfs, storage) + const stack = new Stack(cid, ipfs, storage, loadOnReplicate) await stack.start() return stack diff --git a/packages/lib/src/storage.ts b/packages/lib/src/storage.ts index 56b3b0a..4ef3d17 100644 --- a/packages/lib/src/storage.ts +++ b/packages/lib/src/storage.ts @@ -1,4 +1,14 @@ -export class Storage { +/* eslint-disable @typescript-eslint/no-misused-new */ +export interface Storage { + namespace: string; + + keys(): Promise; + get(key: string): Promise; + set(key: string, value: T): Promise; +} + +export class InMemoryStorage +implements Storage { private data: { [key: string]: T } = {} constructor(public namespace: string) {} diff --git a/packages/lib/src/store.ts b/packages/lib/src/store.ts index 7dc79b6..03efa73 100644 --- a/packages/lib/src/store.ts +++ b/packages/lib/src/store.ts @@ -18,11 +18,13 @@ export type StackMessage = ReplicateMessage | ReplicateRequestMessage; export class Store { private pubsub: PubSub - private storage: Storage - constructor(private stack: Stack, storage: Storage) { + constructor( + private stack: Stack, + private storage: Storage, + private loadOnReplicate: boolean = false + ) { this.pubsub = stack.pubsub.create('$$store') - this.storage = storage } private async onReplicate(msg: Message): Promise { @@ -37,10 +39,22 @@ export class Store { value: msg.data.value, date }) + + if (this.loadOnReplicate && msg.data.value.startsWith('/shard/')) { + Shard.from(this.stack, msg.data.value) + .then((shard) => { + console.debug('shard replicated', shard) + }) + .catch((err) => { + console.warn('Failed to load on replicate', msg.data, err) + }) + } } private watchShard(key: string, watch: Shard) { watch.on('update', async (shard): Promise => { + console.log(shard.toString()) + this.storage.set(key, { value: shard.toString(), date: new Date() @@ -93,7 +107,7 @@ export class Store { await this.pubsub.subscribe('replicateRequest', async (msg) => { if (msg.data.kind !== 'replicateRequest') return const keys = await this.storage.keys() - await Promise.all(keys.map(this.replicate)) + await Promise.all(keys.map((key) => this.replicate(key))) }) await this.pubsub.publish('replicateRequest', { kind: 'replicateRequest' }) diff --git a/packages/relay/package.json b/packages/relay/package.json index 3d08f77..a46435f 100644 --- a/packages/relay/package.json +++ b/packages/relay/package.json @@ -3,12 +3,15 @@ "dstack-relay": "src/index.js" }, "dependencies": { + "@dstack-js/lib": "latest", + "@dstack-js/wrtc": "^0.4.8", "@libp2p/webrtc-star-protocol": "1.0.1", "fastify": "3.27.4", "fastify-cors": "6.0.3", "fastify-socket.io": "3.0.0", "graphql": "16.3.0", "graphql-playground-html": "1.6.30", + "lru-cache": "^7.7.1", "mercurius": "9.3.6", "nexus": "1.3.0", "prom-client": "14.0.1", @@ -16,6 +19,7 @@ "socket.io": "4.4.1" }, "devDependencies": { + "@types/lru-cache": "^7.6.0", "ts-node": "10.7.0", "tsconfig-paths": "3.14.1" }, diff --git a/packages/relay/src/services/signaling/index.ts b/packages/relay/src/services/signaling/index.ts index 3d88958..ac93020 100644 --- a/packages/relay/src/services/signaling/index.ts +++ b/packages/relay/src/services/signaling/index.ts @@ -15,6 +15,7 @@ import { joinsFailureTotal, joinsTotal } from '../metrics' +import { getStack } from '../stack' const handle = (socket: WebRTCStarSocket, cachePrefix: string) => { console.log('signaling', 'handle', socket.id) @@ -104,9 +105,13 @@ export const setSocket = async (server: FastifyInstance) => { await server.ready() server.io.on('connection', (socket) => { - const cachePrefix = socket.handshake.auth['namespace'] - ? socket.handshake.auth['namespace'] - : '' + let cachePrefix = '' + + if (socket.handshake.auth['namespace']) { + cachePrefix = socket.handshake.auth['namespace'] + + getStack(socket.handshake.auth['namespace']).catch() + } // @ts-expect-error: incompatible types return handle(socket, cachePrefix) diff --git a/packages/relay/src/services/stack.ts b/packages/relay/src/services/stack.ts new file mode 100644 index 0000000..bbb91ba --- /dev/null +++ b/packages/relay/src/services/stack.ts @@ -0,0 +1,88 @@ +import { join } from 'path' +import { Stack, Storage } from '@dstack-js/lib' +// @ts-expect-error: no-types +import wrtc from '@dstack-js/wrtc' +import LRUCache from 'lru-cache' +import { tmpdir } from 'os' +import { redis } from './cache' + +export class RedisStorage +implements Storage { + private prefix: string + + constructor(public namespace: string) { + this.prefix = `s!${this.namespace}#` + } + + public async set(key: string, value: T): Promise { + await redis.set(`${this.prefix}${key}`, JSON.stringify(value)) + redis.expire(`${this.prefix}${key}`, 3600) + } + + public async get(key: string): Promise { + const data = await redis.get(key) + if (!data) return null + + return JSON.parse(data) + } + + public async keys(): Promise { + const keys = await redis.keys(`${this.prefix}*`) + return keys.map((key) => key.replace(this.prefix, '')) + } +} + +const cache = new LRUCache({ + ttl: 60000, + ttlAutopurge: true, + updateAgeOnGet: true, + dispose: async (stack, namespace) => { + if (stack === 'allocating') return + + await stack + .stop() + .then(() => { + console.log(namespace, 'stack deallocated') + }) + .catch((error) => { + console.warn( + 'error happened while deallocation stack in', + namespace, + error + ) + }) + } +}) + +export const getStack = async (namespace: string) => { + let stack = cache.get(namespace) + + if (!stack) { + try { + cache.set(namespace, 'allocating') + stack = await Stack.create({ + namespace, + wrtc, + relay: `http://127.0.0.1:${process.env['PORT'] || 13579}/graphql`, + repo: join(tmpdir(), '.dstack', namespace), + storage: new RedisStorage(namespace) + }) + + cache.set(namespace, stack) + console.log(namespace, 'stack allocated') + + return stack + } catch (error) { + console.warn( + 'error happened while allocation stack in', + namespace, + error + ) + throw error + } + } + + if (stack === 'allocating') return null + + return stack +} diff --git a/packages/relay/src/types.ts b/packages/relay/src/types.ts index 1dc0e62..25c81fe 100644 --- a/packages/relay/src/types.ts +++ b/packages/relay/src/types.ts @@ -39,10 +39,21 @@ const peers = queryField('peers', { description: 'Get addresses to bootstrap for IPFS, use `randomize` argument to get 3 random peers', args: { - randomize: booleanArg() + randomize: booleanArg(), + protocol: Protocol.asArg(), + hostname: stringArg(), + port: intArg() }, - async resolve(_, { randomize }, { namespace }) { - const peers = await getPeers(namespace) + async resolve(_, { randomize, protocol, hostname, port }, { namespace }) { + let peers = await getPeers(namespace || '') + + if (protocol && hostname && port) { + peers = peers.map((peer) => { + return getListenAddress(protocol, hostname, port).concat( + peer.split('/').slice(-2).join('/') + ) + }) + } if (randomize) { return peers diff --git a/yarn.lock b/yarn.lock index 9b5d1d3..a5272be 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2725,6 +2725,25 @@ __metadata: languageName: unknown linkType: soft +"@dstack-js/lib@npm:latest": + version: 0.2.47 + resolution: "@dstack-js/lib@npm:0.2.47" + dependencies: + "@dstack-js/ipfs": ^0.2.45 + buffer: 6.0.3 + events: 3.3.0 + it-all: 1.0.6 + it-drain: 1.0.5 + it-last: 1.0.6 + lru-cache: 7.7.1 + tslib: 2.3.1 + uuid: 8.3.2 + peerDependencies: + "@dstack-js/transport": 0.2.47 + checksum: 5ad4e01b6549f257d2372f4adb260b1520653f08e874386dde36d024ab4445fca6c022dccf295dadb6af3d957b515d46282f817217493f4f4c5929b4322381c3 + languageName: node + linkType: hard + "@dstack-js/lib@workspace:packages/lib": version: 0.0.0-use.local resolution: "@dstack-js/lib@workspace:packages/lib" @@ -2749,12 +2768,16 @@ __metadata: version: 0.0.0-use.local resolution: "@dstack-js/relay@workspace:packages/relay" dependencies: + "@dstack-js/lib": latest + "@dstack-js/wrtc": ^0.4.8 "@libp2p/webrtc-star-protocol": 1.0.1 + "@types/lru-cache": ^7.6.0 fastify: 3.27.4 fastify-cors: 6.0.3 fastify-socket.io: 3.0.0 graphql: 16.3.0 graphql-playground-html: 1.6.30 + lru-cache: ^7.7.1 mercurius: 9.3.6 nexus: 1.3.0 prom-client: 14.0.1 @@ -2797,6 +2820,19 @@ __metadata: languageName: unknown linkType: soft +"@dstack-js/wrtc@npm:^0.4.8": + version: 0.4.8 + resolution: "@dstack-js/wrtc@npm:0.4.8" + dependencies: + domexception: ^1.0.1 + node-pre-gyp: ^0.13.0 + dependenciesMeta: + domexception: + optional: true + checksum: 2cfab7ec0793a26969a33f2c745ffd243de6c0e2a44ec68ab43f7d1a0a5ba67f1982f95c9a70d87f8a91e35c7961cb83b680fbd1caf7a9fe223b9bc28e8e204e + languageName: node + linkType: hard + "@emotion/babel-plugin@npm:11.7.2, @emotion/babel-plugin@npm:^11.7.1": version: 11.7.2 resolution: "@emotion/babel-plugin@npm:11.7.2" @@ -6829,6 +6865,13 @@ __metadata: languageName: node linkType: hard +"@types/lru-cache@npm:^7.6.0": + version: 7.6.0 + resolution: "@types/lru-cache@npm:7.6.0" + checksum: 8a1985375d627409dae95945fce4c45001c47980a55d6adea0a212d993963e35fee2af6b06161f01cfdfaeb1fdd9e4385c8d0853a9d90c8bb3cf21d1f21a8ac7 + languageName: node + linkType: hard + "@types/mdast@npm:^3.0.0": version: 3.0.10 resolution: "@types/mdast@npm:3.0.10" @@ -11312,7 +11355,7 @@ __metadata: languageName: node linkType: hard -"debug@npm:^3.1.0, debug@npm:^3.1.1, debug@npm:^3.2.7": +"debug@npm:^3.1.0, debug@npm:^3.1.1, debug@npm:^3.2.6, debug@npm:^3.2.7": version: 3.2.7 resolution: "debug@npm:3.2.7" dependencies: @@ -11561,6 +11604,15 @@ __metadata: languageName: node linkType: hard +"detect-libc@npm:^1.0.2": + version: 1.0.3 + resolution: "detect-libc@npm:1.0.3" + bin: + detect-libc: ./bin/detect-libc.js + checksum: daaaed925ffa7889bd91d56e9624e6c8033911bb60f3a50a74a87500680652969dbaab9526d1e200a4c94acf80fc862a22131841145a0a8482d60a99c24f4a3e + languageName: node + linkType: hard + "detect-newline@npm:^3.0.0, detect-newline@npm:^3.1.0": version: 3.1.0 resolution: "detect-newline@npm:3.1.0" @@ -11830,6 +11882,15 @@ __metadata: languageName: node linkType: hard +"domexception@npm:^1.0.1": + version: 1.0.1 + resolution: "domexception@npm:1.0.1" + dependencies: + webidl-conversions: ^4.0.2 + checksum: f564a9c0915dcb83ceefea49df14aaed106b1468fbe505119e8bcb0b77e242534f3aba861978537c0fc9dc6f35b176d0ffc77b3e342820fb27a8f215e7ae4d52 + languageName: node + linkType: hard + "domexception@npm:^2.0.1": version: 2.0.1 resolution: "domexception@npm:2.0.1" @@ -15102,7 +15163,7 @@ __metadata: languageName: node linkType: hard -"iconv-lite@npm:0.4.24, iconv-lite@npm:^0.4.24": +"iconv-lite@npm:0.4.24, iconv-lite@npm:^0.4.24, iconv-lite@npm:^0.4.4": version: 0.4.24 resolution: "iconv-lite@npm:0.4.24" dependencies: @@ -15152,7 +15213,7 @@ __metadata: languageName: node linkType: hard -"ignore-walk@npm:^3.0.3": +"ignore-walk@npm:^3.0.1, ignore-walk@npm:^3.0.3": version: 3.0.4 resolution: "ignore-walk@npm:3.0.4" dependencies: @@ -19247,7 +19308,7 @@ __metadata: languageName: node linkType: hard -"lru-cache@npm:7.7.1, lru-cache@npm:^7.5.1": +"lru-cache@npm:7.7.1, lru-cache@npm:^7.5.1, lru-cache@npm:^7.7.1": version: 7.7.1 resolution: "lru-cache@npm:7.7.1" checksum: f362c5a2cfa8ad6fe557ec43dc1b7a9695cce84a5652a43ff813609f782f5da576631e7dfad41878bf19a7a69438f38375178635ee80de269aa314280ca2f59e @@ -20317,6 +20378,19 @@ __metadata: languageName: node linkType: hard +"needle@npm:^2.2.1": + version: 2.9.1 + resolution: "needle@npm:2.9.1" + dependencies: + debug: ^3.2.6 + iconv-lite: ^0.4.4 + sax: ^1.2.4 + bin: + needle: ./bin/needle + checksum: 746ae3a3782f0a057ff304a98843cc6f2009f978a0fad0c3e641a9d46d0b5702bb3e197ba08aecd48678067874a991c4f5fc320c7e51a4c041d9dd3441146cf0 + languageName: node + linkType: hard + "negotiator@npm:0.6.3, negotiator@npm:^0.6.2, negotiator@npm:^0.6.3": version: 0.6.3 resolution: "negotiator@npm:0.6.3" @@ -20560,6 +20634,26 @@ __metadata: languageName: node linkType: hard +"node-pre-gyp@npm:^0.13.0": + version: 0.13.0 + resolution: "node-pre-gyp@npm:0.13.0" + dependencies: + detect-libc: ^1.0.2 + mkdirp: ^0.5.1 + needle: ^2.2.1 + nopt: ^4.0.1 + npm-packlist: ^1.1.6 + npmlog: ^4.0.2 + rc: ^1.2.7 + rimraf: ^2.6.1 + semver: ^5.3.0 + tar: ^4 + bin: + node-pre-gyp: ./bin/node-pre-gyp + checksum: 118a8989c2edc5935906a59e9cf8bc508f8183f43da3ceb449bde122901b26bf25aa426481a3b1bed44cb739275e249a8ea85521caa15c33e8377ac0dd31bdbc + languageName: node + linkType: hard + "node-releases@npm:^2.0.1": version: 2.0.1 resolution: "node-releases@npm:2.0.1" @@ -20649,7 +20743,7 @@ __metadata: languageName: node linkType: hard -"npm-bundled@npm:^1.1.1": +"npm-bundled@npm:^1.0.1, npm-bundled@npm:^1.1.1": version: 1.1.2 resolution: "npm-bundled@npm:1.1.2" dependencies: @@ -20701,6 +20795,17 @@ __metadata: languageName: node linkType: hard +"npm-packlist@npm:^1.1.6": + version: 1.4.8 + resolution: "npm-packlist@npm:1.4.8" + dependencies: + ignore-walk: ^3.0.1 + npm-bundled: ^1.0.1 + npm-normalize-package-bin: ^1.0.1 + checksum: 85f764bd0fb516cff34afb4b60ea925ef218cfbdf02d05cda0c115ca30b932b9e0f78bdb186e09d26dd17f983ee1d5aee7ba44b5db84ff3c4c5e73524b537084 + languageName: node + linkType: hard + "npm-packlist@npm:^2.1.4": version: 2.2.2 resolution: "npm-packlist@npm:2.2.2" @@ -20787,7 +20892,7 @@ __metadata: languageName: node linkType: hard -"npmlog@npm:^4.1.2": +"npmlog@npm:^4.0.2, npmlog@npm:^4.1.2": version: 4.1.2 resolution: "npmlog@npm:4.1.2" dependencies: @@ -23346,7 +23451,7 @@ __metadata: languageName: node linkType: hard -"rc@npm:^1.2.8": +"rc@npm:^1.2.7, rc@npm:^1.2.8": version: 1.2.8 resolution: "rc@npm:1.2.8" dependencies: @@ -24352,7 +24457,7 @@ __metadata: languageName: node linkType: hard -"rimraf@npm:^2.6.3": +"rimraf@npm:^2.6.1, rimraf@npm:^2.6.3": version: 2.7.1 resolution: "rimraf@npm:2.7.1" dependencies: @@ -24787,7 +24892,7 @@ __metadata: languageName: node linkType: hard -"semver@npm:2 || 3 || 4 || 5, semver@npm:^5.4.1, semver@npm:^5.5.0, semver@npm:^5.6.0, semver@npm:^5.7.1": +"semver@npm:2 || 3 || 4 || 5, semver@npm:^5.3.0, semver@npm:^5.4.1, semver@npm:^5.5.0, semver@npm:^5.6.0, semver@npm:^5.7.1": version: 5.7.1 resolution: "semver@npm:5.7.1" bin: @@ -26130,7 +26235,7 @@ __metadata: languageName: node linkType: hard -"tar@npm:^4.4.12": +"tar@npm:^4, tar@npm:^4.4.12": version: 4.4.19 resolution: "tar@npm:4.4.19" dependencies: @@ -27651,6 +27756,13 @@ __metadata: languageName: node linkType: hard +"webidl-conversions@npm:^4.0.2": + version: 4.0.2 + resolution: "webidl-conversions@npm:4.0.2" + checksum: c93d8dfe908a0140a4ae9c0ebc87a33805b416a33ee638a605b551523eec94a9632165e54632f6d57a39c5f948c4bab10e0e066525e9a4b87a79f0d04fbca374 + languageName: node + linkType: hard + "webidl-conversions@npm:^5.0.0": version: 5.0.0 resolution: "webidl-conversions@npm:5.0.0"