Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: provide default libp2p instance #127

Merged
merged 3 commits into from
May 19, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 22 additions & 9 deletions packages/helia/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -138,18 +138,30 @@
"release": "aegir release"
},
"dependencies": {
"@chainsafe/libp2p-gossipsub": "^8.0.0",
"@chainsafe/libp2p-noise": "^12.0.0",
"@chainsafe/libp2p-yamux": "^4.0.2",
"@helia/interface": "^1.0.0",
"@ipld/dag-pb": "^4.0.2",
"@libp2p/interface-libp2p": "^1.1.0",
"@libp2p/interfaces": "^3.3.1",
"@libp2p/logger": "^2.0.5",
"@ipld/dag-pb": "^4.0.3",
"@libp2p/bootstrap": "^8.0.0",
"@libp2p/interface-libp2p": "^3.1.0",
"@libp2p/interface-pubsub": "^4.0.1",
"@libp2p/interfaces": "^3.3.2",
"@libp2p/ipni-content-routing": "^1.0.0",
"@libp2p/kad-dht": "^9.3.3",
"@libp2p/logger": "^2.0.7",
"@libp2p/mdns": "^8.0.0",
"@libp2p/mplex": "^8.0.3",
"@libp2p/tcp": "^7.0.1",
"@libp2p/webrtc": "^2.0.4",
"@libp2p/webtransport": "^2.0.1",
"blockstore-core": "^4.0.0",
"cborg": "^1.10.0",
"datastore-core": "^9.0.0",
"interface-blockstore": "^5.0.0",
"interface-datastore": "^8.0.0",
"interface-store": "^5.0.1",
"ipfs-bitswap": "^17.0.0",
"ipfs-bitswap": "^18.0.0",
"it-all": "^3.0.1",
"it-drain": "^3.0.1",
"it-filter": "^3.0.1",
Expand All @@ -162,18 +174,19 @@
"uint8arrays": "^4.0.3"
},
"devDependencies": {
"@chainsafe/libp2p-noise": "^11.0.0",
"@chainsafe/libp2p-yamux": "^3.0.5",
"@ipld/dag-cbor": "^9.0.0",
"@ipld/dag-json": "^10.0.1",
"@libp2p/websockets": "^5.0.3",
"@libp2p/websockets": "^6.0.1",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

should this be "optional" dependencies?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same for others that are browser only?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No, optional dependencies means "if this dependency failed to install, carry on anyway".

I think what you are saying is "let the user decide whether to install this dep or not", which is what "peerDependencies" used to mean, now npm installs peer deps whether you like it or not so I'm not sure there's a way to do that.

"@types/sinon": "^10.0.14",
"aegir": "^39.0.4",
"delay": "^5.0.0",
"libp2p": "^0.44.0",
"libp2p": "^0.45.1",
"sinon": "^15.0.2",
"sinon-ts": "^1.0.0"
},
"browser": {
"./dist/src/utils/libp2p.js": "./dist/src/utils/libp2p.browser.js"
},
"typedoc": {
"entryPoint": "./src/index.ts"
}
Expand Down
65 changes: 22 additions & 43 deletions packages/helia/src/helia.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,4 @@
import { logger } from '@libp2p/logger'
import { MemoryBlockstore } from 'blockstore-core'
import { MemoryDatastore } from 'datastore-core'
import { type Bitswap, createBitswap } from 'ipfs-bitswap'
import drain from 'it-drain'
import { identity } from 'multiformats/hashes/identity'
Expand All @@ -13,12 +11,19 @@ import type { HeliaInit } from '.'
import type { GCOptions, Helia } from '@helia/interface'
import type { Pins } from '@helia/interface/pins'
import type { Libp2p } from '@libp2p/interface-libp2p'
import type { Blockstore } from 'interface-blockstore'
import type { Datastore } from 'interface-datastore'
import type { CID } from 'multiformats/cid'
import type { MultihashHasher } from 'multiformats/hashes/interface'

const log = logger('helia')

interface HeliaImplInit<T extends Libp2p = Libp2p> extends HeliaInit<T> {
libp2p: T
blockstore: Blockstore
datastore: Datastore
}

export class HeliaImpl implements Helia {
public libp2p: Libp2p
public blockstore: BlockStorage
Expand All @@ -27,64 +32,38 @@ export class HeliaImpl implements Helia {

#bitswap?: Bitswap

constructor (init: HeliaInit) {
constructor (init: HeliaImplInit) {
const hashers: MultihashHasher[] = [
sha256,
sha512,
identity,
...(init.hashers ?? [])
]

const datastore = init.datastore ?? new MemoryDatastore()
const blockstore = init.blockstore ?? new MemoryBlockstore()
this.#bitswap = createBitswap(init.libp2p, init.blockstore, {
hashLoader: {
getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => {
const hasher = hashers.find(hasher => {
return hasher.code === codecOrName || hasher.name === codecOrName
})

// @ts-expect-error incomplete libp2p implementation
const libp2p = init.libp2p ?? new Proxy<Libp2p>({}, {
get (_, prop): true | (() => void) {
const noop = (): void => {}
const noops = ['start', 'stop']

if (noops.includes(prop.toString())) {
return noop
}
if (hasher != null) {
return hasher
}

if (prop === 'isProxy') {
return true
throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
}

throw new Error('Please configure Helia with a libp2p instance')
},
set (): never {
throw new Error('Please configure Helia with a libp2p instance')
}
})

if (init.libp2p != null) {
this.#bitswap = createBitswap(libp2p, blockstore, {
hashLoader: {
getHasher: async (codecOrName: string | number): Promise<MultihashHasher<number>> => {
const hasher = hashers.find(hasher => {
return hasher.code === codecOrName || hasher.name === codecOrName
})

if (hasher != null) {
return hasher
}

throw new Error(`Could not load hasher for code/name "${codecOrName}"`)
}
}
})
}

this.pins = new PinsImpl(datastore, blockstore, init.dagWalkers ?? [])
this.pins = new PinsImpl(init.datastore, init.blockstore, init.dagWalkers ?? [])

this.libp2p = libp2p
this.blockstore = new BlockStorage(blockstore, this.pins, {
this.libp2p = init.libp2p
this.blockstore = new BlockStorage(init.blockstore, this.pins, {
bitswap: this.#bitswap,
holdGcLock: init.holdGcLock
})
this.datastore = datastore
this.datastore = init.datastore
}

async start (): Promise<void> {
Expand Down
33 changes: 24 additions & 9 deletions packages/helia/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
* @example
*
* ```typescript
* import { createLibp2p } from 'libp2p'
* import { MemoryDatastore } from 'datastore-core'
* import { MemoryBlockstore } from 'blockstore-core'
* import { createHelia } from 'helia'
Expand All @@ -15,19 +14,21 @@
*
* const node = await createHelia({
* blockstore: new MemoryBlockstore(),
* datastore: new MemoryDatastore(),
* libp2p: await createLibp2p({
* //... libp2p options
* })
* datastore: new MemoryDatastore()
* })
* const fs = unixfs(node)
* fs.cat(CID.parse('bafyFoo'))
* ```
*/

import { MemoryBlockstore } from 'blockstore-core'
import { MemoryDatastore } from 'datastore-core'
import { HeliaImpl } from './helia.js'
import { createLibp2p } from './utils/libp2p.js'
import type { Helia } from '@helia/interface'
import type { Libp2p } from '@libp2p/interface-libp2p'
import type { PubSub } from '@libp2p/interface-pubsub'
import type { DualKadDHT } from '@libp2p/kad-dht'
import type { Blockstore } from 'interface-blockstore'
import type { Datastore } from 'interface-datastore'
import type { CID } from 'multiformats/cid'
Expand All @@ -44,11 +45,11 @@ export interface DAGWalker {
/**
* Options used to create a Helia node.
*/
export interface HeliaInit {
export interface HeliaInit<T extends Libp2p = Libp2p> {
/**
* A libp2p node is required to perform network operations
*/
libp2p?: Libp2p
libp2p?: T

/**
* The blockstore is where blocks are stored
Expand Down Expand Up @@ -100,8 +101,22 @@ export interface HeliaInit {
/**
* Create and return a Helia node
*/
export async function createHelia (init: HeliaInit = {}): Promise<Helia> {
const helia = new HeliaImpl(init)
export async function createHelia <T extends Libp2p> (init: HeliaInit<T>): Promise<Helia<T>>
export async function createHelia (init?: HeliaInit<Libp2p<{ dht: DualKadDHT, pubsub: PubSub }>>): Promise<Helia<Libp2p<{ dht: DualKadDHT, pubsub: PubSub }>>>
export async function createHelia (init: HeliaInit = {}): Promise<Helia<unknown>> {
const datastore = init.datastore ?? new MemoryDatastore()
const blockstore = init.blockstore ?? new MemoryBlockstore()
const libp2p = init.libp2p ?? await createLibp2p({
datastore,
start: false
})

const helia = new HeliaImpl({
...init,
datastore,
blockstore,
libp2p
})

if (init.start !== false) {
await helia.start()
Expand Down
68 changes: 68 additions & 0 deletions packages/helia/src/utils/libp2p.browser.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
import { gossipsub } from '@chainsafe/libp2p-gossipsub'
import { noise } from '@chainsafe/libp2p-noise'
import { yamux } from '@chainsafe/libp2p-yamux'
import { bootstrap } from '@libp2p/bootstrap'
import { ipniContentRouting } from '@libp2p/ipni-content-routing'
import { kadDHT, type DualKadDHT } from '@libp2p/kad-dht'
import { mplex } from '@libp2p/mplex'
import { webRTC, webRTCDirect } from '@libp2p/webrtc'
import { webSockets } from '@libp2p/websockets'
import { webTransport } from '@libp2p/webtransport'
import { createLibp2p as create } from 'libp2p'
import { autoNATService } from 'libp2p/autonat'
import { circuitRelayTransport, circuitRelayServer } from 'libp2p/circuit-relay'
import { identifyService } from 'libp2p/identify'
import type { CreateLibp2pOptions } from './libp2p.js'
import type { Libp2p } from '@libp2p/interface-libp2p'
import type { PubSub } from '@libp2p/interface-pubsub'

export async function createLibp2p (opts: CreateLibp2pOptions): Promise<Libp2p<{ dht: DualKadDHT, pubsub: PubSub }>> {
return create({
...opts,
addresses: {
listen: [
'/webrtc'
]
},
transports: [
webRTC(),
webRTCDirect(),
webTransport(),
webSockets(),
circuitRelayTransport({
discoverRelays: 1
})
],
connectionEncryption: [
noise()
],
streamMuxers: [
yamux(),
mplex()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we kill off mplex?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe - do we have stats for how well used yamux is yet?

js-ipfs never shipped with it included so you'd lose compatibility with any existing deployments.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

actually, I was wanting to make sure we don't proliferate mplex, but as long as we prefer yamux that is the key thing (which you're doing). I'm not aware of harm of one also offering mplex, and assuming there are no downsides to it, I'm game to include it for compatibility.

That said, if go-libp2p ships without mplex by default, I think we'd be fine to remove it. I don't have stats here and I don't think we need to block to find that out.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Muxers are negotiated in-order so yamux will be preferred over mplex by any peer that supports both.

],
peerDiscovery: [
bootstrap({
list: [
'/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt'
]
})
],
contentRouters: [
ipniContentRouting('https://cid.contact')
],
services: {
identify: identifyService(),
autoNAT: autoNATService(),
pubsub: gossipsub(),
dht: kadDHT({
clientMode: true
}),
relay: circuitRelayServer({
advertise: true
})
}
})
}
72 changes: 72 additions & 0 deletions packages/helia/src/utils/libp2p.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { gossipsub } from '@chainsafe/libp2p-gossipsub'
import { noise } from '@chainsafe/libp2p-noise'
import { yamux } from '@chainsafe/libp2p-yamux'
import { bootstrap } from '@libp2p/bootstrap'
import { ipniContentRouting } from '@libp2p/ipni-content-routing'
import { type DualKadDHT, kadDHT } from '@libp2p/kad-dht'
import { mdns } from '@libp2p/mdns'
import { mplex } from '@libp2p/mplex'
import { tcp } from '@libp2p/tcp'
import { webSockets } from '@libp2p/websockets'
import { createLibp2p as create } from 'libp2p'
import { autoNATService } from 'libp2p/autonat'
import { circuitRelayTransport, circuitRelayServer, type CircuitRelayService } from 'libp2p/circuit-relay'
import { identifyService } from 'libp2p/identify'
import { uPnPNATService } from 'libp2p/upnp-nat'
import type { Libp2p } from '@libp2p/interface-libp2p'
import type { PubSub } from '@libp2p/interface-pubsub'
import type { Datastore } from 'interface-datastore'

export interface CreateLibp2pOptions {
datastore: Datastore
start?: boolean
}

export async function createLibp2p (opts: CreateLibp2pOptions): Promise<Libp2p<{ dht: DualKadDHT, pubsub: PubSub, relay: CircuitRelayService }>> {
return create({
...opts,
addresses: {
listen: [
'/ip4/0.0.0.0/tcp/0'
]
},
transports: [
tcp(),
webSockets(),
circuitRelayTransport({
discoverRelays: 1
})
],
connectionEncryption: [
noise()
],
streamMuxers: [
yamux(),
mplex()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we remove?

],
peerDiscovery: [
mdns(),
bootstrap({
list: [
'/dnsaddr/bootstrap.libp2p.io/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb',
'/dnsaddr/bootstrap.libp2p.io/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt'
]
Comment on lines +50 to +55
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is duplicated. Maybe factor it out, and also add comments on where this list comes from?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

})
],
contentRouters: [
ipniContentRouting('https://cid.contact')
],
services: {
identify: identifyService(),
autoNAT: autoNATService(),
upnp: uPnPNATService(),
pubsub: gossipsub(),
dht: kadDHT(),
relay: circuitRelayServer({
advertise: true
})
}
})
}
Loading