docs: Add share p2p documentation

blob/service.go

@@ -51,7 +51,8 @@ func NewService(
 	}
 }
 
-// SubmitOptions contains the information about fee and gasLimit price in order to configure the Submit request.
+// SubmitOptions contains the information about fee and gasLimit price in order to configure the
+// Submit request.
 type SubmitOptions struct {
 	Fee      int64
 	GasLimit uint64

share/p2p/doc.go

@@ -0,0 +1,18 @@
+// Package p2p provides p2p functionality that powers the share exchange protocols used by celestia-node.
+// The available protocols are:
+//
+//   - shrexsub : a floodsub-based pubsub protocol that is used to broadcast/subscribe to the event
+//     of new EDS in the network to peers.
+//
+//   - shrexnd: a request/response protocol that is used to request	shares by namespace or namespace data from peers.
+//
+//   - shrexeds: a request/response protocol that is used to request extended data square shares from peers.
+//     This protocol exchanges the original data square in between the client and server, and it's up to the
+//     receiver to compute the extended data square.
+//
+// This package also defines a peer manager that is used to manage network peers that can be used to exchange
+// shares. The peer manager is primarily responsible for providing peers to request shares from,
+// and is primarily used by `getters.ShrexGetter` in share/getters/shrex.go.
+//
+// Find out more about each protocol in their respective sub-packages.
+package p2p

share/p2p/peers/doc.go

@@ -0,0 +1,52 @@
+// Package peers provides a peer manager that handles peer discovery and peer selection for the shrex getter.
+//
+// The peer manager is responsible for:
+//   - Discovering peers
+//   - Selecting peers for data retrieval
+//   - Validating peers
+//   - Blacklisting peers
+//   - Garbage collecting peers
+//
+// The peer manager is not responsible for:
+//   - Connecting to peers
+//   - Disconnecting from peers
+//   - Sending data to peers
+//   - Receiving data from peers
+//
+// The peer manager is a mechanism to store peers from shrexsub, a mechanism that
+// handles "peer discovery" and "peer selection" by relying on a shrexsub subscription
+// and header subscriptions, such that it listens for new headers and
+// new shares and uses this information to pool peers by shares.
+//
+// This gives the peer manager an ability to block peers that gossip invalid shares, but also access a list of peers
+// that are known to have been gossiping valid shares.
+// The peers are then returned on request using a round-robin algorithm to return a different peer each time.
+// If no peers are found, the peer manager will rely on full nodes retrieved from discovery.
+//
+// The peer manager is only concerned with recent heights, thus it retrieves peers that
+// were active since `initialHeight`.
+// The peer manager will also garbage collect peers such that it blacklists peers that
+// have been active since `initialHeight` but have been found to be invalid.
+//
+// The peer manager is passed to the shrex getter and is used at request time to
+// select peers for a given data hash for data retrieval.
+//
+// # Usage
+//
+// The peer manager is created using [NewManager] constructor:
+//
+//	peerManager := peers.NewManager(headerSub, shrexSub, discovery, host, connGater, opts...)
+//
+// After creating the peer manager, it should be started to kick off listening and
+// validation routines that enable peer selection and retrieval:
+//
+//	err := peerManager.Start(ctx)
+//
+// The peer manager can be stopped at any time to stop all peer discovery and validation routines:
+//
+//	err := peerManager.Stop(ctx)
+//
+// The peer manager can be used to select peers for a given datahash for shares retrieval:
+//
+//	peer, err := peerManager.Peer(ctx, hash)
+package peers

share/p2p/shrexeds/doc.go

@@ -0,0 +1,51 @@
+// This package defines a protocol that is used to request
+// extended data squares from peers in the network.
+//
+// This protocol is a request/response protocol that allows for sending requests for extended data squares by data root
+// to the peers in the network and receiving a response containing the original data square(s), which is used
+// to recompute the extended data square.
+//
+// The streams are established using the protocol ID:
+//
+//   - "{networkID}/shrex/eds/v0.0.1" where networkID is the network ID of the network. (e.g. "arabica")
+//
+// When a peer receives a request for extended data squares, it will read
+// the original data square from the EDS store by retrieving the underlying
+// CARv1 file containing the full extended data square, but will limit reading
+// to the original data square shares only.
+// The client on the other hand will take care of computing the extended data squares from
+// the original data square on receipt.
+//
+// # Usage
+//
+// To use a shrexeds client to request extended data squares from a peer, you must
+// first create a new `shrexeds.Client` instance by:
+//
+//	client, err := shrexeds.NewClient(params, host)
+//
+// where `params` is a `shrexeds.Parameters` instance and `host` is a `libp2p.Host` instance.
+//
+// To request extended data squares from a peer, you must first create a `Client.RequestEDS` instance by:
+//
+//	eds, err := client.RequestEDS(ctx, dataHash, peer)
+//
+// where:
+//   - `ctx` is a `context.Context` instance,
+//   - `dataHash` is the data root of the extended data square and
+//   - `peer` is the peer ID of the peer to request the extended data square from.
+//
+// To use a shrexeds server to respond to requests for extended data squares from peers
+// you must first create a new `shrexeds.Server` instance by:
+//
+//	server, err := shrexeds.NewServer(params, host, store)
+//
+// where `params` is a [Parameters] instance, `host` is a libp2p.Host instance and `store` is a [eds.Store] instance.
+//
+// To start the server, you must call `Start` on the server:
+//
+//	err := server.Start(ctx)
+//
+// To stop the server, you must call `Stop` on the server:
+//
+//	err := server.Stop(ctx)
+package shrexeds

share/p2p/shrexnd/doc.go

@@ -0,0 +1,43 @@
+// This package defines a protocol that is used to request namespaced data from peers in the network.
+//
+// This protocol is a request/response protocol that sends a request for specific data that
+// lives in a specific namespace ID and receives a response with the data.
+//
+// The streams are established using the protocol ID:
+//
+//   - "{networkID}/shrex/nd/0.0.1" where networkID is the network ID of the network. (e.g. "arabica")
+//
+// The protocol uses protobuf to serialize and deserialize messages.
+//
+// # Usage
+//
+// To use a shrexnd client to request data from a peer, you must first create a new `shrexnd.Client` instance by:
+//
+// 1. Create a new client using `NewClient` and pass in the parameters of the protocol and the host:
+//
+//	client, err := shrexnd.NewClient(params, host)
+//
+// 2. Request data from a peer by calling [Client.RequestND] on the client and
+// pass in the context, the data root, the namespace ID and the peer ID:
+//
+//	data, err := client.RequestND(ctx, dataRoot, peerID, namespaceID)
+//
+// where data is of type [share.NamespacedShares]
+//
+// To use a shrexnd server to respond to requests from peers, you must first create a new `shrexnd.Server` instance by:
+//
+// 1. Create a new server using `NewServer` and pass in the parameters of
+// the protocol, the host, the store and store share getter:
+//
+//	server, err := shrexnd.NewServer(params, host, store, storeShareGetter)
+//
+// where store is of type [share.Store] and storeShareGetter is of type [share.Getter]
+//
+// 2. Start the server by calling `Start` on the server:
+//
+//	err := server.Start(ctx)
+//
+// 3. Stop the server by calling `Stop` on the server:
+//
+//	err := server.Stop(ctx)
+package shrexnd

share/p2p/shrexsub/doc.go

@@ -0,0 +1,58 @@
+// This package defines a protocol that is used to broadcast shares to peers over a pubsub network.
+//
+// This protocol runs on a rudimentary floodsub network is primarily a pubsub protocol
+// that broadcasts and listens for shares over a pubsub topic.
+//
+// The pubsub topic used by this protocol is:
+//
+//	"{networkID}/eds-sub/v0.1.0"
+//
+// where networkID is the network ID of the celestia-node that is running the protocol. (e.g. "arabica")
+//
+// # Usage
+//
+// To use this protocol, you must first create a new `shrexsub.PubSub` instance by:
+//
+//	pubsub, err := shrexsub.NewPubSub(ctx, host, networkID)
+//
+// where host is the libp2p host that is running the protocol, and networkID is the network ID of the celestia-node
+// that is running the protocol.
+//
+// After this, you can start the pubsub protocol by:
+//
+//	err := pubsub.Start(ctx)
+//
+// Once you have started the `shrexsub.PubSub` instance, you can broadcast a share by:
+//
+//	err := pubsub.Broadcast(ctx, notification)
+//
+// where `notification` is of type [shrexsub.Notification].
+//
+// and `DataHash` is the hash of the share that you want to broadcast, and `Height` is the height of the share.
+//
+// You can also subscribe to the pubsub topic by:
+//
+//	sub, err := pubsub.Subscribe(ctx)
+//
+// and then receive notifications by:
+//
+//	  for {
+//	  	select {
+//	  	case <-ctx.Done():
+//				sub.Cancel()
+//	  		return
+//	  	case notification, err := <-sub.Next():
+//	  		// handle notification or err
+//	  	}
+//	  }
+//
+// You can also manipulate the received pubsub messages by using the [PubSub.AddValidator] method:
+//
+//	pubsub.AddValidator(validator ValidatorFn)
+//
+// where `validator` is of type [shrexsub.ValidatorFn] and `Notification` is the same as above.
+//
+// You can also stop the pubsub protocol by:
+//
+//	err := pubsub.Stop(ctx)
+package shrexsub