feat: option to serve index provider ads over http

node/config/def.go

@@ -171,14 +171,19 @@ func DefaultBoost() *Boost {
 			MaxConcurrencyStorageCalls: 100,
 			GCInterval:                 lotus_config.Duration(1 * time.Minute),
 		},
-		IndexProvider: lotus_config.IndexProviderConfig{
+		IndexProvider: IndexProviderConfig{
 			Enable:               true,
 			EntriesCacheCapacity: 1024,
 			EntriesChunkSize:     16384,
 			// The default empty TopicName means it is inferred from network name, in the following
 			// format: "/indexer/ingest/<network-name>"
 			TopicName:         "",
 			PurgeCacheOnStart: false,
+
+			HttpEndpoint: IndexProviderHttpEndpointConfig{
+				PublicIP: "",
+				Port:     3104,
+			},
 		},
 	}
 	return cfg

node/config/types.go

@@ -49,7 +49,7 @@ type Boost struct {
 	LotusDealmaking lotus_config.DealmakingConfig
 	LotusFees       FeeConfig
 	DAGStore        lotus_config.DAGStoreConfig
-	IndexProvider   lotus_config.IndexProviderConfig
+	IndexProvider   IndexProviderConfig
 }
 
 func (b *Boost) GetDealmakingConfig() lotus_config.DealmakingConfig {
@@ -279,6 +279,50 @@ type ContractDealsConfig struct {
 	From string
 }
 
+type IndexProviderConfig struct {
+	// Enable set whether to enable indexing announcement to the network and expose endpoints that
+	// allow indexer nodes to process announcements. Enabled by default.
+	Enable bool
+
+	// EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement
+	// entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The
+	// maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and
+	// the length of multihashes being advertised. For example, advertising 128-bit long multihashes
+	// with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to
+	// 256MiB when full.
+	EntriesCacheCapacity int
+
+	// EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk.
+	// Defaults to 16384 if not specified. Note that chunks are chained together for indexing
+	// advertisements that include more multihashes than the configured EntriesChunkSize.
+	EntriesChunkSize int
+
+	// TopicName sets the topic name on which the changes to the advertised content are announced.
+	// If not explicitly specified, the topic name is automatically inferred from the network name
+	// in following format: '/indexer/ingest/<network-name>'
+	// Defaults to empty, which implies the topic name is inferred from network name.
+	TopicName string
+
+	// PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine
+	// starts. By default, the cache is rehydrated from previously cached entries stored in
+	// datastore if any is present.
+	PurgeCacheOnStart bool
+
+	HttpEndpoint IndexProviderHttpEndpointConfig
+}
+
+type IndexProviderHttpEndpointConfig struct {
+	// Set the public IPv4 endpoint for the index provider listener.
+	// eg "82.129.73.111"
+	// This is usually the same as the public IP for the boost node.
+	// If PublicIP is the empty string, requests are served over graphsync
+	// instead.
+	PublicIP string
+	// Set the port on which to listen for index provider requests over HTTP.
+	// Note that this port must be open on the firewall.
+	Port int
+}
+
 type FeeConfig struct {
 	// The maximum fee to pay when sending the PublishStorageDeals message
 	MaxPublishDealsFee types.FIL

node/modules/storageminer_idxprov.go

@@ -5,13 +5,13 @@ import (
 	"fmt"
 	"github.com/filecoin-project/boost/build"
 	"github.com/filecoin-project/boost/indexprovider"
+	"github.com/filecoin-project/boost/node/config"
 	"github.com/filecoin-project/boost/node/modules/dtypes"
 	"github.com/filecoin-project/boost/retrievalmarket/types"
 	"github.com/filecoin-project/go-address"
 	datatransfer "github.com/filecoin-project/go-data-transfer"
 	"github.com/filecoin-project/go-data-transfer/transport/graphsync"
 	datatransferv2 "github.com/filecoin-project/go-data-transfer/v2"
-	"github.com/filecoin-project/lotus/node/config"
 	lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
@@ -93,13 +93,25 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo
 			// The extra data is required by the lotus-specific index-provider gossip message validators.
 			ma := address.Address(maddr)
 			opts = append(opts,
-				engine.WithPublisherKind(engine.DataTransferPublisher),
 				engine.WithDataTransfer(dtV1ToIndexerDT(dt, func() ipld.LinkSystem {
 					return *e.LinkSystem()
 				})),
 				engine.WithExtraGossipData(ma.Bytes()),
 				engine.WithTopic(t),
 			)
+
+			// Advertisements can be served over the data transfer protocol
+			// (on graphsync) or over HTTP
+			if cfg.HttpEndpoint.PublicIP != "" {
+				opts = append(opts,
+					engine.WithPublisherKind(engine.HttpPublisher),
+					engine.WithHttpPublisherListenAddr(fmt.Sprintf("0.0.0.0:%d", cfg.HttpEndpoint.Port)),
+					engine.WithHttpPublisherAnnounceAddr(fmt.Sprintf("/ip4/%s/tcp/%d/http", cfg.HttpEndpoint.PublicIP, cfg.HttpEndpoint.Port)),
+				)
+			} else {
+				opts = append(opts, engine.WithPublisherKind(engine.DataTransferPublisher))
+			}
+
 			llog = llog.With("extraGossipData", ma, "publisher", "data-transfer")
 		} else {
 			opts = append(opts, engine.WithPublisherKind(engine.NoPublisher))