diff --git a/node/config/def.go b/node/config/def.go index 18364c996..36fa93121 100644 --- a/node/config/def.go +++ b/node/config/def.go @@ -171,7 +171,7 @@ func DefaultBoost() *Boost { MaxConcurrencyStorageCalls: 100, GCInterval: lotus_config.Duration(1 * time.Minute), }, - IndexProvider: lotus_config.IndexProviderConfig{ + IndexProvider: IndexProviderConfig{ Enable: true, EntriesCacheCapacity: 1024, EntriesChunkSize: 16384, @@ -179,6 +179,11 @@ func DefaultBoost() *Boost { // format: "/indexer/ingest/" TopicName: "", PurgeCacheOnStart: false, + + HttpEndpoint: IndexProviderHttpEndpointConfig{ + PublicIP: "", + Port: 3104, + }, }, } return cfg diff --git a/node/config/doc_gen.go b/node/config/doc_gen.go index fa3623b9e..cdddcd3cc 100644 --- a/node/config/doc_gen.go +++ b/node/config/doc_gen.go @@ -93,7 +93,7 @@ your node if metadata log is disabled`, }, { Name: "IndexProvider", - Type: "lotus_config.IndexProviderConfig", + Type: "IndexProviderConfig", Comment: ``, }, @@ -436,6 +436,76 @@ for any other deal.`, Comment: `The port that the graphql server listens on`, }, }, + "IndexProviderConfig": []DocField{ + { + Name: "Enable", + Type: "bool", + + Comment: `Enable set whether to enable indexing announcement to the network and expose endpoints that +allow indexer nodes to process announcements. Enabled by default.`, + }, + { + Name: "EntriesCacheCapacity", + Type: "int", + + Comment: `EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement +entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The +maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and +the length of multihashes being advertised. For example, advertising 128-bit long multihashes +with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to +256MiB when full.`, + }, + { + Name: "EntriesChunkSize", + Type: "int", + + Comment: `EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk. +Defaults to 16384 if not specified. Note that chunks are chained together for indexing +advertisements that include more multihashes than the configured EntriesChunkSize.`, + }, + { + Name: "TopicName", + Type: "string", + + Comment: `TopicName sets the topic name on which the changes to the advertised content are announced. +If not explicitly specified, the topic name is automatically inferred from the network name +in following format: '/indexer/ingest/' +Defaults to empty, which implies the topic name is inferred from network name.`, + }, + { + Name: "PurgeCacheOnStart", + Type: "bool", + + Comment: `PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine +starts. By default, the cache is rehydrated from previously cached entries stored in +datastore if any is present.`, + }, + { + Name: "HttpEndpoint", + Type: "IndexProviderHttpEndpointConfig", + + Comment: ``, + }, + }, + "IndexProviderHttpEndpointConfig": []DocField{ + { + Name: "PublicIP", + Type: "string", + + Comment: `Set the public IPv4 endpoint for the index provider listener. +eg "82.129.73.111" +This is usually the same as the public IP for the boost node. +If PublicIP is the empty string, requests are served over graphsync +instead.`, + }, + { + Name: "Port", + Type: "int", + + Comment: `Set the port on which to listen for index provider requests over HTTP. +Note that this port must be open on the firewall.`, + }, + }, "LotusDealmakingConfig": []DocField{ { Name: "PieceCidBlocklist", diff --git a/node/config/types.go b/node/config/types.go index 2aae18cb8..e715178fc 100644 --- a/node/config/types.go +++ b/node/config/types.go @@ -49,7 +49,7 @@ type Boost struct { LotusDealmaking lotus_config.DealmakingConfig LotusFees FeeConfig DAGStore lotus_config.DAGStoreConfig - IndexProvider lotus_config.IndexProviderConfig + IndexProvider IndexProviderConfig } func (b *Boost) GetDealmakingConfig() lotus_config.DealmakingConfig { @@ -279,6 +279,50 @@ type ContractDealsConfig struct { From string } +type IndexProviderConfig struct { + // Enable set whether to enable indexing announcement to the network and expose endpoints that + // allow indexer nodes to process announcements. Enabled by default. + Enable bool + + // EntriesCacheCapacity sets the maximum capacity to use for caching the indexing advertisement + // entries. Defaults to 1024 if not specified. The cache is evicted using LRU policy. The + // maximum storage used by the cache is a factor of EntriesCacheCapacity, EntriesChunkSize and + // the length of multihashes being advertised. For example, advertising 128-bit long multihashes + // with the default EntriesCacheCapacity, and EntriesChunkSize means the cache size can grow to + // 256MiB when full. + EntriesCacheCapacity int + + // EntriesChunkSize sets the maximum number of multihashes to include in a single entries chunk. + // Defaults to 16384 if not specified. Note that chunks are chained together for indexing + // advertisements that include more multihashes than the configured EntriesChunkSize. + EntriesChunkSize int + + // TopicName sets the topic name on which the changes to the advertised content are announced. + // If not explicitly specified, the topic name is automatically inferred from the network name + // in following format: '/indexer/ingest/' + // Defaults to empty, which implies the topic name is inferred from network name. + TopicName string + + // PurgeCacheOnStart sets whether to clear any cached entries chunks when the provider engine + // starts. By default, the cache is rehydrated from previously cached entries stored in + // datastore if any is present. + PurgeCacheOnStart bool + + HttpEndpoint IndexProviderHttpEndpointConfig +} + +type IndexProviderHttpEndpointConfig struct { + // Set the public IPv4 endpoint for the index provider listener. + // eg "82.129.73.111" + // This is usually the same as the public IP for the boost node. + // If PublicIP is the empty string, requests are served over graphsync + // instead. + PublicIP string + // Set the port on which to listen for index provider requests over HTTP. + // Note that this port must be open on the firewall. + Port int +} + type FeeConfig struct { // The maximum fee to pay when sending the PublishStorageDeals message MaxPublishDealsFee types.FIL diff --git a/node/modules/storageminer_idxprov.go b/node/modules/storageminer_idxprov.go index 9e3585672..3c32cf0e1 100644 --- a/node/modules/storageminer_idxprov.go +++ b/node/modules/storageminer_idxprov.go @@ -5,13 +5,13 @@ import ( "fmt" "github.com/filecoin-project/boost/build" "github.com/filecoin-project/boost/indexprovider" + "github.com/filecoin-project/boost/node/config" "github.com/filecoin-project/boost/node/modules/dtypes" "github.com/filecoin-project/boost/retrievalmarket/types" "github.com/filecoin-project/go-address" datatransfer "github.com/filecoin-project/go-data-transfer" "github.com/filecoin-project/go-data-transfer/transport/graphsync" datatransferv2 "github.com/filecoin-project/go-data-transfer/v2" - "github.com/filecoin-project/lotus/node/config" lotus_dtypes "github.com/filecoin-project/lotus/node/modules/dtypes" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -93,13 +93,25 @@ func IndexProvider(cfg config.IndexProviderConfig) func(params IdxProv, marketHo // The extra data is required by the lotus-specific index-provider gossip message validators. ma := address.Address(maddr) opts = append(opts, - engine.WithPublisherKind(engine.DataTransferPublisher), engine.WithDataTransfer(dtV1ToIndexerDT(dt, func() ipld.LinkSystem { return *e.LinkSystem() })), engine.WithExtraGossipData(ma.Bytes()), engine.WithTopic(t), ) + + // Advertisements can be served over the data transfer protocol + // (on graphsync) or over HTTP + if cfg.HttpEndpoint.PublicIP != "" { + opts = append(opts, + engine.WithPublisherKind(engine.HttpPublisher), + engine.WithHttpPublisherListenAddr(fmt.Sprintf("0.0.0.0:%d", cfg.HttpEndpoint.Port)), + engine.WithHttpPublisherAnnounceAddr(fmt.Sprintf("/ip4/%s/tcp/%d/http", cfg.HttpEndpoint.PublicIP, cfg.HttpEndpoint.Port)), + ) + } else { + opts = append(opts, engine.WithPublisherKind(engine.DataTransferPublisher)) + } + llog = llog.With("extraGossipData", ma, "publisher", "data-transfer") } else { opts = append(opts, engine.WithPublisherKind(engine.NoPublisher))