feat: introduce indexer module #285
Conversation
kant777
force-pushed
the
feature/indexer-explorer
branch
from
110dffa
to
814fea5
Compare
mrekucci
changed the title
kant777
force-pushed
the
feature/indexer-explorer
branch
6 times, most recently
from
c6844ad
to
4e2c562
Compare
kant777
force-pushed
the
feature/indexer-explorer
branch
6 times, most recently
from
e887a8b
to
b9a0fb9
Compare
kant777
force-pushed
the
feature/indexer-explorer
branch
2 times, most recently
from
2af9de1
to
011d8ec
Compare
kant777
force-pushed
the
feature/indexer-explorer
branch
from
011d8ec
to
faacca6
Compare
| return err | ||
| } | ||
|
|
||
| if err = bi.initializeBackwardIndex(ctx, latestBlockNumber.Uint64()); err != nil { |
There was a problem hiding this comment.
The latestBlockNumber should be passed as big.Int, otherwise IsUint64() should be used before converting to Uint64().
| return err | ||
| } | ||
|
|
||
| go bi.fetchForwardBlocks(ctx) |
There was a problem hiding this comment.
I'm not sure why the fetch and store blocks run asynchronously and are connected by a buffered channel if they are mapped 1:1. If one of them is slower/faster, the buffer channel will only help for a while. I would suggest doing block fetching and storing in one loop under one gorutine.
|
|
||
| bi.logger.Info("last indexed block", "blockNumber", lastForwardIndexedBlock, "direction", "forward") | ||
|
|
||
| if lastForwardIndexedBlock == nil || lastForwardIndexedBlock.Sign() == 0 { |
There was a problem hiding this comment.
Why is the lastForwardIndexedBlock == nil check needed here?
|
|
||
| bi.logger.Info("last indexed block", "blockNumber", lastBackwardIndexedBlock, "direction", "backward") | ||
|
|
||
| if lastBackwardIndexedBlock == nil || lastBackwardIndexedBlock.Sign() == 0 { |
There was a problem hiding this comment.
Why is the lastBackwardIndexedBlock == nil check needed here?
| continue | ||
| } | ||
|
|
||
| for blockNum := new(big.Int).Add(bi.lastForwardIndexedBlock, big.NewInt(1)); blockNum.Cmp(latestBlockNumber) <= 0; blockNum.Add(blockNum, big.NewInt(5)) { |
There was a problem hiding this comment.
Can we improve readability of this for loop?
| } | ||
|
|
||
| func (c *ESClient) GetAddresses(ctx context.Context) ([]string, error) { | ||
| query := &search.Request{ |
There was a problem hiding this comment.
This request should be made using the typed client.
| } | ||
|
|
||
| func (c *ESClient) CreateIndices(ctx context.Context) error { | ||
| indices := []string{"blocks", "transactions", "accounts"} |
There was a problem hiding this comment.
Index names are repeated everywhere, I would suggest declaring constants instead.
| return fmt.Errorf("failed to check if index %s exists: %w", index, err) | ||
| } | ||
| if !res { | ||
| indexSettings := esapi.IndicesCreateRequest{ |
There was a problem hiding this comment.
This request should be made using the typed client.
|
|
||
| func (c *ESClient) Bulk(ctx context.Context, indexName string, docs []interface{}) error { | ||
| var ( | ||
| countSuccessful uint64 |
There was a problem hiding this comment.
What are these counters for?
| type Storage interface { | ||
| IndexBlock(ctx context.Context, block *IndexBlock) error | ||
| IndexTransactions(ctx context.Context, transactions []*IndexTransaction) error | ||
| GetLastIndexedBlock(ctx context.Context, direction string) (*big.Int, error) |
There was a problem hiding this comment.
Avoid Get prefix, the same is below.
Indexer fetches blocks and transactions from the mev-commit chain and indexes them into Elasticsearch. It does both forwards and backward indexing so that users don't have to wait until the indexing process catches up with the latest block. Instead, it has two go routines. one that starts from latest block and moving forward and other that starts from latest-1 and backfills all the way to 0.