From 1b9d98331b8c75f7e9bf3be3fb384fa0c538caaf Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 24 Jan 2023 15:19:02 +0700 Subject: [PATCH 01/23] save --- cmd/rpcdaemon/commands/eth_receipts.go | 95 ++++++++++++++------------ 1 file changed, 51 insertions(+), 44 deletions(-) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index 96e34bab72c..b2e18d6af08 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -8,7 +8,6 @@ import ( "math/big" "github.com/RoaringBitmap/roaring" - "github.com/RoaringBitmap/roaring/roaring64" "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -301,51 +300,53 @@ func applyFilters(out *roaring.Bitmap, tx kv.Tx, begin, end uint64, crit filters return nil } -func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) error { +func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) (out iter.U64, err error) { //[from,to) var fromTxNum, toTxNum uint64 - var err error if begin > 0 { fromTxNum, err = rawdbv3.TxNums.Min(tx, begin) if err != nil { - return err + return out, err } } toTxNum, err = rawdbv3.TxNums.Max(tx, end) if err != nil { - return err + return out, err } toTxNum++ - out.AddRange(fromTxNum, toTxNum) // [from,to) + //out.AddRange(fromTxNum, toTxNum) // [from,to) topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum) if err != nil { - return err + return out, err } if topicsBitmap != nil { - out.And(topicsBitmap) + out = topicsBitmap + //out.And(topicsBitmap) } addrBitmap, err := getAddrsBitmapV3(tx, crit.Addresses, fromTxNum, toTxNum) if err != nil { - return err + return out, err } if addrBitmap != nil { - out.And(addrBitmap) + out = iter.Intersect[uint64](out, addrBitmap) + //out.And(addrBitmap) } - return nil + return nil, nil } func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) ([]*types.Log, error) { logs := []*types.Log{} - txNumbers := bitmapdb.NewBitmap64() - defer bitmapdb.ReturnToPool64(txNumbers) - if err := applyFiltersV3(txNumbers, tx, begin, end, crit); err != nil { + //txNumbers := bitmapdb.NewBitmap64() + //defer bitmapdb.ReturnToPool64(txNumbers) + txNumbers, err := applyFiltersV3(tx, begin, end, crit) + if err != nil { return logs, err } - if txNumbers.IsEmpty() { - return logs, nil - } + //if txNumbers.IsEmpty() { + // return logs, nil + //} addrMap := make(map[libcommon.Address]struct{}, len(crit.Addresses)) for _, v := range crit.Addresses { @@ -361,7 +362,7 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end var blockHash libcommon.Hash var header *types.Header - iter := MapTxNum2BlockNum(tx, bitmapdb.ToIter(txNumbers.Iterator())) + iter := MapTxNum2BlockNum(tx, txNumbers) for iter.HasNext() { if err = ctx.Err(); err != nil { return nil, err @@ -492,57 +493,63 @@ func (e *intraBlockExec) execTx(txNum uint64, txIndex int, txn types.Transaction // {{}, {B}} matches any topic in first position AND B in second position // {{A}, {B}} matches topic A in first position AND B in second position // {{A, B}, {C, D}} matches topic (A OR B) in first position AND (C OR D) in second position -func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (*roaring64.Bitmap, error) { - var result *roaring64.Bitmap +func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (iter.U64, error) { + var result iter.U64 + //var result *roaring64.Bitmap for _, sub := range topics { - bitmapForORing := bitmapdb.NewBitmap64() - defer bitmapdb.ReturnToPool64(bitmapForORing) + var bitmapForORing iter.U64 = iter.Array[uint64]([]uint64{}) + //defer bitmapdb.ReturnToPool64(bitmapForORing) for _, topic := range sub { it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to, true, -1) if err != nil { return nil, err } - bm, err := it.(bitmapdb.ToBitmap).ToBitmap() - if err != nil { - return nil, err - } - bitmapForORing.Or(bm) + bitmapForORing = iter.Union[uint64](bitmapForORing, it) + //bitmapForORing.Or(bm) } - if bitmapForORing.GetCardinality() == 0 { - continue - } + //if bitmapForORing.GetCardinality() == 0 { + // continue + //} if result == nil { - result = bitmapForORing.Clone() + result = bitmapForORing + //result = bitmapForORing.Clone() continue } - result = roaring64.And(bitmapForORing, result) + result = iter.Intersect[uint64](result, bitmapForORing) + //result = roaring64.And(bitmapForORing, result) } return result, nil } -func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint64) (*roaring64.Bitmap, error) { +func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint64) (iter.U64, error) { if len(addrs) == 0 { return nil, nil } - rx := make([]*roaring64.Bitmap, len(addrs)) - defer func() { - for _, bm := range rx { - bitmapdb.ReturnToPool64(bm) - } - }() - for idx, addr := range addrs { + //rx := make([]*roaring64.Bitmap, len(addrs)) + //defer func() { + // for _, bm := range rx { + // bitmapdb.ReturnToPool64(bm) + // } + //}() + var rx iter.U64 + for _, addr := range addrs { it, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], from, to, true, -1) if err != nil { return nil, err } - rx[idx], err = it.(bitmapdb.ToBitmap).ToBitmap() - if err != nil { - return nil, err + //rx[idx], err = it.(bitmapdb.ToBitmap).ToBitmap() + //if err != nil { + // return nil, err + //} + if rx == nil { + rx = it + } else { + rx = iter.Union[uint64](rx, it) } } - return roaring64.FastOr(rx...), nil + return rx, nil } // GetTransactionReceipt implements eth_getTransactionReceipt. Returns the receipt of a transaction given the transaction's hash. From 098379dc9c3dea002b75eda4626d8648d07e44ec Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 24 Jan 2023 15:19:38 +0700 Subject: [PATCH 02/23] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 53b1065eb46..2a41ffd90b0 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230122123931-ed637538bf95 + github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 6c9f2b871fa..4ed8a3d4920 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230122123931-ed637538bf95 h1:ITWQ5N+aQh3A5THpyO4+1x1lRxs1x0Imu8KjOo9c2Lo= -github.com/ledgerwatch/erigon-lib v0.0.0-20230122123931-ed637538bf95/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w= +github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4 h1:nsORV/9b2BJoUuDXS6yxIwFG9fJWCgIs0RqNctOzfmY= +github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 6a6b7b73af002dfbe576b10c24004a245fc6600d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 24 Jan 2023 15:42:09 +0700 Subject: [PATCH 03/23] save --- cmd/rpcdaemon/commands/eth_receipts.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index b2e18d6af08..a30b6136aa1 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -329,10 +329,17 @@ func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCrit return out, err } if addrBitmap != nil { - out = iter.Intersect[uint64](out, addrBitmap) + if out == nil { + out = addrBitmap + } else { + out = iter.Intersect[uint64](out, addrBitmap) + } //out.And(addrBitmap) } - return nil, nil + if out == nil { + out = iter.Range[uint64](fromTxNum, toTxNum) + } + return out, nil } func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) ([]*types.Log, error) { From 2ec4d76a3f1e9fc283742b16e6e0c36a10ee3727 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 24 Jan 2023 15:45:24 +0700 Subject: [PATCH 04/23] save --- cmd/rpcdaemon/commands/erigon_receipts_test.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/rpcdaemon/commands/erigon_receipts_test.go b/cmd/rpcdaemon/commands/erigon_receipts_test.go index 339b3e045e6..0ab32a9cb18 100644 --- a/cmd/rpcdaemon/commands/erigon_receipts_test.go +++ b/cmd/rpcdaemon/commands/erigon_receipts_test.go @@ -57,7 +57,7 @@ func TestGetLogs(t *testing.T) { // //api := NewErigonAPI(baseApi, m.DB, nil) //logs, err := api.GetLogs(m.Ctx, filters.FilterCriteria{FromBlock: big.NewInt(0), ToBlock: big.NewInt(10)}) - //assert.NoError(err) + //assert.NoError(err)s //assert.Equal(uint64(10), logs[0].BlockNumber) // ////filer by wrong address diff --git a/go.mod b/go.mod index 2a41ffd90b0..39609261eae 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4 + github.com/ledgerwatch/erigon-lib v0.0.0-20230124084401-0bb0e3768af5 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 4ed8a3d4920..515e8160c6f 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4 h1:nsORV/9b2BJoUuDXS6yxIwFG9fJWCgIs0RqNctOzfmY= -github.com/ledgerwatch/erigon-lib v0.0.0-20230124081902-66c84fc7c4f4/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w= +github.com/ledgerwatch/erigon-lib v0.0.0-20230124084401-0bb0e3768af5 h1:vbhPMdiWrs3TkI8VnmVDNdLZld224T49Kb2Jj9hz7V4= +github.com/ledgerwatch/erigon-lib v0.0.0-20230124084401-0bb0e3768af5/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From ef1ade095a98faa418340ff0425071677ab67ba7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 10:56:27 +0700 Subject: [PATCH 05/23] save --- cmd/integration/commands/reset_state.go | 2 +- cmd/integration/commands/stages.go | 2 +- cmd/rpcdaemon/cli/config.go | 5 ++--- cmd/rpcdaemon/commands/debug_api.go | 2 +- cmd/rpcdaemon/commands/eth_receipts.go | 2 +- cmd/rpcdaemon/commands/otterscan_api.go | 2 +- cmd/rpcdaemon/commands/trace_filtering.go | 2 +- core/chain_makers.go | 4 ++-- core/genesis.go | 7 +++---- core/state/temporal/kv_temporal.go | 19 ++++++++++--------- eth/stagedsync/exec3.go | 2 +- eth/stagedsync/stage_bodies.go | 2 +- eth/stagedsync/stage_execute.go | 2 +- eth/stagedsync/stage_execute_test.go | 2 +- eth/stagedsync/stage_hashstate.go | 2 +- eth/stagedsync/stage_interhashes.go | 2 +- eth/stagedsync/stage_snapshots.go | 2 +- turbo/app/snapshots.go | 2 +- turbo/rpchelper/helper.go | 2 +- 19 files changed, 32 insertions(+), 33 deletions(-) diff --git a/cmd/integration/commands/reset_state.go b/cmd/integration/commands/reset_state.go index 4a6fb960b3a..7530ae2e3d6 100644 --- a/cmd/integration/commands/reset_state.go +++ b/cmd/integration/commands/reset_state.go @@ -7,9 +7,9 @@ import ( "text/tabwriter" "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/core/rawdb/rawdbhelpers" reset2 "github.com/ledgerwatch/erigon/core/rawdb/rawdbreset" diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index d217b95a4a2..58053807c9e 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -13,9 +13,9 @@ import ( "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb" "github.com/ledgerwatch/erigon/cmd/sentry/sentry" diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 461bd092976..28f014ad8f6 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -17,13 +17,12 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/erigon-lib/common/hexutility" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" - "github.com/ledgerwatch/erigon/core/systemcontracts" - "github.com/ledgerwatch/erigon/core/state/historyv2read" "github.com/ledgerwatch/erigon/core/state/temporal" + "github.com/ledgerwatch/erigon/core/systemcontracts" "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/rpc/rpccfg" diff --git a/cmd/rpcdaemon/commands/debug_api.go b/cmd/rpcdaemon/commands/debug_api.go index fd4c037291f..1d1a0824c26 100644 --- a/cmd/rpcdaemon/commands/debug_api.go +++ b/cmd/rpcdaemon/commands/debug_api.go @@ -6,8 +6,8 @@ import ( jsoniter "github.com/json-iterator/go" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/common/changeset" "github.com/ledgerwatch/erigon/common/hexutil" diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index a30b6136aa1..46536fd861b 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -12,10 +12,10 @@ import ( "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" "github.com/ledgerwatch/erigon-lib/kv/iter" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/consensus" diff --git a/cmd/rpcdaemon/commands/otterscan_api.go b/cmd/rpcdaemon/commands/otterscan_api.go index 99eca602596..b099769dfd5 100644 --- a/cmd/rpcdaemon/commands/otterscan_api.go +++ b/cmd/rpcdaemon/commands/otterscan_api.go @@ -10,9 +10,9 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/iter" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/log/v3" diff --git a/cmd/rpcdaemon/commands/trace_filtering.go b/cmd/rpcdaemon/commands/trace_filtering.go index a901103f1d8..5158a0fd4ee 100644 --- a/cmd/rpcdaemon/commands/trace_filtering.go +++ b/cmd/rpcdaemon/commands/trace_filtering.go @@ -10,9 +10,9 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/bitmapdb" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/consensus/ethash" diff --git a/core/chain_makers.go b/core/chain_makers.go index 46fedd91a2d..49ed2ef1fd8 100644 --- a/core/chain_makers.go +++ b/core/chain_makers.go @@ -394,7 +394,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E c.Close() if GenerateTrace { fmt.Printf("State after %d================\n", b.header.Number) - it, err := tx.Stream(kv.HashedAccounts, nil, nil) + it, err := tx.Range(kv.HashedAccounts, nil, nil) if err != nil { return nil, nil, err } @@ -406,7 +406,7 @@ func GenerateChain(config *chain.Config, parent *types.Block, engine consensus.E fmt.Printf("%x: %x\n", k, v) } fmt.Printf("..................\n") - it, err = tx.Stream(kv.HashedStorage, nil, nil) + it, err = tx.Range(kv.HashedStorage, nil, nil) if err != nil { return nil, nil, err } diff --git a/core/genesis.go b/core/genesis.go index fa3e206b8b0..85d51b28b47 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -32,12 +32,9 @@ import ( "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" - "github.com/ledgerwatch/log/v3" - "golang.org/x/exp/slices" - + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/common" "github.com/ledgerwatch/erigon/common/hexutil" "github.com/ledgerwatch/erigon/common/math" @@ -50,6 +47,8 @@ import ( "github.com/ledgerwatch/erigon/params" "github.com/ledgerwatch/erigon/params/networkname" "github.com/ledgerwatch/erigon/turbo/trie" + "github.com/ledgerwatch/log/v3" + "golang.org/x/exp/slices" ) //go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index b950b0065cc..aad4a92f850 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -8,10 +8,11 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/length" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/iter" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/order" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" ) @@ -168,7 +169,7 @@ func (tx *Tx) DomainRangeAscend(name kv.Domain, k1, fromKey []byte, asOfTs uint6 copy(toPrefix, k1) binary.BigEndian.PutUint64(toPrefix[length.Addr:], inc+1) - it2, err := tx.StreamAscend(kv.PlainState, startkey, toPrefix, limit) + it2, err := tx.RangeAscend(kv.PlainState, startkey, toPrefix, limit) if err != nil { return nil, err } @@ -270,36 +271,36 @@ type Cursor struct { hitoryV3 bool } -func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, orderAscend bool, limit int) (timestamps iter.U64, err error) { - return tx.IndexStream(name, key, fromTs, toTs, orderAscend, limit) +func (tx *Tx) IndexRange(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, asc order.By, limit int) (timestamps iter.U64, err error) { + return tx.IndexStream(name, key, fromTs, toTs, asc, limit) } // [fromTs, toTs) -func (tx *Tx) IndexStream(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, orderAscend bool, limit int) (timestamps iter.U64, err error) { +func (tx *Tx) IndexStream(name kv.InvertedIdx, key []byte, fromTs, toTs uint64, asc order.By, limit int) (timestamps iter.U64, err error) { switch name { case LogTopicIdx: - t, err := tx.agg.LogTopicIterator(key, fromTs, toTs, orderAscend, limit, tx) + t, err := tx.agg.LogTopicIterator(key, fromTs, toTs, asc, limit, tx) if err != nil { return nil, err } tx.resourcesToClose = append(tx.resourcesToClose, t) return t, nil case LogAddrIdx: - t, err := tx.agg.LogAddrIterator(key, fromTs, toTs, orderAscend, limit, tx) + t, err := tx.agg.LogAddrIterator(key, fromTs, toTs, asc, limit, tx) if err != nil { return nil, err } tx.resourcesToClose = append(tx.resourcesToClose, t) return t, nil case TracesFromIdx: - t, err := tx.agg.TraceFromIterator(key, fromTs, toTs, orderAscend, limit, tx) + t, err := tx.agg.TraceFromIterator(key, fromTs, toTs, asc, limit, tx) if err != nil { return nil, err } tx.resourcesToClose = append(tx.resourcesToClose, t) return t, nil case TracesToIdx: - t, err := tx.agg.TraceToIterator(key, fromTs, toTs, orderAscend, limit, tx) + t, err := tx.agg.TraceToIterator(key, fromTs, toTs, asc, limit, tx) if err != nil { return nil, err } diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 1bb105ef14f..9e43eb9e923 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -21,10 +21,10 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/dir" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" kv2 "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" state2 "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" diff --git a/eth/stagedsync/stage_bodies.go b/eth/stagedsync/stage_bodies.go index 78ce9c0c92f..ed5f188b621 100644 --- a/eth/stagedsync/stage_bodies.go +++ b/eth/stagedsync/stage_bodies.go @@ -9,8 +9,8 @@ import ( "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/log/v3" "github.com/ledgerwatch/erigon/core/rawdb" diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index 09c65da6b4d..f8d0f961298 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -17,9 +17,9 @@ import ( "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/common/length" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index af35b95b863..4dbd03c68ff 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -7,9 +7,9 @@ import ( "testing" "time" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/cmd/state/exec22" diff --git a/eth/stagedsync/stage_hashstate.go b/eth/stagedsync/stage_hashstate.go index dd9c4d81ee1..c1b755e0b22 100644 --- a/eth/stagedsync/stage_hashstate.go +++ b/eth/stagedsync/stage_hashstate.go @@ -15,9 +15,9 @@ import ( "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/common/length" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" diff --git a/eth/stagedsync/stage_interhashes.go b/eth/stagedsync/stage_interhashes.go index 99e7aa627b2..174ab41ec2a 100644 --- a/eth/stagedsync/stage_interhashes.go +++ b/eth/stagedsync/stage_interhashes.go @@ -10,9 +10,9 @@ import ( libcommon "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/common/length" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv/temporal/historyv2" "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index c9510469b86..0d50152189d 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -15,12 +15,12 @@ import ( "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" proto_downloader "github.com/ledgerwatch/erigon-lib/gointerfaces/downloader" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" "golang.org/x/sync/semaphore" diff --git a/turbo/app/snapshots.go b/turbo/app/snapshots.go index 923d1d03fe3..02005217c11 100644 --- a/turbo/app/snapshots.go +++ b/turbo/app/snapshots.go @@ -16,13 +16,13 @@ import ( "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/compress" "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" "github.com/ledgerwatch/log/v3" diff --git a/turbo/rpchelper/helper.go b/turbo/rpchelper/helper.go index f7dc817bcb1..1fe4b8f7467 100644 --- a/turbo/rpchelper/helper.go +++ b/turbo/rpchelper/helper.go @@ -5,9 +5,9 @@ import ( "fmt" libcommon "github.com/ledgerwatch/erigon-lib/common" - "github.com/ledgerwatch/erigon-lib/common/rawdbv3" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcache" + "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" "github.com/ledgerwatch/erigon/core/systemcontracts" From 4ea75e7107a482b51788976080678edf6abe31a2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 12:25:58 +0700 Subject: [PATCH 06/23] save --- core/state/temporal/kv_temporal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index aad4a92f850..ef9d14015af 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -173,7 +173,7 @@ func (tx *Tx) DomainRangeAscend(name kv.Domain, k1, fromKey []byte, asOfTs uint6 if err != nil { return nil, err } - it3 := iter.TransformPairs(it2, func(k, v []byte) ([]byte, []byte) { + it3 := iter.TransformKV(it2, func(k, v []byte) ([]byte, []byte) { return append(append([]byte{}, k[:20]...), k[28:]...), v }) //TODO: seems MergePairs can't handle "amount" request From de607f9be4cf444e5dbcfdc900a90f99d1160318 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 13:58:59 +0700 Subject: [PATCH 07/23] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7a6bb23386d..be4ae55e530 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b + github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index eb5454149bc..fe13f10210d 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b h1:WyiyLh4CgS3ppKA8Q52x8X6hH8prIDaGx7WAGJm/sxA= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125023217-b5dc69285e6b/go.mod h1:Y3iOj42R4XP6OwYd98pztIjiQfhj5/MBLqIQfPQS61w= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6 h1:Kq9exSwVpFdiJN0LevslPQlhYK4d8EUxzuuD9YxM26A= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6/go.mod h1:4Eitih9Yxzwr437VsUr0TY+cRyXU9ep2RaCvjEw9hrE= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From a1e228da6778f49983020fa300bce3f639f1fa3e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:05:27 +0700 Subject: [PATCH 08/23] save --- go.mod | 2 +- go.sum | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index be4ae55e530..e9554853d74 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6 + github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index fe13f10210d..66e4bc9acca 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,10 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6 h1:Kq9exSwVpFdiJN0LevslPQlhYK4d8EUxzuuD9YxM26A= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125065825-37f7b5ce35d6/go.mod h1:4Eitih9Yxzwr437VsUr0TY+cRyXU9ep2RaCvjEw9hrE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125070448-5e3c613a70ec h1:23CoYuSA52ox+qV2+gTBkPSmj0wq9eZJhznQ2Y64HRI= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125070448-5e3c613a70ec/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3 h1:BAgxE9oM6TEZewzNcPHX6r57eVrb44rBn3JQbIo4110= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From a5bbbaaddb01dabc7e5d10854cb26e8760934c4e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:06:54 +0700 Subject: [PATCH 09/23] save --- go.sum | 2 -- 1 file changed, 2 deletions(-) diff --git a/go.sum b/go.sum index 66e4bc9acca..bb8126b184f 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,6 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125070448-5e3c613a70ec h1:23CoYuSA52ox+qV2+gTBkPSmj0wq9eZJhznQ2Y64HRI= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125070448-5e3c613a70ec/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3 h1:BAgxE9oM6TEZewzNcPHX6r57eVrb44rBn3JQbIo4110= github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= From bfe2d4ca55e1a1ed4205aa507ec59b0db53f2e94 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:30:24 +0700 Subject: [PATCH 10/23] save --- cmd/erigon-el/backend/backend.go | 2 +- cmd/integration/commands/stages.go | 2 +- cmd/rpcdaemon/cli/config.go | 2 +- core/state/temporal/kv_temporal.go | 2 +- eth/backend.go | 2 +- eth/stagedsync/stage_execute_test.go | 2 +- turbo/app/snapshots.go | 4 ++-- turbo/stages/mock_sentry.go | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/erigon-el/backend/backend.go b/cmd/erigon-el/backend/backend.go index 52e4066605c..4e1c75718ba 100644 --- a/cmd/erigon-el/backend/backend.go +++ b/cmd/erigon-el/backend/backend.go @@ -888,7 +888,7 @@ func (s *Ethereum) setUpBlockReader(ctx context.Context, dirs datadir.Dirs, snCo } dir.MustExist(dirs.SnapHistory) - agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB) + agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB) if err != nil { return nil, nil, nil, err } diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 58053807c9e..6969ac91885 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -1149,7 +1149,7 @@ func allSnapshots(ctx context.Context, db kv.RoDB) (*snapshotsync.RoSnapshots, * _allSnapshotsSingleton = snapshotsync.NewRoSnapshots(snapCfg, dirs.Snap) var err error - _aggSingleton, err = libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) + _aggSingleton, err = libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) if err != nil { panic(err) } diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 28f014ad8f6..2c2b3af4ef1 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -349,7 +349,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, allSnapshots.OptimisticReopenWithDB(db) allSnapshots.LogStat() - if agg, err = libstate.NewAggregator22(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil { + if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil { return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("create aggregator: %w", err) } _ = agg.ReopenFiles() diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index ef9d14015af..747931be6dd 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -240,7 +240,7 @@ func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok b if records, ok := tx.db.systemContractLookup[common.BytesToAddress(key)]; ok { p := sort.Search(len(records), func(i int) bool { return records[i].TxNumber > ts - }) + })2 hash := records[p-1].CodeHash force = &hash } diff --git a/eth/backend.go b/eth/backend.go index 989459b23ed..5110991a95d 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -1002,7 +1002,7 @@ func (s *Ethereum) setUpBlockReader(ctx context.Context, dirs datadir.Dirs, snCo } dir.MustExist(dirs.SnapHistory) - agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB) + agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, s.chainDB) if err != nil { return nil, nil, nil, err } diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index 4dbd03c68ff..fae31a027b9 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -165,7 +165,7 @@ func apply(tx kv.RwTx, agg *libstate.AggregatorV3) (beforeBlock, afterBlock test func newAgg(t *testing.T) *libstate.AggregatorV3 { t.Helper() dir, ctx := t.TempDir(), context.Background() - agg, err := libstate.NewAggregator22(ctx, dir, dir, ethconfig.HistoryV3AggregationStep, nil) + agg, err := libstate.NewAggregatorV3(ctx, dir, dir, ethconfig.HistoryV3AggregationStep, nil) require.NoError(t, err) err = agg.ReopenFiles() require.NoError(t, err) diff --git a/turbo/app/snapshots.go b/turbo/app/snapshots.go index 02005217c11..67941fec9b2 100644 --- a/turbo/app/snapshots.go +++ b/turbo/app/snapshots.go @@ -220,7 +220,7 @@ func doIndicesCommand(cliCtx *cli.Context) error { if err := snapshotsync.BuildMissedIndices("Indexing", ctx, dirs, *chainID, sem); err != nil { return err } - agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB) + agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, chainDB) if err != nil { return err } @@ -348,7 +348,7 @@ func doRetireCommand(cliCtx *cli.Context) error { } br := snapshotsync.NewBlockRetire(estimate.CompressSnapshot.Workers(), dirs.Tmp, snapshots, db, nil, nil) - agg, err := libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) + agg, err := libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) if err != nil { return err } diff --git a/turbo/stages/mock_sentry.go b/turbo/stages/mock_sentry.go index 2a31d89efdf..efe2f84f403 100644 --- a/turbo/stages/mock_sentry.go +++ b/turbo/stages/mock_sentry.go @@ -246,7 +246,7 @@ func MockWithEverything(t *testing.T, gspec *core.Genesis, key *ecdsa.PrivateKey var agg *libstate.AggregatorV3 if cfg.HistoryV3 { dir.MustExist(dirs.SnapHistory) - agg, err = libstate.NewAggregator22(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) + agg, err = libstate.NewAggregatorV3(ctx, dirs.SnapHistory, dirs.Tmp, ethconfig.HistoryV3AggregationStep, db) if err != nil { panic(err) } From 48f8c117e3c3c09bd3389baede95fb44cea0d500 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:30:50 +0700 Subject: [PATCH 11/23] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e9554853d74..b0a20c48337 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3 + github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index bb8126b184f..28386346c96 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3 h1:BAgxE9oM6TEZewzNcPHX6r57eVrb44rBn3JQbIo4110= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125070507-82d4a9f4c8d3/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662 h1:b4w1QxLmezr/DuzeG8QFMbE5w66tFLfkr7riQbX2ZzI= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 162cfa3031167807db3df2e6c90fa8f7e40488e9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:35:29 +0700 Subject: [PATCH 12/23] save --- cmd/rpcdaemon/commands/eth_receipts.go | 128 ++++++++++++++++++------- cmd/state/exec3/state.go | 4 +- core/state/rw_v3.go | 50 +++++----- core/state/temporal/kv_temporal.go | 2 +- 4 files changed, 124 insertions(+), 60 deletions(-) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index 46536fd861b..b88e7fb0eb2 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -18,22 +18,20 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/log/v3" - "github.com/ledgerwatch/erigon/consensus" - "github.com/ledgerwatch/erigon/core/vm/evmtypes" - "github.com/ledgerwatch/erigon/turbo/services" - - "github.com/ledgerwatch/erigon/core/state/temporal" - "github.com/ledgerwatch/erigon/common/hexutil" + "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/state" + "github.com/ledgerwatch/erigon/core/state/temporal" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/core/vm" + "github.com/ledgerwatch/erigon/core/vm/evmtypes" "github.com/ledgerwatch/erigon/eth/filters" "github.com/ledgerwatch/erigon/ethdb/cbor" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/rpchelper" + "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/transactions" ) @@ -300,6 +298,43 @@ func applyFilters(out *roaring.Bitmap, tx kv.Tx, begin, end uint64, crit filters return nil } +/* + +func applyFiltersV3(out *roaring64.Bitmap, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) error { + //[from,to) + var fromTxNum, toTxNum uint64 + var err error + if begin > 0 { + fromTxNum, err = rawdbv3.TxNums.Min(tx, begin) + if err != nil { + return err + } + } + toTxNum, err = rawdbv3.TxNums.Max(tx, end) + if err != nil { + return err + } + toTxNum++ + + out.AddRange(fromTxNum, toTxNum) // [from,to) + topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum) + if err != nil { + return err + } + if topicsBitmap != nil { + out.And(topicsBitmap) + } + addrBitmap, err := getAddrsBitmapV3(tx, crit.Addresses, fromTxNum, toTxNum) + if err != nil { + return err + } + if addrBitmap != nil { + out.And(addrBitmap) + } + return nil +} +*/ + func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) (out iter.U64, err error) { //[from,to) var fromTxNum, toTxNum uint64 @@ -315,14 +350,12 @@ func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCrit } toTxNum++ - //out.AddRange(fromTxNum, toTxNum) // [from,to) topicsBitmap, err := getTopicsBitmapV3(tx, crit.Topics, fromTxNum, toTxNum) if err != nil { return out, err } if topicsBitmap != nil { out = topicsBitmap - //out.And(topicsBitmap) } addrBitmap, err := getAddrsBitmapV3(tx, crit.Addresses, fromTxNum, toTxNum) if err != nil { @@ -334,7 +367,6 @@ func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCrit } else { out = iter.Intersect[uint64](out, addrBitmap) } - //out.And(addrBitmap) } if out == nil { out = iter.Range[uint64](fromTxNum, toTxNum) @@ -345,15 +377,10 @@ func applyFiltersV3(tx kv.TemporalTx, begin, end uint64, crit filters.FilterCrit func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end uint64, crit filters.FilterCriteria) ([]*types.Log, error) { logs := []*types.Log{} - //txNumbers := bitmapdb.NewBitmap64() - //defer bitmapdb.ReturnToPool64(txNumbers) txNumbers, err := applyFiltersV3(tx, begin, end, crit) if err != nil { return logs, err } - //if txNumbers.IsEmpty() { - // return logs, nil - //} addrMap := make(map[libcommon.Address]struct{}, len(crit.Addresses)) for _, v := range crit.Addresses { @@ -502,10 +529,8 @@ func (e *intraBlockExec) execTx(txNum uint64, txIndex int, txn types.Transaction // {{A, B}, {C, D}} matches topic (A OR B) in first position AND (C OR D) in second position func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (iter.U64, error) { var result iter.U64 - //var result *roaring64.Bitmap for _, sub := range topics { var bitmapForORing iter.U64 = iter.Array[uint64]([]uint64{}) - //defer bitmapdb.ReturnToPool64(bitmapForORing) for _, topic := range sub { it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to, true, -1) @@ -513,19 +538,13 @@ func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uin return nil, err } bitmapForORing = iter.Union[uint64](bitmapForORing, it) - //bitmapForORing.Or(bm) } - //if bitmapForORing.GetCardinality() == 0 { - // continue - //} if result == nil { result = bitmapForORing - //result = bitmapForORing.Clone() continue } result = iter.Intersect[uint64](result, bitmapForORing) - //result = roaring64.And(bitmapForORing, result) } return result, nil } @@ -534,22 +553,12 @@ func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint if len(addrs) == 0 { return nil, nil } - //rx := make([]*roaring64.Bitmap, len(addrs)) - //defer func() { - // for _, bm := range rx { - // bitmapdb.ReturnToPool64(bm) - // } - //}() var rx iter.U64 for _, addr := range addrs { it, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], from, to, true, -1) if err != nil { return nil, err } - //rx[idx], err = it.(bitmapdb.ToBitmap).ToBitmap() - //if err != nil { - // return nil, err - //} if rx == nil { rx = it } else { @@ -559,6 +568,61 @@ func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint return rx, nil } +/* +func getTopicsBitmapV3(tx kv.TemporalTx, topics [][]libcommon.Hash, from, to uint64) (*roaring64.Bitmap, error) { + var result *roaring64.Bitmap + for _, sub := range topics { + bitmapForORing := bitmapdb.NewBitmap64() + defer bitmapdb.ReturnToPool64(bitmapForORing) + + for _, topic := range sub { + it, err := tx.IndexRange(temporal.LogTopicIdx, topic.Bytes(), from, to, true, -1) + if err != nil { + return nil, err + } + bm, err := it.(bitmapdb.ToBitmap).ToBitmap() + if err != nil { + return nil, err + } + bitmapForORing.Or(bm) + } + + if bitmapForORing.GetCardinality() == 0 { + continue + } + if result == nil { + result = bitmapForORing.Clone() + continue + } + result = roaring64.And(bitmapForORing, result) + } + return result, nil +} + +func getAddrsBitmapV3(tx kv.TemporalTx, addrs []libcommon.Address, from, to uint64) (*roaring64.Bitmap, error) { + if len(addrs) == 0 { + return nil, nil + } + rx := make([]*roaring64.Bitmap, len(addrs)) + defer func() { + for _, bm := range rx { + bitmapdb.ReturnToPool64(bm) + } + }() + for idx, addr := range addrs { + it, err := tx.IndexRange(temporal.LogAddrIdx, addr[:], from, to, true, -1) + if err != nil { + return nil, err + } + rx[idx], err = it.(bitmapdb.ToBitmap).ToBitmap() + if err != nil { + return nil, err + } + } + return roaring64.FastOr(rx...), nil +} +*/ + // GetTransactionReceipt implements eth_getTransactionReceipt. Returns the receipt of a transaction given the transaction's hash. func (api *APIImpl) GetTransactionReceipt(ctx context.Context, txnHash libcommon.Hash) (map[string]interface{}, error) { tx, err := api.db.BeginRo(ctx) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index a7c8d4d87cd..88c8376ca48 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -30,8 +30,8 @@ type Worker struct { background bool // if true - worker does manage RoTx (begin/rollback) in .ResetTx() blockReader services.FullBlockReader rs *state.StateV3 - stateWriter *state.StateWriter22 - stateReader *state.StateReader22 + stateWriter *state.StateWriterV3 + stateReader *state.StateReaderV3 chainConfig *chain.Config getHeader func(hash libcommon.Hash, number uint64) *types.Header diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 7f73b154fb7..f1a9ec0295f 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -582,7 +582,7 @@ func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool { return true } -type StateWriter22 struct { +type StateWriterV3 struct { rs *StateV3 txNum uint64 writeLists map[string]*exec22.KvList @@ -592,8 +592,8 @@ type StateWriter22 struct { codePrevs map[string]uint64 } -func NewStateWriter22(rs *StateV3) *StateWriter22 { - return &StateWriter22{ +func NewStateWriter22(rs *StateV3) *StateWriterV3 { + return &StateWriterV3{ rs: rs, writeLists: newWriteList(), accountPrevs: map[string][]byte{}, @@ -603,11 +603,11 @@ func NewStateWriter22(rs *StateV3) *StateWriter22 { } } -func (w *StateWriter22) SetTxNum(txNum uint64) { +func (w *StateWriterV3) SetTxNum(txNum uint64) { w.txNum = txNum } -func (w *StateWriter22) ResetWriteSet() { +func (w *StateWriterV3) ResetWriteSet() { w.writeLists = newWriteList() w.accountPrevs = map[string][]byte{} w.accountDels = map[string]*accounts.Account{} @@ -615,15 +615,15 @@ func (w *StateWriter22) ResetWriteSet() { w.codePrevs = map[string]uint64{} } -func (w *StateWriter22) WriteSet() map[string]*exec22.KvList { +func (w *StateWriterV3) WriteSet() map[string]*exec22.KvList { return w.writeLists } -func (w *StateWriter22) PrevAndDels() (map[string][]byte, map[string]*accounts.Account, map[string][]byte, map[string]uint64) { +func (w *StateWriterV3) PrevAndDels() (map[string][]byte, map[string]*accounts.Account, map[string][]byte, map[string]uint64) { return w.accountPrevs, w.accountDels, w.storagePrevs, w.codePrevs } -func (w *StateWriter22) UpdateAccountData(address common2.Address, original, account *accounts.Account) error { +func (w *StateWriterV3) UpdateAccountData(address common2.Address, original, account *accounts.Account) error { addressBytes := address.Bytes() value := make([]byte, account.EncodingLengthForStorage()) account.EncodeForStorage(value) @@ -638,7 +638,7 @@ func (w *StateWriter22) UpdateAccountData(address common2.Address, original, acc return nil } -func (w *StateWriter22) UpdateAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash, code []byte) error { +func (w *StateWriterV3) UpdateAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash, code []byte) error { addressBytes, codeHashBytes := address.Bytes(), codeHash.Bytes() w.writeLists[kv.Code].Keys = append(w.writeLists[kv.Code].Keys, codeHashBytes) w.writeLists[kv.Code].Vals = append(w.writeLists[kv.Code].Vals, code) @@ -651,7 +651,7 @@ func (w *StateWriter22) UpdateAccountCode(address common2.Address, incarnation u return nil } -func (w *StateWriter22) DeleteAccount(address common2.Address, original *accounts.Account) error { +func (w *StateWriterV3) DeleteAccount(address common2.Address, original *accounts.Account) error { addressBytes := address.Bytes() w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, addressBytes) w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, []byte{}) @@ -667,7 +667,7 @@ func (w *StateWriter22) DeleteAccount(address common2.Address, original *account return nil } -func (w *StateWriter22) WriteAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash, original, value *uint256.Int) error { +func (w *StateWriterV3) WriteAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash, original, value *uint256.Int) error { if *original == *value { return nil } @@ -679,11 +679,11 @@ func (w *StateWriter22) WriteAccountStorage(address common2.Address, incarnation return nil } -func (w *StateWriter22) CreateContract(address common2.Address) error { +func (w *StateWriterV3) CreateContract(address common2.Address) error { return nil } -type StateReader22 struct { +type StateReaderV3 struct { tx kv.Tx txNum uint64 trace bool @@ -692,34 +692,34 @@ type StateReader22 struct { readLists map[string]*exec22.KvList } -func NewStateReader22(rs *StateV3) *StateReader22 { - return &StateReader22{ +func NewStateReader22(rs *StateV3) *StateReaderV3 { + return &StateReaderV3{ rs: rs, readLists: newReadList(), } } -func (r *StateReader22) SetTxNum(txNum uint64) { +func (r *StateReaderV3) SetTxNum(txNum uint64) { r.txNum = txNum } -func (r *StateReader22) SetTx(tx kv.Tx) { +func (r *StateReaderV3) SetTx(tx kv.Tx) { r.tx = tx } -func (r *StateReader22) ResetReadSet() { +func (r *StateReaderV3) ResetReadSet() { r.readLists = newReadList() } -func (r *StateReader22) ReadSet() map[string]*exec22.KvList { +func (r *StateReaderV3) ReadSet() map[string]*exec22.KvList { return r.readLists } -func (r *StateReader22) SetTrace(trace bool) { +func (r *StateReaderV3) SetTrace(trace bool) { r.trace = trace } -func (r *StateReader22) ReadAccountData(address common2.Address) (*accounts.Account, error) { +func (r *StateReaderV3) ReadAccountData(address common2.Address) (*accounts.Account, error) { addr := address.Bytes() enc := r.rs.Get(kv.PlainState, addr) if enc == nil { @@ -745,7 +745,7 @@ func (r *StateReader22) ReadAccountData(address common2.Address) (*accounts.Acco return &a, nil } -func (r *StateReader22) ReadAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash) ([]byte, error) { +func (r *StateReaderV3) ReadAccountStorage(address common2.Address, incarnation uint64, key *common2.Hash) ([]byte, error) { composite := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) enc := r.rs.Get(kv.PlainState, composite) if enc == nil { @@ -770,7 +770,7 @@ func (r *StateReader22) ReadAccountStorage(address common2.Address, incarnation return enc, nil } -func (r *StateReader22) ReadAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash) ([]byte, error) { +func (r *StateReaderV3) ReadAccountCode(address common2.Address, incarnation uint64, codeHash common2.Hash) ([]byte, error) { addr, codeHashBytes := address.Bytes(), codeHash.Bytes() enc := r.rs.Get(kv.Code, codeHashBytes) if enc == nil { @@ -788,7 +788,7 @@ func (r *StateReader22) ReadAccountCode(address common2.Address, incarnation uin return enc, nil } -func (r *StateReader22) ReadAccountCodeSize(address common2.Address, incarnation uint64, codeHash common2.Hash) (int, error) { +func (r *StateReaderV3) ReadAccountCodeSize(address common2.Address, incarnation uint64, codeHash common2.Hash) (int, error) { codeHashBytes := codeHash.Bytes() enc := r.rs.Get(kv.Code, codeHashBytes) if enc == nil { @@ -809,7 +809,7 @@ func (r *StateReader22) ReadAccountCodeSize(address common2.Address, incarnation return size, nil } -func (r *StateReader22) ReadAccountIncarnation(address common2.Address) (uint64, error) { +func (r *StateReaderV3) ReadAccountIncarnation(address common2.Address) (uint64, error) { enc := r.rs.Get(kv.IncarnationMap, address.Bytes()) if enc == nil { var err error diff --git a/core/state/temporal/kv_temporal.go b/core/state/temporal/kv_temporal.go index 747931be6dd..ef9d14015af 100644 --- a/core/state/temporal/kv_temporal.go +++ b/core/state/temporal/kv_temporal.go @@ -240,7 +240,7 @@ func (tx *Tx) HistoryGet(name kv.History, key []byte, ts uint64) (v []byte, ok b if records, ok := tx.db.systemContractLookup[common.BytesToAddress(key)]; ok { p := sort.Search(len(records), func(i int) bool { return records[i].TxNumber > ts - })2 + }) hash := records[p-1].CodeHash force = &hash } From 91a9a2defa6425e04e0195ee911be6e4c526a1bb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:36:02 +0700 Subject: [PATCH 13/23] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b0a20c48337..8fe5ca039f1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662 + github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 28386346c96..51766fec371 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662 h1:b4w1QxLmezr/DuzeG8QFMbE5w66tFLfkr7riQbX2ZzI= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125073024-2c08c3d15662/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e h1:gh8cdRVIyPHcK0tJ5dRIqnIrsEvf9XHRH6iHoG7wEug= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From ccfc26ee4955e52d6bac6b57d9aa1ac59d53d7cd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 14:39:43 +0700 Subject: [PATCH 14/23] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8fe5ca039f1..41713c3e8e5 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e + github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 51766fec371..f1ce0433aa6 100644 --- a/go.sum +++ b/go.sum @@ -555,8 +555,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e h1:gh8cdRVIyPHcK0tJ5dRIqnIrsEvf9XHRH6iHoG7wEug= -github.com/ledgerwatch/erigon-lib v0.0.0-20230125073529-44daa0dc151e/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7 h1:pUTs7rXO2WZTJGJ6tKvmdpOJcpF5BNTsB/9JYdB1rFI= +github.com/ledgerwatch/erigon-lib v0.0.0-20230125073906-c9ca126527c7/go.mod h1:NVtnPjcsVh/+Yu3QYbPoyg/58Q1a+qdbwktLmccSe9U= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da h1:lQQBOHzAUThkymfXJj/m07vAjyMx9XoMMy3OomaeOrA= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230120022649-cd9409a200da/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 4b5715f64a5eac5718a91a7a7caf56e02b82810f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:33:04 +0700 Subject: [PATCH 15/23] save --- cmd/rpcdaemon/commands/eth_receipts.go | 2 +- cmd/rpcdaemon/commands/otterscan_api.go | 2 +- turbo/snapshotsync/block_reader.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index b88e7fb0eb2..e76491c8b0d 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -415,7 +415,7 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end return nil, err } if header == nil { - log.Warn("header is nil", "blockNum", blockNum) + log.Warn("[rpc] header is nil", "blockNum", blockNum) continue } blockHash = header.Hash() diff --git a/cmd/rpcdaemon/commands/otterscan_api.go b/cmd/rpcdaemon/commands/otterscan_api.go index b099769dfd5..5a9922ade5b 100644 --- a/cmd/rpcdaemon/commands/otterscan_api.go +++ b/cmd/rpcdaemon/commands/otterscan_api.go @@ -298,7 +298,7 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co return nil, err } if header == nil { - log.Warn("header is nil", "blockNum", blockNum) + log.Warn("[rpc] header is nil", "blockNum", blockNum) continue } blockHash = header.Hash() diff --git a/turbo/snapshotsync/block_reader.go b/turbo/snapshotsync/block_reader.go index b3b30315c33..7bf4ae14628 100644 --- a/turbo/snapshotsync/block_reader.go +++ b/turbo/snapshotsync/block_reader.go @@ -262,6 +262,7 @@ func NewBlockReaderWithSnapshots(snapshots *RoSnapshots) *BlockReaderWithSnapsho func (back *BlockReaderWithSnapshots) Snapshots() *RoSnapshots { return back.sn } func (back *BlockReaderWithSnapshots) HeaderByNumber(ctx context.Context, tx kv.Getter, blockHeight uint64) (h *types.Header, err error) { + fmt.Printf("dbg: %d, %d, %d\n", back.sn.SegmentsMax(), back.sn.idxAvailability(), blockHeight) ok, err := back.sn.ViewHeaders(blockHeight, func(segment *HeaderSegment) error { h, _, err = back.headerFromSnapshot(blockHeight, segment, nil) if err != nil { From ad9d52f382dcd64e68cb99f3e5a1564bd1c8fc85 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:34:28 +0700 Subject: [PATCH 16/23] save --- cmd/rpcdaemon/cli/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 2c2b3af4ef1..e98c18974a0 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -347,6 +347,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, // To povide good UX - immediatly can read snapshots after RPCDaemon start, even if Erigon is down // Erigon does store list of snapshots in db: means RPCDaemon can read this list now, but read by `remoteKvClient.Snapshots` after establish grpc connection allSnapshots.OptimisticReopenWithDB(db) + allSnapshots.ReopenFolder() allSnapshots.LogStat() if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil { From 6990c21564c29f4db4ada611c2ca62ba81c21dc8 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:37:19 +0700 Subject: [PATCH 17/23] save --- turbo/transactions/call.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/turbo/transactions/call.go b/turbo/transactions/call.go index 16d9106defd..d1ceaeadcad 100644 --- a/turbo/transactions/call.go +++ b/turbo/transactions/call.go @@ -117,6 +117,10 @@ func MakeHeaderGetter(requireCanonical bool, tx kv.Tx, headerReader services.Hea log.Error("Can't get block hash by number", "number", n, "only-canonical", requireCanonical) return libcommon.Hash{} } + if h == nil { + log.Warn("[evm] header is nil", "blockNum", n) + return libcommon.Hash{} + } return h.Hash() } } From ceddc1c61b774c969f303a38e1d1ac916d4f17b5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:44:32 +0700 Subject: [PATCH 18/23] save --- cmd/rpcdaemon/commands/eth_receipts.go | 5 +++-- cmd/rpcdaemon/commands/otterscan_api.go | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/rpcdaemon/commands/eth_receipts.go b/cmd/rpcdaemon/commands/eth_receipts.go index e76491c8b0d..eea2f2ed15c 100644 --- a/cmd/rpcdaemon/commands/eth_receipts.go +++ b/cmd/rpcdaemon/commands/eth_receipts.go @@ -391,7 +391,7 @@ func (api *APIImpl) getLogsV3(ctx context.Context, tx kv.TemporalTx, begin, end if err != nil { return nil, err } - exec := newIntraBlockExec(tx, chainConfig, api.engine()) + exec := newIntraBlockExec(tx, chainConfig, api.engine(), api._blockReader) var blockHash libcommon.Hash var header *types.Header @@ -474,12 +474,13 @@ type intraBlockExec struct { vmConfig *vm.Config } -func newIntraBlockExec(tx kv.TemporalTx, chainConfig *chain.Config, engine consensus.EngineReader) *intraBlockExec { +func newIntraBlockExec(tx kv.TemporalTx, chainConfig *chain.Config, engine consensus.EngineReader, br services.FullBlockReader) *intraBlockExec { stateReader := state.NewHistoryReaderV3() stateReader.SetTx(tx) return &intraBlockExec{ engine: engine, chainConfig: chainConfig, + br: br, stateReader: stateReader, evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), vmConfig: &vm.Config{}, diff --git a/cmd/rpcdaemon/commands/otterscan_api.go b/cmd/rpcdaemon/commands/otterscan_api.go index 5a9922ade5b..ab0b7c59bcf 100644 --- a/cmd/rpcdaemon/commands/otterscan_api.go +++ b/cmd/rpcdaemon/commands/otterscan_api.go @@ -277,7 +277,7 @@ func (api *OtterscanAPIImpl) searchTransactionsBeforeV3(tx kv.TemporalTx, ctx co txNums := iter.Union[uint64](itFrom, itTo) txNumsIter := MapDescendTxNum2BlockNum(tx, txNums) - exec := newIntraBlockExec(tx, chainConfig, api.engine()) + exec := newIntraBlockExec(tx, chainConfig, api.engine(), api._blockReader) var blockHash libcommon.Hash var header *types.Header txs := make([]*RPCTransaction, 0, pageSize) From 7b53ce2de70fe8453ff27924056a78411c17f584 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:45:12 +0700 Subject: [PATCH 19/23] save --- turbo/snapshotsync/block_reader.go | 1 - 1 file changed, 1 deletion(-) diff --git a/turbo/snapshotsync/block_reader.go b/turbo/snapshotsync/block_reader.go index 7bf4ae14628..b3b30315c33 100644 --- a/turbo/snapshotsync/block_reader.go +++ b/turbo/snapshotsync/block_reader.go @@ -262,7 +262,6 @@ func NewBlockReaderWithSnapshots(snapshots *RoSnapshots) *BlockReaderWithSnapsho func (back *BlockReaderWithSnapshots) Snapshots() *RoSnapshots { return back.sn } func (back *BlockReaderWithSnapshots) HeaderByNumber(ctx context.Context, tx kv.Getter, blockHeight uint64) (h *types.Header, err error) { - fmt.Printf("dbg: %d, %d, %d\n", back.sn.SegmentsMax(), back.sn.idxAvailability(), blockHeight) ok, err := back.sn.ViewHeaders(blockHeight, func(segment *HeaderSegment) error { h, _, err = back.headerFromSnapshot(blockHeight, segment, nil) if err != nil { From bc67d809a6d01b256c3f3b820f4b8506c65692b7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:45:51 +0700 Subject: [PATCH 20/23] save --- cmd/rpcdaemon/cli/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index e98c18974a0..2c2b3af4ef1 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -347,7 +347,6 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, // To povide good UX - immediatly can read snapshots after RPCDaemon start, even if Erigon is down // Erigon does store list of snapshots in db: means RPCDaemon can read this list now, but read by `remoteKvClient.Snapshots` after establish grpc connection allSnapshots.OptimisticReopenWithDB(db) - allSnapshots.ReopenFolder() allSnapshots.LogStat() if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil { From c7775ee3f3ea5169c1367679c7b417ad2de83409 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:49:45 +0700 Subject: [PATCH 21/23] save --- cmd/rpcdaemon/cli/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index 2c2b3af4ef1..e98c18974a0 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -347,6 +347,7 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, // To povide good UX - immediatly can read snapshots after RPCDaemon start, even if Erigon is down // Erigon does store list of snapshots in db: means RPCDaemon can read this list now, but read by `remoteKvClient.Snapshots` after establish grpc connection allSnapshots.OptimisticReopenWithDB(db) + allSnapshots.ReopenFolder() allSnapshots.LogStat() if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil { From 6e7d3a95067ecf4d156f83596e22b52f227f96f9 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 15:55:16 +0700 Subject: [PATCH 22/23] save --- cmd/integration/commands/stages.go | 2 +- core/rawdb/rawdbreset/reset_stages.go | 5 +++-- eth/stagedsync/stage_snapshots.go | 7 +++++-- migrations/reset_blocks.go | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index 6969ac91885..a66f2fdb88c 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -487,7 +487,7 @@ func stageHeaders(db kv.RwDB, ctx context.Context) error { if reset { dirs := datadir.New(datadirCli) - if err := reset2.ResetBlocks(tx, db, sn, br, dirs, *chainConfig, engine); err != nil { + if err := reset2.ResetBlocks(tx, db, sn, agg, br, dirs, *chainConfig, engine); err != nil { return err } return nil diff --git a/core/rawdb/rawdbreset/reset_stages.go b/core/rawdb/rawdbreset/reset_stages.go index bac56e749eb..9ce97199cd7 100644 --- a/core/rawdb/rawdbreset/reset_stages.go +++ b/core/rawdb/rawdbreset/reset_stages.go @@ -12,6 +12,7 @@ import ( "github.com/ledgerwatch/erigon-lib/common/hexutility" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" + "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/log/v3" "go.uber.org/atomic" "golang.org/x/sync/errgroup" @@ -55,7 +56,7 @@ func ResetState(db kv.RwDB, ctx context.Context, chain string) error { return nil } -func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, br services.FullBlockReader, dirs datadir.Dirs, cc chain.Config, engine consensus.Engine) error { +func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, agg *state.AggregatorV3, br services.FullBlockReader, dirs datadir.Dirs, cc chain.Config, engine consensus.Engine) error { // keep Genesis if err := rawdb.TruncateBlocks(context.Background(), tx, 1); err != nil { return err @@ -105,7 +106,7 @@ func ResetBlocks(tx kv.RwTx, db kv.RoDB, snapshots *snapshotsync.RoSnapshots, br } if snapshots != nil && snapshots.Cfg().Enabled && snapshots.BlocksAvailable() > 0 { - if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, snapshots, br, cc, engine); err != nil { + if err := stagedsync.FillDBFromSnapshots("fillind_db_from_snapshots", context.Background(), tx, dirs, snapshots, br, cc, engine, agg); err != nil { return err } _ = stages.SaveStageProgress(tx, stages.Snapshots, snapshots.BlocksAvailable()) diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 0d50152189d..62d94fcb905 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -183,13 +183,13 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R s.BlockNumber = blocksAvailable } - if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.snapshots, cfg.blockReader, cfg.chainConfig, cfg.engine); err != nil { + if err := FillDBFromSnapshots(s.LogPrefix(), ctx, tx, cfg.dirs, cfg.snapshots, cfg.blockReader, cfg.chainConfig, cfg.engine, cfg.agg); err != nil { return err } return nil } -func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs datadir.Dirs, sn *snapshotsync.RoSnapshots, blockReader services.FullBlockReader, chainConfig chain.Config, engine consensus.Engine) error { +func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs datadir.Dirs, sn *snapshotsync.RoSnapshots, blockReader services.FullBlockReader, chainConfig chain.Config, engine consensus.Engine, agg *state.AggregatorV3) error { blocksAvailable := sn.BlocksAvailable() logEvery := time.NewTicker(logInterval) defer logEvery.Stop() @@ -327,6 +327,9 @@ func FillDBFromSnapshots(logPrefix string, ctx context.Context, tx kv.RwTx, dirs return fmt.Errorf("build txNum => blockNum mapping: %w", err) } } + if err := rawdb.WriteSnapshots(tx, sn.Files(), agg.Files()); err != nil { + return err + } } } return nil diff --git a/migrations/reset_blocks.go b/migrations/reset_blocks.go index 9afbfd63559..6b49d030c78 100644 --- a/migrations/reset_blocks.go +++ b/migrations/reset_blocks.go @@ -113,7 +113,7 @@ var resetBlocks4 = Migration{ } cc := tool.ChainConfig(tx) - if err := rawdbreset.ResetBlocks(tx, db, nil, nil, dirs, *cc, nil); err != nil { + if err := rawdbreset.ResetBlocks(tx, db, nil, nil, nil, dirs, *cc, nil); err != nil { return err } From 745d4797079fa3c62f6e775163ac77c65061b321 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 25 Jan 2023 16:28:25 +0700 Subject: [PATCH 23/23] save --- cmd/rpcdaemon/cli/config.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/rpcdaemon/cli/config.go b/cmd/rpcdaemon/cli/config.go index e98c18974a0..2c2b3af4ef1 100644 --- a/cmd/rpcdaemon/cli/config.go +++ b/cmd/rpcdaemon/cli/config.go @@ -347,7 +347,6 @@ func RemoteServices(ctx context.Context, cfg httpcfg.HttpCfg, logger log.Logger, // To povide good UX - immediatly can read snapshots after RPCDaemon start, even if Erigon is down // Erigon does store list of snapshots in db: means RPCDaemon can read this list now, but read by `remoteKvClient.Snapshots` after establish grpc connection allSnapshots.OptimisticReopenWithDB(db) - allSnapshots.ReopenFolder() allSnapshots.LogStat() if agg, err = libstate.NewAggregatorV3(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, ethconfig.HistoryV3AggregationStep, db); err != nil {