silkworm: add state snapshots

erigon-lib/state/aggregator.go

@@ -692,7 +692,7 @@ func (a *Aggregator) mergeLoopStep(ctx context.Context, toTxNum uint64) (somethi
 		return false, nil
 	}
 
-	outs, err := aggTx.staticFilesInRange(r)
+	outs, err := aggTx.StaticFilesInRange(r)
 	defer func() {
 		if closeAll {
 			outs.Close()
@@ -1244,6 +1244,10 @@ type RangesV3 struct {
 	invertedIndex []*MergeRange
 }
 
+func NewRangesV3(domain [kv.DomainLen]DomainRanges, invertedIndex []*MergeRange) RangesV3 {
+	return RangesV3{domain: domain, invertedIndex: invertedIndex}
+}
+
 func (r RangesV3) String() string {
 	ss := []string{}
 	for _, d := range &r.domain {

erigon-lib/state/aggregator_files.go

@@ -17,6 +17,7 @@
 package state
 
 import (
+	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/kv"
 )
 
@@ -27,7 +28,23 @@ type SelectedStaticFilesV3 struct {
 	ii    [][]*filesItem
 }
 
-func (sf SelectedStaticFilesV3) Close() {
+func (sf *SelectedStaticFilesV3) DomainFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.d[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) DomainHistoryFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.dHist[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) DomainInvertedIndexFiles(name kv.Domain) []FilesItem {
+	return common.SliceMap(sf.dIdx[name], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) InvertedIndexFiles(id int) []FilesItem {
+	return common.SliceMap(sf.ii[id], func(item *filesItem) FilesItem { return item })
+}
+
+func (sf *SelectedStaticFilesV3) Close() {
 	clist := make([][]*filesItem, 0, int(kv.DomainLen)+len(sf.ii))
 	for id := range sf.d {
 		clist = append(clist, sf.d[id], sf.dIdx[id], sf.dHist[id])
@@ -48,7 +65,7 @@ func (sf SelectedStaticFilesV3) Close() {
 	}
 }
 
-func (ac *AggregatorRoTx) staticFilesInRange(r *RangesV3) (*SelectedStaticFilesV3, error) {
+func (ac *AggregatorRoTx) StaticFilesInRange(r *RangesV3) (*SelectedStaticFilesV3, error) {
 	sf := &SelectedStaticFilesV3{ii: make([][]*filesItem, len(r.invertedIndex))}
 	for id := range ac.d {
 		if !r.domain[id].any() {
@@ -65,6 +82,14 @@ func (ac *AggregatorRoTx) staticFilesInRange(r *RangesV3) (*SelectedStaticFilesV
 	return sf, nil
 }
 
+func (ac *AggregatorRoTx) InvertedIndicesLen() int {
+	return len(ac.iis)
+}
+
+func (ac *AggregatorRoTx) InvertedIndexName(id int) kv.InvertedIdx {
+	return ac.iis[id].name
+}
+
 type MergedFilesV3 struct {
 	d     [kv.DomainLen]*filesItem
 	dHist [kv.DomainLen]*filesItem

erigon-lib/state/btree_index.go

@@ -30,6 +30,7 @@ import (
 	"strings"
 	"sync"
 	"time"
+	"unsafe"
 
 	"github.com/c2h5oh/datasize"
 	"github.com/edsrzf/mmap-go"
@@ -994,6 +995,10 @@ func (b *BtIndex) newCursor(k, v []byte, d uint64, g *seg.Reader) *Cursor {
 	return c
 }
 
+func (b *BtIndex) DataHandle() unsafe.Pointer {
+	return unsafe.Pointer(&b.data[0])
+}
+
 func (b *BtIndex) Size() int64 { return b.size }
 
 func (b *BtIndex) ModTime() time.Time { return b.modTime }

erigon-lib/state/files_item.go

@@ -62,13 +62,30 @@ type filesItem struct {
 	canDelete atomic.Bool
 }
 
+type FilesItem interface {
+	Segment() *seg.Decompressor
+	AccessorIndex() *recsplit.Index
+	BtIndex() *BtIndex
+	ExistenceFilter() *ExistenceFilter
+}
+
+var _ FilesItem = (*filesItem)(nil)
+
 func newFilesItem(startTxNum, endTxNum, stepSize uint64) *filesItem {
 	startStep := startTxNum / stepSize
 	endStep := endTxNum / stepSize
 	frozen := endStep-startStep == config3.StepsInFrozenFile
 	return &filesItem{startTxNum: startTxNum, endTxNum: endTxNum, frozen: frozen}
 }
 
+func (i *filesItem) Segment() *seg.Decompressor { return i.decompressor }
+
+func (i *filesItem) AccessorIndex() *recsplit.Index { return i.index }
+
+func (i *filesItem) BtIndex() *BtIndex { return i.bindex }
+
+func (i *filesItem) ExistenceFilter() *ExistenceFilter { return i.existence }
+
 // isSubsetOf - when `j` covers `i` but not equal `i`
 func (i *filesItem) isSubsetOf(j *filesItem) bool {
 	return (j.startTxNum <= i.startTxNum && i.endTxNum <= j.endTxNum) && (j.startTxNum != i.startTxNum || i.endTxNum != j.endTxNum)

erigon-lib/state/inverted_index.go

@@ -506,6 +506,10 @@ type MergeRange struct {
 	to        uint64
 }
 
+func NewMergeRange(name string, needMerge bool, from uint64, to uint64) *MergeRange {
+	return &MergeRange{name: name, needMerge: needMerge, from: from, to: to}
+}
+
 func (mr *MergeRange) FromTo() (uint64, uint64) {
 	return mr.from, mr.to
 }

erigon-lib/state/merge.go

@@ -81,6 +81,10 @@ type DomainRanges struct {
 	aggStep uint64
 }
 
+func NewDomainRanges(name kv.Domain, values MergeRange, history HistoryRanges, aggStep uint64) DomainRanges {
+	return DomainRanges{name: name, values: values, history: history, aggStep: aggStep}
+}
+
 func (r DomainRanges) String() string {
 	var b strings.Builder
 	if r.values.needMerge {
@@ -219,6 +223,10 @@ type HistoryRanges struct {
 	index   MergeRange
 }
 
+func NewHistoryRanges(history MergeRange, index MergeRange) HistoryRanges {
+	return HistoryRanges{history: history, index: index}
+}
+
 func (r HistoryRanges) String(aggStep uint64) string {
 	var str string
 	if r.history.needMerge {

erigon-lib/state/squeeze.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/c2h5oh/datasize"
+
 	"github.com/erigontech/erigon-lib/common"
 	"github.com/erigontech/erigon-lib/common/datadir"
 	"github.com/erigontech/erigon-lib/common/dir"
@@ -132,7 +133,7 @@ func (ac *AggregatorRoTx) SqueezeCommitmentFiles() error {
 			},
 		},
 	}
-	sf, err := ac.staticFilesInRange(rng)
+	sf, err := ac.StaticFilesInRange(rng)
 	if err != nil {
 		return err
 	}
@@ -331,7 +332,7 @@ func (a *Aggregator) RebuildCommitmentFiles(ctx context.Context, rwDb kv.RwDB, t
 			},
 		},
 	}
-	sf, err := acRo.staticFilesInRange(rng)
+	sf, err := acRo.StaticFilesInRange(rng)
 	if err != nil {
 		return nil, err
 	}

eth/stagedsync/stage_snapshots.go

@@ -300,12 +300,6 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
 		return err
 	}
 
-	if cfg.silkworm != nil {
-		if err := cfg.blockReader.Snapshots().(silkworm.CanAddSnapshotsToSilkwarm).AddSnapshotsToSilkworm(cfg.silkworm); err != nil {
-			return err
-		}
-	}
-
 	indexWorkers := estimate.IndexSnapshot.Workers()
 	diagnostics.Send(diagnostics.CurrentSyncSubStage{SubStage: "E3 Indexing"})
 	if err := agg.BuildMissedIndices(ctx, indexWorkers); err != nil {
@@ -321,6 +315,18 @@ func DownloadAndIndexSnapshotsIfNeed(s *StageState, ctx context.Context, tx kv.R
 		cfg.notifier.Events.OnNewSnapshot()
 	}
 
+	if cfg.silkworm != nil {
+		repository := silkworm.NewSnapshotsRepository(
+			cfg.silkworm,
+			cfg.blockReader.Snapshots().(*freezeblocks.RoSnapshots),
+			agg,
+			logger,
+		)
+		if err := repository.Update(); err != nil {
+			return err
+		}
+	}
+
 	frozenBlocks := cfg.blockReader.FrozenBlocks()
 	if s.BlockNumber < frozenBlocks { // allow genesis
 		if err := s.Update(tx, frozenBlocks); err != nil {

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/erigontech/erigonwatch v0.0.0-20240718131902-b6576bde1116
 	github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f
 	github.com/erigontech/secp256k1 v1.1.0
-	github.com/erigontech/silkworm-go v0.18.0
+	github.com/erigontech/silkworm-go v0.24.0
 )
 
 require (

go.sum

@@ -275,8 +275,8 @@ github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f h1:2uZv1SGd7
 github.com/erigontech/mdbx-go v0.38.6-0.20250205222432-e4dd01978d7f/go.mod h1:lkqHAZqXtFaIPlvTaGAx3VUDuGYZcuhve1l4JVVN1Z0=
 github.com/erigontech/secp256k1 v1.1.0 h1:mO3YJMUSoASE15Ya//SoHiisptUhdXExuMUN1M0X9qY=
 github.com/erigontech/secp256k1 v1.1.0/go.mod h1:GokhPepsMB+EYDs7I5JZCprxHW6+yfOcJKaKtoZ+Fls=
-github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA=
-github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
+github.com/erigontech/silkworm-go v0.24.0 h1:fFe74CjQM5LI7ouMYjmqfFaqIFzQTpMrt+ls+a5PxpE=
+github.com/erigontech/silkworm-go v0.24.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU=
 github.com/erigontech/speedtest v0.0.2 h1:W9Cvky/8AMUtUONwkLA/dZjeQ2XfkBdYfJzvhMZUO+U=
 github.com/erigontech/speedtest v0.0.2/go.mod h1:vulsRNiM51BmSTbVtch4FWxKxx53pS2D35lZTtao0bw=
 github.com/erigontech/torrent v1.54.3-alpha-1 h1:oyT9YpMr82g566v0STVKW0ZTdX/eun03cW2mKmKuTAQ=

turbo/silkworm/silkworm.go

@@ -33,16 +33,18 @@ type SilkwormLogLevel = silkworm_go.SilkwormLogLevel
 type SentrySettings = silkworm_go.SentrySettings
 type RpcDaemonSettings = silkworm_go.RpcDaemonSettings
 type RpcInterfaceLogSettings = silkworm_go.RpcInterfaceLogSettings
-type MappedHeaderSnapshot = silkworm_go.MappedHeaderSnapshot
-type MappedBodySnapshot = silkworm_go.MappedBodySnapshot
-type MappedTxnSnapshot = silkworm_go.MappedTxnSnapshot
-type MappedChainSnapshot = silkworm_go.MappedChainSnapshot
-
-var NewMemoryMappedRegion = silkworm_go.NewMemoryMappedRegion
-var NewMappedHeaderSnapshot = silkworm_go.NewMappedHeaderSnapshot
-var NewMappedBodySnapshot = silkworm_go.NewMappedBodySnapshot
-var NewMappedTxnSnapshot = silkworm_go.NewMappedTxnSnapshot
 
+type HeadersSnapshot = silkworm_go.HeadersSnapshot
+type BodiesSnapshot = silkworm_go.BodiesSnapshot
+type TransactionsSnapshot = silkworm_go.TransactionsSnapshot
+type BlocksSnapshotBundle = silkworm_go.BlocksSnapshotBundle
+type InvertedIndexSnapshot = silkworm_go.InvertedIndexSnapshot
+type HistorySnapshot = silkworm_go.HistorySnapshot
+type DomainSnapshot = silkworm_go.DomainSnapshot
+type StateSnapshotBundleLatest = silkworm_go.StateSnapshotBundleLatest
+type StateSnapshotBundleHistorical = silkworm_go.StateSnapshotBundleHistorical
+
+var NewFilePath = silkworm_go.NewFilePath
 var ErrInterrupted = silkworm_go.ErrInterrupted
 
 func New(dataDirPath string, libMdbxVersion string, numIOContexts uint32, logLevel log.Lvl) (*Silkworm, error) {
@@ -125,7 +127,3 @@ func ExecuteBlocksPerpetual(s *Silkworm, db kv.RwDB, chainID *big.Int, startBloc
 	}
 	return lastExecutedBlock, err
 }
-
-type CanAddSnapshotsToSilkwarm interface {
-	AddSnapshotsToSilkworm(*Silkworm) error
-}