diff --git a/cmd/silkworm_api/snapshot_idx.go b/cmd/silkworm_api/snapshot_idx.go index b6e3b5a50d3..3b85c2db1dd 100644 --- a/cmd/silkworm_api/snapshot_idx.go +++ b/cmd/silkworm_api/snapshot_idx.go @@ -10,6 +10,7 @@ import ( "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/kv/mdbx" "github.com/ledgerwatch/erigon/cmd/hack/tool/fromdb" + coresnaptype "github.com/ledgerwatch/erigon/core/snaptype" "github.com/ledgerwatch/erigon/turbo/debug" "github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks" "github.com/ledgerwatch/log/v3" @@ -91,7 +92,7 @@ func buildIndex(cliCtx *cli.Context, dataDir string, snapshotPaths []string, min } switch segment.Type.Enum() { - case snaptype.Enums.Headers, snaptype.Enums.Bodies, snaptype.Enums.Transactions: + case coresnaptype.Enums.Headers, coresnaptype.Enums.Bodies, coresnaptype.Enums.Transactions: g.Go(func() error { jobProgress := &background.Progress{} ps.Add(jobProgress) diff --git a/cmd/snapshots/cmp/cmp.go b/cmd/snapshots/cmp/cmp.go index a904e41d9e3..d1cb0fc8414 100644 --- a/cmd/snapshots/cmp/cmp.go +++ b/cmd/snapshots/cmp/cmp.go @@ -260,13 +260,13 @@ func cmp(cliCtx *cli.Context) error { }) } else { for _, snapType := range snapTypes { - if snapType.Enum() == snaptype.Enums.Headers { + if snapType.Enum() == coresnaptype.Enums.Headers { funcs = append(funcs, func(ctx context.Context) (time.Duration, time.Duration, time.Duration, error) { return c.compareHeaders(ctx, h1ents, h2ents, headerWorkers, logger) }) } - if snapType.Enum() == snaptype.Enums.Bodies { + if snapType.Enum() == coresnaptype.Enums.Bodies { funcs = append(funcs, func(ctx context.Context) (time.Duration, time.Duration, time.Duration, error) { return c.compareBodies(ctx, b1ents, b2ents, bodyWorkers, logger) }) @@ -325,11 +325,11 @@ func splitEntries(files []fs.DirEntry, version snaptype.Version, firstBlock, las (firstBlock == 0 || snapInfo.From() >= firstBlock) && (lastBlock == 0 || snapInfo.From() < lastBlock) { - if snapInfo.Type().Enum() == snaptype.Enums.Headers { + if snapInfo.Type().Enum() == coresnaptype.Enums.Headers { hents = append(hents, ent) } - if snapInfo.Type().Enum() == snaptype.Enums.Bodies { + if snapInfo.Type().Enum() == coresnaptype.Enums.Bodies { found := false for _, bent := range bents { @@ -345,7 +345,7 @@ func splitEntries(files []fs.DirEntry, version snaptype.Version, firstBlock, las } } - if snapInfo.Type().Enum() == snaptype.Enums.Transactions { + if snapInfo.Type().Enum() == coresnaptype.Enums.Transactions { found := false for _, bent := range bents { diff --git a/core/snaptype/block_types.go b/core/snaptype/block_types.go index 15f9ad00188..7e3ed775e70 100644 --- a/core/snaptype/block_types.go +++ b/core/snaptype/block_types.go @@ -35,15 +35,40 @@ func init() { snapcfg.RegisterKnownTypes(networkname.ChiadoChainName, ethereumTypes) } +var Enums = struct { + snaptype.Enums + Headers, + Bodies, + Transactions snaptype.Enum +}{ + Enums: snaptype.Enums{}, + Headers: snaptype.MinCoreEnum, + Bodies: snaptype.MinCoreEnum + 1, + Transactions: snaptype.MinCoreEnum + 2, +} + +var Indexes = struct { + HeaderHash, + BodyHash, + TxnHash, + TxnHash2BlockNum snaptype.Index +}{ + HeaderHash: snaptype.Index{Name: "headers"}, + BodyHash: snaptype.Index{Name: "bodies"}, + TxnHash: snaptype.Index{Name: "transactions"}, + TxnHash2BlockNum: snaptype.Index{Name: "transactions-to-block", Offset: 1}, +} + var ( Headers = snaptype.RegisterType( - snaptype.Enums.Headers, + Enums.Headers, + "headers", snaptype.Versions{ Current: 1, //2, MinSupported: 1, }, nil, - []snaptype.Index{snaptype.Indexes.HeaderHash}, + []snaptype.Index{Indexes.HeaderHash}, snaptype.IndexBuilderFunc( func(ctx context.Context, info snaptype.FileInfo, salt uint32, _ *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { hasher := crypto.NewKeccakState() @@ -70,13 +95,14 @@ var ( ) Bodies = snaptype.RegisterType( - snaptype.Enums.Bodies, + Enums.Bodies, + "bodies", snaptype.Versions{ Current: 1, //2, MinSupported: 1, }, nil, - []snaptype.Index{snaptype.Indexes.BodyHash}, + []snaptype.Index{Indexes.BodyHash}, snaptype.IndexBuilderFunc( func(ctx context.Context, info snaptype.FileInfo, salt uint32, _ *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { num := make([]byte, 8) @@ -98,13 +124,14 @@ var ( ) Transactions = snaptype.RegisterType( - snaptype.Enums.Transactions, + Enums.Transactions, + "transactions", snaptype.Versions{ Current: 1, //2, MinSupported: 1, }, nil, - []snaptype.Index{snaptype.Indexes.TxnHash, snaptype.Indexes.TxnHash2BlockNum}, + []snaptype.Index{Indexes.TxnHash, Indexes.TxnHash2BlockNum}, snaptype.IndexBuilderFunc( func(ctx context.Context, sn snaptype.FileInfo, salt uint32, chainConfig *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { defer func() { @@ -162,7 +189,7 @@ var ( BucketSize: 2000, LeafSize: 8, TmpDir: tmpDir, - IndexFile: filepath.Join(sn.Dir(), sn.Type.IdxFileName(sn.Version, sn.From, sn.To, snaptype.Indexes.TxnHash2BlockNum)), + IndexFile: filepath.Join(sn.Dir(), sn.Type.IdxFileName(sn.Version, sn.From, sn.To, Indexes.TxnHash2BlockNum)), BaseDataID: firstBlockNum, }, logger) if err != nil { diff --git a/core/snaptype/block_types_test.go b/core/snaptype/block_types_test.go new file mode 100644 index 00000000000..ec704a7a3a9 --- /dev/null +++ b/core/snaptype/block_types_test.go @@ -0,0 +1,38 @@ +package snaptype_test + +import ( + "testing" + + "github.com/ledgerwatch/erigon/core/snaptype" +) + +func TestEnumeration(t *testing.T) { + + if snaptype.Headers.Enum() != snaptype.Enums.Headers { + t.Fatal("enum mismatch", snaptype.Headers, snaptype.Headers.Enum(), snaptype.Enums.Headers) + } + + if snaptype.Bodies.Enum() != snaptype.Enums.Bodies { + t.Fatal("enum mismatch", snaptype.Bodies, snaptype.Bodies.Enum(), snaptype.Enums.Bodies) + } + + if snaptype.Transactions.Enum() != snaptype.Enums.Transactions { + t.Fatal("enum mismatch", snaptype.Transactions, snaptype.Transactions.Enum(), snaptype.Enums.Transactions) + } + +} + +func TestNames(t *testing.T) { + + if snaptype.Headers.Name() != snaptype.Enums.Headers.String() { + t.Fatal("name mismatch", snaptype.Headers, snaptype.Headers.Name(), snaptype.Enums.Headers.String()) + } + + if snaptype.Bodies.Name() != snaptype.Enums.Bodies.String() { + t.Fatal("name mismatch", snaptype.Bodies, snaptype.Bodies.Name(), snaptype.Enums.Bodies.String()) + } + + if snaptype.Transactions.Name() != snaptype.Enums.Transactions.String() { + t.Fatal("name mismatch", snaptype.Transactions, snaptype.Transactions.Name(), snaptype.Enums.Transactions.String()) + } +} diff --git a/erigon-lib/chain/snapcfg/util.go b/erigon-lib/chain/snapcfg/util.go index d33d7b3ce05..51a0e84577e 100644 --- a/erigon-lib/chain/snapcfg/util.go +++ b/erigon-lib/chain/snapcfg/util.go @@ -85,7 +85,7 @@ func (p Preverified) Typed(types []snaptype.Type) Preverified { include := false for _, typ := range types { - if typeName == typ.String() { + if typeName == typ.Name() { preferredVersion = typ.Versions().Current minVersion = typ.Versions().MinSupported include = true @@ -285,7 +285,7 @@ func (c Cfg) Seedable(info snaptype.FileInfo) bool { } func (c Cfg) MergeLimit(t snaptype.Enum, fromBlock uint64) uint64 { - hasType := t == snaptype.Enums.Headers + hasType := t == snaptype.MinCoreEnum for _, p := range c.Preverified { info, _, ok := snaptype.ParseFileName("", p.Name) @@ -293,7 +293,7 @@ func (c Cfg) MergeLimit(t snaptype.Enum, fromBlock uint64) uint64 { continue } - if info.Ext != ".seg" || (t != snaptype.Enums.Unknown && t != info.Type.Enum()) { + if info.Ext != ".seg" || (t != snaptype.Unknown && t != info.Type.Enum()) { continue } @@ -322,7 +322,7 @@ func (c Cfg) MergeLimit(t snaptype.Enum, fromBlock uint64) uint64 { return snaptype.Erigon2MergeLimit } - return c.MergeLimit(snaptype.Enums.Headers, fromBlock) + return c.MergeLimit(snaptype.MinCoreEnum, fromBlock) } var knownPreverified = map[string]Preverified{ @@ -359,7 +359,7 @@ func MaxSeedableSegment(chain string, dir string) uint64 { if list, err := snaptype.Segments(dir); err == nil { for _, info := range list { - if Seedable(chain, info) && info.Type.Enum() == snaptype.Enums.Headers && info.To > max { + if Seedable(chain, info) && info.Type.Enum() == snaptype.MinCoreEnum && info.To > max { max = info.To } } diff --git a/erigon-lib/downloader/snaptype/caplin_types.go b/erigon-lib/downloader/snaptype/caplin_types.go index 7aede6112a3..adae4e7af34 100644 --- a/erigon-lib/downloader/snaptype/caplin_types.go +++ b/erigon-lib/downloader/snaptype/caplin_types.go @@ -2,20 +2,22 @@ package snaptype var ( BeaconBlocks = snapType{ - enum: Enums.BeaconBlocks, + enum: CaplinEnums.BeaconBlocks, + name: "beaconblocks", versions: Versions{ Current: 1, MinSupported: 1, }, - indexes: []Index{Indexes.BeaconBlockSlot}, + indexes: []Index{CaplinIndexes.BeaconBlockSlot}, } BlobSidecars = snapType{ - enum: Enums.BlobSidecars, + enum: CaplinEnums.BlobSidecars, + name: "blobsidecars", versions: Versions{ Current: 1, MinSupported: 1, }, - indexes: []Index{Indexes.BlobSidecarSlot}, + indexes: []Index{CaplinIndexes.BlobSidecarSlot}, } CaplinSnapshotTypes = []Type{BeaconBlocks, BlobSidecars} diff --git a/erigon-lib/downloader/snaptype/caplin_types_test.go b/erigon-lib/downloader/snaptype/caplin_types_test.go new file mode 100644 index 00000000000..d4567ea8bd5 --- /dev/null +++ b/erigon-lib/downloader/snaptype/caplin_types_test.go @@ -0,0 +1,30 @@ +package snaptype_test + +import ( + "testing" + + "github.com/ledgerwatch/erigon-lib/downloader/snaptype" +) + +func TestEnumeration(t *testing.T) { + + if snaptype.BlobSidecars.Enum() != snaptype.CaplinEnums.BlobSidecars { + t.Fatal("enum mismatch", snaptype.BlobSidecars, snaptype.BlobSidecars.Enum(), snaptype.CaplinEnums.BlobSidecars) + } + + if snaptype.BeaconBlocks.Enum() != snaptype.CaplinEnums.BeaconBlocks { + t.Fatal("enum mismatch", snaptype.BeaconBlocks, snaptype.BeaconBlocks.Enum(), snaptype.CaplinEnums.BeaconBlocks) + } +} + +func TestNames(t *testing.T) { + + if snaptype.BeaconBlocks.Name() != snaptype.CaplinEnums.BeaconBlocks.String() { + t.Fatal("name mismatch", snaptype.BeaconBlocks, snaptype.BeaconBlocks.Name(), snaptype.CaplinEnums.BeaconBlocks.String()) + } + + if snaptype.BlobSidecars.Name() != snaptype.CaplinEnums.BlobSidecars.String() { + t.Fatal("name mismatch", snaptype.BlobSidecars, snaptype.BlobSidecars.Name(), snaptype.CaplinEnums.BlobSidecars.String()) + } + +} diff --git a/erigon-lib/downloader/snaptype/files.go b/erigon-lib/downloader/snaptype/files.go index a87db3077d0..7628ff3e1e7 100644 --- a/erigon-lib/downloader/snaptype/files.go +++ b/erigon-lib/downloader/snaptype/files.go @@ -60,7 +60,7 @@ func FilterExt(in []FileInfo, expectExt string) (out []FileInfo) { } slices.SortFunc(out, func(a, b FileInfo) int { - if cmp := strings.Compare(a.Type.String(), b.Type.String()); cmp != 0 { + if cmp := strings.Compare(a.Type.Name(), b.Type.Name()); cmp != 0 { return cmp } @@ -234,7 +234,8 @@ func (f FileInfo) CompareTo(o FileInfo) int { return res } - return strings.Compare(f.Type.String(), o.Type.String()) + // this is a lexical comparison (don't use enum) + return strings.Compare(f.Type.Name(), o.Type.Name()) } func (f FileInfo) As(t Type) FileInfo { diff --git a/erigon-lib/downloader/snaptype/type.go b/erigon-lib/downloader/snaptype/type.go index 4d80dd8f895..cd8d76db83c 100644 --- a/erigon-lib/downloader/snaptype/type.go +++ b/erigon-lib/downloader/snaptype/type.go @@ -112,73 +112,22 @@ func GetIndexSalt(baseDir string) (uint32, error) { return salt, nil } -type Index int - -var Indexes = struct { - Unknown, - HeaderHash, - BodyHash, - TxnHash, - TxnHash2BlockNum, - BorTxnHash, - BorSpanId, - BorCheckpointId, - BorMilestoneId, +type Index struct { + Name string + Offset int +} + +var CaplinIndexes = struct { BeaconBlockSlot, BlobSidecarSlot Index }{ - Unknown: -1, - HeaderHash: 0, - BodyHash: 1, - TxnHash: 2, - TxnHash2BlockNum: 3, - BorTxnHash: 4, - BorSpanId: 5, - BorCheckpointId: 6, - BorMilestoneId: 7, - BeaconBlockSlot: 8, - BlobSidecarSlot: 9, -} - -func (i Index) Offset() int { - switch i { - case Indexes.TxnHash2BlockNum: - return 1 - default: - return 0 - } -} - -func (i Index) String() string { - switch i { - case Indexes.HeaderHash: - return Enums.Headers.String() - case Indexes.BodyHash: - return Enums.Bodies.String() - case Indexes.TxnHash: - return Enums.Transactions.String() - case Indexes.TxnHash2BlockNum: - return "transactions-to-block" - case Indexes.BorTxnHash: - return Enums.BorEvents.String() - case Indexes.BorSpanId: - return Enums.BorSpans.String() - case Indexes.BorCheckpointId: - return Enums.BorCheckpoints.String() - case Indexes.BorMilestoneId: - return Enums.BorMilestones.String() - case Indexes.BeaconBlockSlot: - return Enums.BeaconBlocks.String() - case Indexes.BlobSidecarSlot: - return Enums.BlobSidecars.String() - default: - panic(fmt.Sprintf("unknown index: %d", i)) - } + BeaconBlockSlot: Index{Name: "beaconblocks"}, + BlobSidecarSlot: Index{Name: "blocksidecars"}, } func (i Index) HasFile(info FileInfo, logger log.Logger) bool { dir := info.Dir() - fName := IdxFileName(info.Version, info.From, info.To, i.String()) + fName := IdxFileName(info.Version, info.From, info.To, i.Name) segment, err := seg.NewDecompressor(info.Path) @@ -202,7 +151,7 @@ func (i Index) HasFile(info FileInfo, logger log.Logger) bool { type Type interface { Enum() Enum Versions() Versions - String() string + Name() string FileName(version Version, from uint64, to uint64) string FileInfo(dir string, from uint64, to uint64) FileInfo IdxFileName(version Version, from uint64, to uint64, index ...Index) string @@ -215,20 +164,26 @@ type Type interface { type snapType struct { enum Enum + name string versions Versions indexes []Index indexBuilder IndexBuilder rangeExtractor RangeExtractor } +// These are raw maps with no mutex protection becuase they are +// expected to be written to once during program initialization +// and them be readonly var registeredTypes = map[Enum]Type{} +var namedTypes = map[string]Type{} -func RegisterType(enum Enum, versions Versions, rangeExtractor RangeExtractor, indexes []Index, indexBuilder IndexBuilder) Type { +func RegisterType(enum Enum, name string, versions Versions, rangeExtractor RangeExtractor, indexes []Index, indexBuilder IndexBuilder) Type { t := snapType{ - enum: enum, versions: versions, indexes: indexes, rangeExtractor: rangeExtractor, indexBuilder: indexBuilder, + enum: enum, name: name, versions: versions, indexes: indexes, rangeExtractor: rangeExtractor, indexBuilder: indexBuilder, } registeredTypes[enum] = t + namedTypes[strings.ToLower(name)] = t return t } @@ -241,8 +196,12 @@ func (s snapType) Versions() Versions { return s.versions } +func (s snapType) Name() string { + return s.name +} + func (s snapType) String() string { - return s.enum.String() + return s.Name() } func (s snapType) FileName(version Version, from uint64, to uint64) string { @@ -289,7 +248,7 @@ func (s snapType) HasIndexFiles(info FileInfo, logger log.Logger) bool { func (s snapType) IdxFileNames(version Version, from uint64, to uint64) []string { fileNames := make([]string, len(s.indexes)) for i, index := range s.indexes { - fileNames[i] = IdxFileName(version, from, to, index.String()) + fileNames[i] = IdxFileName(version, from, to, index.Name) } return fileNames @@ -318,7 +277,7 @@ func (s snapType) IdxFileName(version Version, from uint64, to uint64, index ... } } - return IdxFileName(version, from, to, index[0].String()) + return IdxFileName(version, from, to, index[0].Name) } func ParseFileType(s string) (Type, bool) { @@ -333,60 +292,46 @@ func ParseFileType(s string) (Type, bool) { type Enum int -var Enums = struct { - Unknown, - Headers, - Bodies, - Transactions, - BorEvents, - BorSpans, - BorCheckpoints, - BorMilestones, +const Unknown Enum = 0 + +type Enums struct { + Unknown Enum +} + +const MinCoreEnum = 1 +const MinBorEnum = 4 +const MinCaplinEnum = 8 + +var CaplinEnums = struct { + Enums BeaconBlocks, BlobSidecars Enum }{ - Unknown: -1, - Headers: 0, - Bodies: 1, - Transactions: 2, - BorEvents: 3, - BorSpans: 4, - BorCheckpoints: 5, - BorMilestones: 6, - BeaconBlocks: 7, - BlobSidecars: 8, + Enums: Enums{}, + BeaconBlocks: MinCaplinEnum, + BlobSidecars: MinCaplinEnum + 1, } func (ft Enum) String() string { switch ft { - case Enums.Headers: - return "headers" - case Enums.Bodies: - return "bodies" - case Enums.Transactions: - return "transactions" - case Enums.BorEvents: - return "borevents" - case Enums.BorSpans: - return "borspans" - case Enums.BorCheckpoints: - return "borcheckpoints" - case Enums.BorMilestones: - return "bormilestones" - case Enums.BeaconBlocks: + case CaplinEnums.BeaconBlocks: return "beaconblocks" - case Enums.BlobSidecars: + case CaplinEnums.BlobSidecars: return "blobsidecars" default: + if t, ok := registeredTypes[ft]; ok { + return t.Name() + } + panic(fmt.Sprintf("unknown file type: %d", ft)) } } func (ft Enum) Type() Type { switch ft { - case Enums.BeaconBlocks: + case CaplinEnums.BeaconBlocks: return BeaconBlocks - case Enums.BlobSidecars: + case CaplinEnums.BlobSidecars: return BlobSidecars default: return registeredTypes[ft] @@ -411,27 +356,17 @@ func (e Enum) BuildIndexes(ctx context.Context, info FileInfo, chainConfig *chai } func ParseEnum(s string) (Enum, bool) { + s = strings.ToLower(s) switch s { - case "headers": - return Enums.Headers, true - case "bodies": - return Enums.Bodies, true - case "transactions": - return Enums.Transactions, true - case "borevents": - return Enums.BorEvents, true - case "borspans": - return Enums.BorSpans, true - case "borcheckpoints": - return Enums.BorCheckpoints, true - case "bormilestones": - return Enums.BorMilestones, true case "beaconblocks": - return Enums.BeaconBlocks, true + return CaplinEnums.BeaconBlocks, true case "blobsidecars": - return Enums.BlobSidecars, true + return CaplinEnums.BlobSidecars, true default: - return Enums.Unknown, false + if t, ok := namedTypes[s]; ok { + return t.Enum(), true + } + return Enums{}.Unknown, false } } @@ -510,7 +445,7 @@ func BuildIndex(ctx context.Context, info FileInfo, salt uint32, firstDataId uin func ExtractRange(ctx context.Context, f FileInfo, extractor RangeExtractor, firstKey FirstKeyGetter, chainDB kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) { var lastKeyValue uint64 - sn, err := seg.NewCompressor(ctx, "Snapshot "+f.Type.String(), f.Path, tmpDir, seg.MinPatternScore, workers, log.LvlTrace, logger) + sn, err := seg.NewCompressor(ctx, "Snapshot "+f.Type.Name(), f.Path, tmpDir, seg.MinPatternScore, workers, log.LvlTrace, logger) if err != nil { return lastKeyValue, err diff --git a/eth/backend.go b/eth/backend.go index 1aa7dd52211..94c77703f22 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -796,13 +796,13 @@ func New(ctx context.Context, stack *node.Node, config *ethconfig.Config, logger if !config.Sync.UseSnapshots && backend.downloaderClient != nil { for _, p := range blockReader.AllTypes() { backend.downloaderClient.ProhibitNewDownloads(ctx, &protodownloader.ProhibitNewDownloadsRequest{ - Type: p.String(), + Type: p.Name(), }) } for _, p := range snaptype.CaplinSnapshotTypes { backend.downloaderClient.ProhibitNewDownloads(ctx, &protodownloader.ProhibitNewDownloadsRequest{ - Type: p.String(), + Type: p.Name(), }) } diff --git a/eth/stagedsync/stage_snapshots.go b/eth/stagedsync/stage_snapshots.go index 318e041570e..2081faf3258 100644 --- a/eth/stagedsync/stage_snapshots.go +++ b/eth/stagedsync/stage_snapshots.go @@ -38,10 +38,12 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" "github.com/ledgerwatch/erigon-lib/state" "github.com/ledgerwatch/erigon/core/rawdb" + coresnaptype "github.com/ledgerwatch/erigon/core/snaptype" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" "github.com/ledgerwatch/erigon/eth/stagedsync/stages" + borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype" "github.com/ledgerwatch/erigon/rpc" "github.com/ledgerwatch/erigon/turbo/services" "github.com/ledgerwatch/erigon/turbo/shards" @@ -537,14 +539,14 @@ func (u *snapshotUploader) maxUploadedHeader() uint64 { for _, state := range u.files { if state.local && state.remote { if state.info != nil { - if state.info.Type.Enum() == snaptype.Enums.Headers { + if state.info.Type.Enum() == coresnaptype.Enums.Headers { if state.info.To > max { max = state.info.To } } } else { if info, _, ok := snaptype.ParseFileName(u.cfg.dirs.Snap, state.file); ok { - if info.Type.Enum() == snaptype.Enums.Headers { + if info.Type.Enum() == coresnaptype.Enums.Headers { if info.To > max { max = info.To } @@ -1016,8 +1018,8 @@ func (u *snapshotUploader) removeBefore(before uint64) { for _, f := range list { if f.To > before { switch f.Type.Enum() { - case snaptype.Enums.BorEvents, snaptype.Enums.BorSpans, - snaptype.Enums.BorCheckpoints, snaptype.Enums.BorMilestones: + case borsnaptype.Enums.BorEvents, borsnaptype.Enums.BorSpans, + borsnaptype.Enums.BorCheckpoints, borsnaptype.Enums.BorMilestones: borToReopen = append(borToReopen, filepath.Base(f.Path)) default: toReopen = append(toReopen, filepath.Base(f.Path)) diff --git a/migrations/prohibit_new_downloads2.go b/migrations/prohibit_new_downloads2.go index 114cd64a0b1..22cd00372a9 100644 --- a/migrations/prohibit_new_downloads2.go +++ b/migrations/prohibit_new_downloads2.go @@ -42,16 +42,16 @@ var ProhibitNewDownloadsLock2 = Migration{ locked := []string{} for _, t := range coresnaptype.BlockSnapshotTypes { - locked = append(locked, t.String()) + locked = append(locked, t.Name()) } for _, t := range borsnaptype.BorSnapshotTypes { - locked = append(locked, t.String()) + locked = append(locked, t.Name()) } for _, t := range snaptype.CaplinSnapshotTypes { - if t.String() != snaptype.BlobSidecars.String() { - locked = append(locked, t.String()) + if t.Name() != snaptype.BlobSidecars.Name() { + locked = append(locked, t.Name()) } } diff --git a/p2p/sentry/simulator/sentry_simulator.go b/p2p/sentry/simulator/sentry_simulator.go index 6b0663f8bd5..effc0995860 100644 --- a/p2p/sentry/simulator/sentry_simulator.go +++ b/p2p/sentry/simulator/sentry_simulator.go @@ -440,7 +440,7 @@ func (s *server) getHeaderByHash(ctx context.Context, hash common.Hash) (*corety } func (s *server) downloadHeaders(ctx context.Context, header *freezeblocks.Segment) error { - fileName := snaptype.SegmentFileName(0, header.From(), header.To(), snaptype.Enums.Headers) + fileName := snaptype.SegmentFileName(0, header.From(), header.To(), coresnaptype.Enums.Headers) session := sync.NewTorrentSession(s.downloader, s.chain) s.logger.Info("Downloading", "file", fileName) diff --git a/polygon/bor/snaptype/types.go b/polygon/bor/snaptype/types.go index 86ff9caa662..5a2e32b273d 100644 --- a/polygon/bor/snaptype/types.go +++ b/polygon/bor/snaptype/types.go @@ -38,9 +38,36 @@ func init() { snapcfg.RegisterKnownTypes(networkname.BorMainnetChainName, borTypes) } +var Enums = struct { + snaptype.Enums + BorEvents, + BorSpans, + BorCheckpoints, + BorMilestones snaptype.Enum +}{ + Enums: snaptype.Enums{}, + BorEvents: snaptype.MinBorEnum, + BorSpans: snaptype.MinBorEnum + 1, + BorCheckpoints: snaptype.MinBorEnum + 2, + BorMilestones: snaptype.MinBorEnum + 3, +} + +var Indexes = struct { + BorTxnHash, + BorSpanId, + BorCheckpointId, + BorMilestoneId snaptype.Index +}{ + BorTxnHash: snaptype.Index{Name: "borevents"}, + BorSpanId: snaptype.Index{Name: "borspans"}, + BorCheckpointId: snaptype.Index{Name: "borcheckpoints"}, + BorMilestoneId: snaptype.Index{Name: "bormilestones"}, +} + var ( BorEvents = snaptype.RegisterType( - snaptype.Enums.BorEvents, + Enums.BorEvents, + "borevents", snaptype.Versions{ Current: 1, //2, MinSupported: 1, @@ -108,7 +135,7 @@ var ( return lastEventId, nil }), - []snaptype.Index{snaptype.Indexes.BorTxnHash}, + []snaptype.Index{Indexes.BorTxnHash}, snaptype.IndexBuilderFunc( func(ctx context.Context, sn snaptype.FileInfo, salt uint32, chainConfig *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { defer func() { @@ -151,7 +178,7 @@ var ( BucketSize: 2000, LeafSize: 8, TmpDir: tmpDir, - IndexFile: filepath.Join(sn.Dir(), snaptype.IdxFileName(sn.Version, sn.From, sn.To, snaptype.Enums.BorEvents.String())), + IndexFile: filepath.Join(sn.Dir(), snaptype.IdxFileName(sn.Version, sn.From, sn.To, Enums.BorEvents.String())), BaseDataID: baseEventId, }, logger) if err != nil { @@ -198,7 +225,8 @@ var ( })) BorSpans = snaptype.RegisterType( - snaptype.Enums.BorSpans, + Enums.BorSpans, + "borspans", snaptype.Versions{ Current: 1, //2, MinSupported: 1, @@ -209,7 +237,7 @@ var ( spanTo := uint64(heimdall.SpanIdAt(blockTo)) return extractValueRange(ctx, kv.BorSpans, spanFrom, spanTo, db, collect, workers, lvl, logger) }), - []snaptype.Index{snaptype.Indexes.BorSpanId}, + []snaptype.Index{Indexes.BorSpanId}, snaptype.IndexBuilderFunc( func(ctx context.Context, sn snaptype.FileInfo, salt uint32, _ *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { d, err := seg.NewDecompressor(sn.Path) @@ -226,7 +254,8 @@ var ( ) BorCheckpoints = snaptype.RegisterType( - snaptype.Enums.BorCheckpoints, + Enums.BorCheckpoints, + "borcheckpoints", snaptype.Versions{ Current: 1, //2, MinSupported: 1, @@ -270,7 +299,7 @@ var ( return extractValueRange(ctx, kv.BorCheckpoints, uint64(checkpointFrom), uint64(checkpointTo), db, collect, workers, lvl, logger) }), - []snaptype.Index{snaptype.Indexes.BorCheckpointId}, + []snaptype.Index{Indexes.BorCheckpointId}, snaptype.IndexBuilderFunc( func(ctx context.Context, sn snaptype.FileInfo, salt uint32, _ *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { d, err := seg.NewDecompressor(sn.Path) @@ -300,7 +329,8 @@ var ( ) BorMilestones = snaptype.RegisterType( - snaptype.Enums.BorMilestones, + Enums.BorMilestones, + "bormilestones", snaptype.Versions{ Current: 1, //2, MinSupported: 1, @@ -344,7 +374,7 @@ var ( return extractValueRange(ctx, kv.BorMilestones, uint64(milestoneFrom), uint64(milestoneTo), db, collect, workers, lvl, logger) }), - []snaptype.Index{snaptype.Indexes.BorMilestoneId}, + []snaptype.Index{Indexes.BorMilestoneId}, snaptype.IndexBuilderFunc( func(ctx context.Context, sn snaptype.FileInfo, salt uint32, _ *chain.Config, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) { d, err := seg.NewDecompressor(sn.Path) diff --git a/polygon/bor/snaptype/types_test.go b/polygon/bor/snaptype/types_test.go new file mode 100644 index 00000000000..c1e9431bfbd --- /dev/null +++ b/polygon/bor/snaptype/types_test.go @@ -0,0 +1,45 @@ +package snaptype_test + +import ( + "testing" + + "github.com/ledgerwatch/erigon/polygon/bor/snaptype" +) + +func TestEnumeration(t *testing.T) { + + if snaptype.BorEvents.Enum() != snaptype.Enums.BorEvents { + t.Fatal("enum mismatch", snaptype.BorEvents, snaptype.BorEvents.Enum(), snaptype.Enums.BorEvents) + } + + if snaptype.BorSpans.Enum() != snaptype.Enums.BorSpans { + t.Fatal("enum mismatch", snaptype.BorSpans, snaptype.BorSpans.Enum(), snaptype.Enums.BorSpans) + } + + if snaptype.BorCheckpoints.Enum() != snaptype.Enums.BorCheckpoints { + t.Fatal("enum mismatch", snaptype.BorCheckpoints, snaptype.BorCheckpoints.Enum(), snaptype.Enums.BorCheckpoints) + } + + if snaptype.BorMilestones.Enum() != snaptype.Enums.BorMilestones { + t.Fatal("enum mismatch", snaptype.BorMilestones, snaptype.BorMilestones.Enum(), snaptype.Enums.BorMilestones) + } +} + +func TestNames(t *testing.T) { + + if snaptype.BorEvents.Name() != snaptype.Enums.BorEvents.String() { + t.Fatal("name mismatch", snaptype.BorEvents, snaptype.BorEvents.Name(), snaptype.Enums.BorEvents.String()) + } + + if snaptype.BorSpans.Name() != snaptype.Enums.BorSpans.String() { + t.Fatal("name mismatch", snaptype.BorSpans, snaptype.BorSpans.Name(), snaptype.Enums.BorSpans.String()) + } + + if snaptype.BorCheckpoints.Name() != snaptype.Enums.BorCheckpoints.String() { + t.Fatal("name mismatch", snaptype.BorCheckpoints, snaptype.BorCheckpoints.Name(), snaptype.Enums.BorCheckpoints.String()) + } + + if snaptype.BorMilestones.Name() != snaptype.Enums.BorMilestones.String() { + t.Fatal("name mismatch", snaptype.BorMilestones, snaptype.BorMilestones.Name(), snaptype.Enums.BorMilestones.String()) + } +} diff --git a/polygon/heimdall/simulator/heimdall_simulator.go b/polygon/heimdall/simulator/heimdall_simulator.go index ce87c8c6a6e..ef98c2c19d3 100644 --- a/polygon/heimdall/simulator/heimdall_simulator.go +++ b/polygon/heimdall/simulator/heimdall_simulator.go @@ -72,14 +72,14 @@ func NewHeimdall(ctx context.Context, chain string, snapshotLocation string, log for _, file := range localFiles { info, _, _ := snaptype.ParseFileName(torrentDir, file.Name()) if info.Ext == ".seg" { - if info.Type.Enum() == snaptype.Enums.BorSpans { + if info.Type.Enum() == borsnaptype.Enums.BorSpans { err = info.Type.BuildIndexes(ctx, info, nil, torrentDir, nil, log.LvlWarn, logger) if err != nil { return HeimdallSimulator{}, err } } - if info.Type.Enum() == snaptype.Enums.BorEvents { + if info.Type.Enum() == borsnaptype.Enums.BorEvents { err = info.Type.BuildIndexes(ctx, info, nil, torrentDir, nil, log.LvlWarn, logger) if err != nil { return HeimdallSimulator{}, err diff --git a/turbo/app/snapshots_cmd.go b/turbo/app/snapshots_cmd.go index ed7764e8d23..ad4b2770a95 100644 --- a/turbo/app/snapshots_cmd.go +++ b/turbo/app/snapshots_cmd.go @@ -25,7 +25,6 @@ import ( "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dbg" "github.com/ledgerwatch/erigon-lib/common/dir" - "github.com/ledgerwatch/erigon-lib/downloader/snaptype" "github.com/ledgerwatch/erigon-lib/etl" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/kvcfg" @@ -38,6 +37,7 @@ import ( "github.com/ledgerwatch/erigon/cmd/utils" "github.com/ledgerwatch/erigon/core/rawdb" "github.com/ledgerwatch/erigon/core/rawdb/blockio" + coresnaptype "github.com/ledgerwatch/erigon/core/snaptype" "github.com/ledgerwatch/erigon/diagnostics" "github.com/ledgerwatch/erigon/eth/ethconfig" "github.com/ledgerwatch/erigon/eth/ethconfig/estimate" @@ -577,7 +577,7 @@ func doRetireCommand(cliCtx *cli.Context) error { return err }) blockReader, _ := br.IO() - from2, to2, ok := freezeblocks.CanRetire(forwardProgress, blockReader.FrozenBlocks(), snaptype.Enums.Headers, nil) + from2, to2, ok := freezeblocks.CanRetire(forwardProgress, blockReader.FrozenBlocks(), coresnaptype.Enums.Headers, nil) if ok { from, to, every = from2, to2, to2-from2 } diff --git a/turbo/snapshotsync/freezeblocks/block_reader.go b/turbo/snapshotsync/freezeblocks/block_reader.go index 6d30370d89a..6d292763874 100644 --- a/turbo/snapshotsync/freezeblocks/block_reader.go +++ b/turbo/snapshotsync/freezeblocks/block_reader.go @@ -24,6 +24,7 @@ import ( "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/recsplit" "github.com/ledgerwatch/erigon/core/rawdb" + coresnaptype "github.com/ledgerwatch/erigon/core/snaptype" "github.com/ledgerwatch/erigon/core/types" "github.com/ledgerwatch/erigon/eth/ethconfig" bortypes "github.com/ledgerwatch/erigon/polygon/bor/types" @@ -792,7 +793,7 @@ func (r *BlockReader) txsFromSnapshot(baseTxnID uint64, txsAmount uint32, txsSeg } }() // avoid crash because Erigon's core does many things - idxTxnHash := txsSeg.Index(snaptype.Indexes.TxnHash) + idxTxnHash := txsSeg.Index(coresnaptype.Indexes.TxnHash) if idxTxnHash == nil { return nil, nil, nil @@ -830,7 +831,7 @@ func (r *BlockReader) txsFromSnapshot(baseTxnID uint64, txsAmount uint32, txsSeg } func (r *BlockReader) txnByID(txnID uint64, sn *Segment, buf []byte) (txn types.Transaction, err error) { - idxTxnHash := sn.Index(snaptype.Indexes.TxnHash) + idxTxnHash := sn.Index(coresnaptype.Indexes.TxnHash) offset := idxTxnHash.OrdinalLookup(txnID - idxTxnHash.BaseDataID()) gg := sn.MakeGetter() @@ -853,8 +854,8 @@ func (r *BlockReader) txnByHash(txnHash common.Hash, segments []*Segment, buf [] for i := len(segments) - 1; i >= 0; i-- { sn := segments[i] - idxTxnHash := sn.Index(snaptype.Indexes.TxnHash) - idxTxnHash2BlockNum := sn.Index(snaptype.Indexes.TxnHash2BlockNum) + idxTxnHash := sn.Index(coresnaptype.Indexes.TxnHash) + idxTxnHash2BlockNum := sn.Index(coresnaptype.Indexes.TxnHash2BlockNum) if idxTxnHash == nil || idxTxnHash2BlockNum == nil { continue @@ -969,7 +970,7 @@ func (r *BlockReader) FirstTxnNumNotInSnapshots() uint64 { return 0 } - lastTxnID := sn.Index(snaptype.Indexes.TxnHash).BaseDataID() + uint64(sn.Count()) + lastTxnID := sn.Index(coresnaptype.Indexes.TxnHash).BaseDataID() + uint64(sn.Count()) return lastTxnID } diff --git a/turbo/snapshotsync/freezeblocks/block_reader_test.go b/turbo/snapshotsync/freezeblocks/block_reader_test.go index f2a849e798f..2b3309fd79c 100644 --- a/turbo/snapshotsync/freezeblocks/block_reader_test.go +++ b/turbo/snapshotsync/freezeblocks/block_reader_test.go @@ -25,7 +25,7 @@ func TestBlockReaderLastFrozenSpanIdWhenSegmentFilesArePresent(t *testing.T) { logger := testlog.Logger(t, log.LvlInfo) dir := t.TempDir() createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) defer borRoSnapshots.Close() err := borRoSnapshots.ReopenFolder() @@ -57,11 +57,11 @@ func TestBlockReaderLastFrozenSpanIdReturnsLastSegWithIdx(t *testing.T) { createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 500_000, 1_000_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 500_000, 1_000_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) // delete idx file for last bor span segment to simulate segment with missing idx file - idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorSpans.String())) + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorSpans.Name())) err := os.Remove(idxFileToDelete) require.NoError(t, err) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) @@ -81,17 +81,17 @@ func TestBlockReaderLastFrozenSpanIdReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *te createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 500_000, 1_000_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 500_000, 1_000_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) // delete idx file for all bor span segments to simulate segments with missing idx files - idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1, 500_000, borsnaptype.BorSpans.String())) + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1, 500_000, borsnaptype.BorSpans.Name())) err := os.Remove(idxFileToDelete) require.NoError(t, err) - idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 500_000, 1_000_000, borsnaptype.BorSpans.String())) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 500_000, 1_000_000, borsnaptype.BorSpans.Name())) err = os.Remove(idxFileToDelete) require.NoError(t, err) - idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorSpans.String())) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorSpans.Name())) err = os.Remove(idxFileToDelete) require.NoError(t, err) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) @@ -109,7 +109,7 @@ func TestBlockReaderLastFrozenEventIdWhenSegmentFilesArePresent(t *testing.T) { logger := testlog.Logger(t, log.LvlInfo) dir := t.TempDir() createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) defer borRoSnapshots.Close() err := borRoSnapshots.ReopenFolder() @@ -141,11 +141,11 @@ func TestBlockReaderLastFrozenEventIdReturnsLastSegWithIdx(t *testing.T) { createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 500_000, 1_000_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 500_000, 1_000_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) // delete idx file for last bor events segment to simulate segment with missing idx file - idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorEvents.String())) + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorEvents.Name())) err := os.Remove(idxFileToDelete) require.NoError(t, err) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) @@ -165,17 +165,17 @@ func TestBlockReaderLastFrozenEventIdReturnsZeroWhenAllSegmentsDoNotHaveIdx(t *t createTestBorEventSegmentFile(t, 0, 500_000, 132, dir, logger) createTestBorEventSegmentFile(t, 500_000, 1_000_000, 264, dir, logger) createTestBorEventSegmentFile(t, 1_000_000, 1_500_000, 528, dir, logger) - createTestSegmentFile(t, 0, 500_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 500_000, 1_000_000, snaptype.Enums.BorSpans, dir, 1, logger) - createTestSegmentFile(t, 1_000_000, 1_500_000, snaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 0, 500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 500_000, 1_000_000, borsnaptype.Enums.BorSpans, dir, 1, logger) + createTestSegmentFile(t, 1_000_000, 1_500_000, borsnaptype.Enums.BorSpans, dir, 1, logger) // delete idx files for all bor events segment to simulate segment files with missing idx files - idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 0, 500_000, borsnaptype.BorEvents.String())) + idxFileToDelete := filepath.Join(dir, snaptype.IdxFileName(1, 0, 500_000, borsnaptype.BorEvents.Name())) err := os.Remove(idxFileToDelete) require.NoError(t, err) - idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 500_000, 1_000_000, borsnaptype.BorEvents.String())) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 500_000, 1_000_000, borsnaptype.BorEvents.Name())) err = os.Remove(idxFileToDelete) require.NoError(t, err) - idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorEvents.String())) + idxFileToDelete = filepath.Join(dir, snaptype.IdxFileName(1, 1_000_000, 1_500_000, borsnaptype.BorEvents.Name())) err = os.Remove(idxFileToDelete) require.NoError(t, err) borRoSnapshots := NewBorRoSnapshots(ethconfig.BlocksFreezing{Enabled: true}, dir, 0, logger) @@ -191,7 +191,7 @@ func createTestBorEventSegmentFile(t *testing.T, from, to, eventId uint64, dir s compressor, err := seg.NewCompressor( context.Background(), "test", - filepath.Join(dir, snaptype.SegmentFileName(1, from, to, snaptype.Enums.BorEvents)), + filepath.Join(dir, snaptype.SegmentFileName(1, from, to, borsnaptype.Enums.BorEvents)), dir, 100, 1, @@ -212,7 +212,7 @@ func createTestBorEventSegmentFile(t *testing.T, from, to, eventId uint64, dir s KeyCount: 1, BucketSize: 10, TmpDir: dir, - IndexFile: filepath.Join(dir, snaptype.IdxFileName(1, from, to, borsnaptype.BorEvents.String())), + IndexFile: filepath.Join(dir, snaptype.IdxFileName(1, from, to, borsnaptype.BorEvents.Name())), LeafSize: 8, }, logger, diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go index 7fbdc820612..a0e97cc5d8e 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go @@ -83,14 +83,14 @@ func (s Segment) Version() snaptype.Version { func (s Segment) Index(index ...snaptype.Index) *recsplit.Index { if len(index) == 0 { - index = []snaptype.Index{0} + index = []snaptype.Index{{}} } - if len(s.indexes) <= index[0].Offset() { + if len(s.indexes) <= index[0].Offset { return nil } - return s.indexes[index[0].Offset()] + return s.indexes[index[0].Offset] } func (s Segment) IsIndexed() bool { @@ -213,9 +213,9 @@ func (sn *Segment) mappedBodySnapshot() *silkworm.MappedBodySnapshot { func (sn *Segment) mappedTxnSnapshot() *silkworm.MappedTxnSnapshot { segmentRegion := silkworm.NewMemoryMappedRegion(sn.FilePath(), sn.DataHandle(), sn.Size()) - idxTxnHash := sn.Index(snaptype.Indexes.TxnHash) + idxTxnHash := sn.Index(coresnaptype.Indexes.TxnHash) idxTxnHashRegion := silkworm.NewMemoryMappedRegion(idxTxnHash.FilePath(), idxTxnHash.DataHandle(), idxTxnHash.Size()) - idxTxnHash2BlockNum := sn.Index(snaptype.Indexes.TxnHash2BlockNum) + idxTxnHash2BlockNum := sn.Index(coresnaptype.Indexes.TxnHash2BlockNum) idxTxnHash2BlockRegion := silkworm.NewMemoryMappedRegion(idxTxnHash2BlockNum.FilePath(), idxTxnHash2BlockNum.DataHandle(), idxTxnHash2BlockNum.Size()) return silkworm.NewMappedTxnSnapshot(segmentRegion, idxTxnHashRegion, idxTxnHash2BlockRegion) } @@ -859,7 +859,7 @@ func (s *RoSnapshots) PrintDebug() { func (s *RoSnapshots) AddSnapshotsToSilkworm(silkwormInstance *silkworm.Silkworm) error { mappedHeaderSnapshots := make([]*silkworm.MappedHeaderSnapshot, 0) - if headers, ok := s.segments.Get(snaptype.Enums.Headers); ok { + if headers, ok := s.segments.Get(coresnaptype.Enums.Headers); ok { err := headers.View(func(segments []*Segment) error { for _, headerSegment := range segments { mappedHeaderSnapshots = append(mappedHeaderSnapshots, headerSegment.mappedHeaderSnapshot()) @@ -872,7 +872,7 @@ func (s *RoSnapshots) AddSnapshotsToSilkworm(silkwormInstance *silkworm.Silkworm } mappedBodySnapshots := make([]*silkworm.MappedBodySnapshot, 0) - if bodies, ok := s.segments.Get(snaptype.Enums.Bodies); ok { + if bodies, ok := s.segments.Get(coresnaptype.Enums.Bodies); ok { err := bodies.View(func(segments []*Segment) error { for _, bodySegment := range segments { mappedBodySnapshots = append(mappedBodySnapshots, bodySegment.mappedBodySnapshot()) @@ -885,7 +885,7 @@ func (s *RoSnapshots) AddSnapshotsToSilkworm(silkwormInstance *silkworm.Silkworm } mappedTxnSnapshots := make([]*silkworm.MappedTxnSnapshot, 0) - if txs, ok := s.segments.Get(snaptype.Enums.Transactions); ok { + if txs, ok := s.segments.Get(coresnaptype.Enums.Transactions); ok { err := txs.View(func(segments []*Segment) error { for _, txnSegment := range segments { mappedTxnSnapshots = append(mappedTxnSnapshots, txnSegment.mappedTxnSnapshot()) @@ -1061,10 +1061,10 @@ func SegmentsCaplin(dir string, minBlock uint64) (res []snaptype.FileInfo, missi var l, lSidecars []snaptype.FileInfo var m []Range for _, f := range list { - if f.Type.Enum() != snaptype.Enums.BeaconBlocks && f.Type.Enum() != snaptype.Enums.BlobSidecars { + if f.Type.Enum() != snaptype.CaplinEnums.BeaconBlocks && f.Type.Enum() != snaptype.CaplinEnums.BlobSidecars { continue } - if f.Type.Enum() == snaptype.Enums.BlobSidecars { + if f.Type.Enum() == snaptype.CaplinEnums.BlobSidecars { lSidecars = append(lSidecars, f) // blobs are an exception continue } @@ -1073,7 +1073,7 @@ func SegmentsCaplin(dir string, minBlock uint64) (res []snaptype.FileInfo, missi l, m = noGaps(noOverlaps(l), minBlock) if len(m) > 0 { lst := m[len(m)-1] - log.Debug("[snapshots] see gap", "type", snaptype.Enums.BeaconBlocks, "from", lst.from) + log.Debug("[snapshots] see gap", "type", snaptype.CaplinEnums.BeaconBlocks, "from", lst.from) } res = append(res, l...) res = append(res, lSidecars...) @@ -1278,7 +1278,7 @@ func (br *BlockRetire) retireBlocks(ctx context.Context, minBlockNum uint64, max notifier, logger, blockReader, tmpDir, db, workers := br.notifier, br.logger, br.blockReader, br.tmpDir, br.db, br.workers snapshots := br.snapshots() - blockFrom, blockTo, ok := CanRetire(maxBlockNum, minBlockNum, snaptype.Enums.Unknown, br.chainConfig) + blockFrom, blockTo, ok := CanRetire(maxBlockNum, minBlockNum, snaptype.Unknown, br.chainConfig) if ok { if has, err := br.dbHasEnoughDataForBlocksRetire(ctx); err != nil { @@ -1446,8 +1446,8 @@ func (br *BlockRetire) BuildMissedIndicesIfNeed(ctx context.Context, logPrefix s func DumpBlocks(ctx context.Context, blockFrom, blockTo uint64, chainConfig *chain.Config, tmpDir, snapDir string, chainDB kv.RoDB, workers int, lvl log.Lvl, logger log.Logger, blockReader services.FullBlockReader) error { firstTxNum := blockReader.FirstTxnNumNotInSnapshots() - for i := blockFrom; i < blockTo; i = chooseSegmentEnd(i, blockTo, snaptype.Enums.Headers, chainConfig) { - lastTxNum, err := dumpBlocksRange(ctx, i, chooseSegmentEnd(i, blockTo, snaptype.Enums.Headers, chainConfig), tmpDir, snapDir, firstTxNum, chainDB, chainConfig, workers, lvl, logger) + for i := blockFrom; i < blockTo; i = chooseSegmentEnd(i, blockTo, coresnaptype.Enums.Headers, chainConfig) { + lastTxNum, err := dumpBlocksRange(ctx, i, chooseSegmentEnd(i, blockTo, coresnaptype.Enums.Headers, chainConfig), tmpDir, snapDir, firstTxNum, chainDB, chainConfig, workers, lvl, logger) if err != nil { return err } @@ -1484,7 +1484,7 @@ type dumpFunc func(ctx context.Context, db kv.RoDB, chainConfig *chain.Config, b func dumpRange(ctx context.Context, f snaptype.FileInfo, dumper dumpFunc, firstKey firstKeyGetter, chainDB kv.RoDB, chainConfig *chain.Config, tmpDir string, workers int, lvl log.Lvl, logger log.Logger) (uint64, error) { var lastKeyValue uint64 - sn, err := seg.NewCompressor(ctx, "Snapshot "+f.Type.String(), f.Path, tmpDir, seg.MinPatternScore, workers, log.LvlTrace, logger) + sn, err := seg.NewCompressor(ctx, "Snapshot "+f.Type.Name(), f.Path, tmpDir, seg.MinPatternScore, workers, log.LvlTrace, logger) if err != nil { return lastKeyValue, err @@ -1888,11 +1888,11 @@ func (m *Merger) DisableFsync() { m.noFsync = true } func (m *Merger) FindMergeRanges(currentRanges []Range, maxBlockNum uint64) (toMerge []Range) { for i := len(currentRanges) - 1; i > 0; i-- { r := currentRanges[i] - mergeLimit := snapcfg.MergeLimit(m.chainConfig.ChainName, snaptype.Enums.Unknown, r.from) + mergeLimit := snapcfg.MergeLimit(m.chainConfig.ChainName, snaptype.Unknown, r.from) if r.to-r.from >= mergeLimit { continue } - for _, span := range snapcfg.MergeSteps(m.chainConfig.ChainName, snaptype.Enums.Unknown, r.from) { + for _, span := range snapcfg.MergeSteps(m.chainConfig.ChainName, snaptype.Unknown, r.from) { if r.to%span != 0 { continue } @@ -2051,7 +2051,7 @@ func removeOldFiles(toDel []string, snapDir string) { ext := filepath.Ext(f) withoutExt := f[:len(f)-len(ext)] _ = os.Remove(withoutExt + ".idx") - isTxnType := strings.HasSuffix(withoutExt, coresnaptype.Transactions.String()) + isTxnType := strings.HasSuffix(withoutExt, coresnaptype.Transactions.Name()) if isTxnType { _ = os.Remove(withoutExt + "-to-block.idx") } diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go index d5025b06f3a..9eae5a710c3 100644 --- a/turbo/snapshotsync/freezeblocks/block_snapshots_test.go +++ b/turbo/snapshotsync/freezeblocks/block_snapshots_test.go @@ -49,7 +49,7 @@ func createTestSegmentFile(t *testing.T, from, to uint64, name snaptype.Enum, di KeyCount: 1, BucketSize: 10, TmpDir: dir, - IndexFile: filepath.Join(dir, snaptype.IdxFileName(1, from, to, snaptype.Indexes.TxnHash2BlockNum.String())), + IndexFile: filepath.Join(dir, snaptype.IdxFileName(1, from, to, coresnaptype.Indexes.TxnHash2BlockNum.Name)), LeafSize: 8, }, logger) require.NoError(t, err) @@ -269,7 +269,7 @@ func TestCanRetire(t *testing.T) { {1_001_000, 2_000_000, 1_001_000, 1_002_000, true}, } for i, tc := range cases { - from, to, can := canRetire(tc.inFrom, tc.inTo, snaptype.Enums.Unknown, nil) + from, to, can := canRetire(tc.inFrom, tc.inTo, snaptype.Unknown, nil) require.Equal(int(tc.outFrom), int(from), i) require.Equal(int(tc.outTo), int(to), i) require.Equal(tc.can, can, tc.inFrom, tc.inTo, i) @@ -291,19 +291,19 @@ func TestOpenAllSnapshot(t *testing.T) { defer s.Close() err := s.ReopenFolder() require.NoError(err) - require.NotNil(s.segments.Get(snaptype.Enums.Headers)) + require.NotNil(s.segments.Get(coresnaptype.Enums.Headers)) getSegs := func(e snaptype.Enum) *segments { res, _ := s.segments.Get(e) return res } - require.Equal(0, len(getSegs(snaptype.Enums.Headers).segments)) + require.Equal(0, len(getSegs(coresnaptype.Enums.Headers).segments)) s.Close() createFile(500_000, 1_000_000, coresnaptype.Bodies) s = NewRoSnapshots(cfg, dir, 0, logger) defer s.Close() - require.NotNil(getSegs(snaptype.Enums.Bodies)) - require.Equal(0, len(getSegs(snaptype.Enums.Bodies).segments)) + require.NotNil(getSegs(coresnaptype.Enums.Bodies)) + require.Equal(0, len(getSegs(coresnaptype.Enums.Bodies).segments)) s.Close() createFile(500_000, 1_000_000, coresnaptype.Headers) @@ -311,8 +311,8 @@ func TestOpenAllSnapshot(t *testing.T) { s = NewRoSnapshots(cfg, dir, 0, logger) err = s.ReopenFolder() require.NoError(err) - require.NotNil(getSegs(snaptype.Enums.Headers)) - require.Equal(0, len(getSegs(snaptype.Enums.Headers).segments)) + require.NotNil(getSegs(coresnaptype.Enums.Headers)) + require.Equal(0, len(getSegs(coresnaptype.Enums.Headers).segments)) s.Close() createFile(0, 500_000, coresnaptype.Bodies) @@ -323,8 +323,8 @@ func TestOpenAllSnapshot(t *testing.T) { err = s.ReopenFolder() require.NoError(err) - require.NotNil(getSegs(snaptype.Enums.Headers)) - require.Equal(2, len(getSegs(snaptype.Enums.Headers).segments)) + require.NotNil(getSegs(coresnaptype.Enums.Headers)) + require.Equal(2, len(getSegs(coresnaptype.Enums.Headers).segments)) view := s.View() defer view.Close() @@ -347,8 +347,8 @@ func TestOpenAllSnapshot(t *testing.T) { err = s.ReopenFolder() require.NoError(err) defer s.Close() - require.NotNil(getSegs(snaptype.Enums.Headers)) - require.Equal(2, len(getSegs(snaptype.Enums.Headers).segments)) + require.NotNil(getSegs(coresnaptype.Enums.Headers)) + require.Equal(2, len(getSegs(coresnaptype.Enums.Headers).segments)) createFile(500_000, 900_000, coresnaptype.Headers) createFile(500_000, 900_000, coresnaptype.Bodies) diff --git a/turbo/snapshotsync/freezeblocks/bor_snapshots.go b/turbo/snapshotsync/freezeblocks/bor_snapshots.go index 38fe037c7fe..4c6fa8a8221 100644 --- a/turbo/snapshotsync/freezeblocks/bor_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/bor_snapshots.go @@ -149,7 +149,7 @@ func removeBorOverlaps(dir string, active []snaptype.FileInfo, max uint64) { l := make([]snaptype.FileInfo, 0, len(list)) for _, f := range list { - if !(f.Type.Enum() == snaptype.Enums.BorSpans || f.Type.Enum() == snaptype.Enums.BorEvents) { + if !(f.Type.Enum() == borsnaptype.Enums.BorSpans || f.Type.Enum() == borsnaptype.Enums.BorEvents) { continue } l = append(l, f) @@ -167,7 +167,7 @@ func removeBorOverlaps(dir string, active []snaptype.FileInfo, max uint64) { } for _, a := range active { - if a.Type.Enum() != snaptype.Enums.BorSpans { + if a.Type.Enum() != borsnaptype.Enums.BorSpans { continue } diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go index b5eac30f7f2..e458e91ade2 100644 --- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go @@ -39,7 +39,7 @@ func BeaconSimpleIdx(ctx context.Context, sn snaptype.FileInfo, salt uint32, tmp num := make([]byte, binary.MaxVarintLen64) if err := snaptype.BuildIndex(ctx, sn, salt, sn.From, tmpDir, log.LvlDebug, p, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error { if i%20_000 == 0 { - logger.Log(lvl, fmt.Sprintf("Generating idx for %s", sn.Type.String()), "progress", i) + logger.Log(lvl, fmt.Sprintf("Generating idx for %s", sn.Type.Name()), "progress", i) } p.Processed.Add(1) n := binary.PutUvarint(num, i) @@ -139,7 +139,7 @@ Loop: var processed bool = true switch f.Type.Enum() { - case snaptype.Enums.BeaconBlocks: + case snaptype.CaplinEnums.BeaconBlocks: var sn *Segment var exists bool for _, sn2 := range s.BeaconBlocks.segments { @@ -188,7 +188,7 @@ Loop: } segmentsMaxSet = true } - case snaptype.Enums.BlobSidecars: + case snaptype.CaplinEnums.BlobSidecars: var sn *Segment var exists bool for _, sn2 := range s.BlobSidecars.segments { @@ -480,13 +480,13 @@ func dumpBlobSidecarsRange(ctx context.Context, db kv.RoDB, storage blob_storage func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, workers int, lvl log.Lvl, logger log.Logger) error { - for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.Enums.BeaconBlocks, nil) { - blocksPerFile := snapcfg.MergeLimit("", snaptype.Enums.BeaconBlocks, i) + for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) { + blocksPerFile := snapcfg.MergeLimit("", snaptype.CaplinEnums.BeaconBlocks, i) if toSlot-i < blocksPerFile { break } - to := chooseSegmentEnd(i, toSlot, snaptype.Enums.BeaconBlocks, nil) + to := chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BeaconBlocks, nil) logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to) if err := dumpBeaconBlocksRange(ctx, db, i, to, salt, dirs, workers, lvl, logger); err != nil { return err @@ -496,13 +496,13 @@ func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, fromSlot, toSlot uint64, } func DumpBlobsSidecar(ctx context.Context, blobStorage blob_storage.BlobStorage, db kv.RoDB, fromSlot, toSlot uint64, salt uint32, dirs datadir.Dirs, compressWorkers int, lvl log.Lvl, logger log.Logger) error { - for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.Enums.BlobSidecars, nil) { - blocksPerFile := snapcfg.MergeLimit("", snaptype.Enums.BlobSidecars, i) + for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BlobSidecars, nil) { + blocksPerFile := snapcfg.MergeLimit("", snaptype.CaplinEnums.BlobSidecars, i) if toSlot-i < blocksPerFile { break } - to := chooseSegmentEnd(i, toSlot, snaptype.Enums.BlobSidecars, nil) + to := chooseSegmentEnd(i, toSlot, snaptype.CaplinEnums.BlobSidecars, nil) logger.Log(lvl, "Dumping blobs sidecars", "from", i, "to", to) if err := dumpBlobSidecarsRange(ctx, db, blobStorage, i, to, salt, dirs, compressWorkers, lvl, logger); err != nil { return err @@ -527,7 +527,7 @@ func (s *CaplinSnapshots) BuildMissingIndices(ctx context.Context, logger log.Lo for index := range segments { segment := segments[index] // The same slot=>offset mapping is used for both beacon blocks and blob sidecars. - if segment.Type.Enum() != snaptype.Enums.BeaconBlocks && segment.Type.Enum() != snaptype.Enums.BlobSidecars { + if segment.Type.Enum() != snaptype.CaplinEnums.BeaconBlocks && segment.Type.Enum() != snaptype.CaplinEnums.BlobSidecars { continue } if segment.Type.HasIndexFiles(segment, logger) { diff --git a/turbo/snapshotsync/snapshotsync.go b/turbo/snapshotsync/snapshotsync.go index 46104d08c5d..efbcefc74d9 100644 --- a/turbo/snapshotsync/snapshotsync.go +++ b/turbo/snapshotsync/snapshotsync.go @@ -209,7 +209,7 @@ func WaitForDownloader(ctx context.Context, logPrefix string, histV3, blobs bool // prohibits further downloads, except some exceptions for _, p := range blockReader.AllTypes() { if _, err := snapshotDownloader.ProhibitNewDownloads(ctx, &proto_downloader.ProhibitNewDownloadsRequest{ - Type: p.String(), + Type: p.Name(), }); err != nil { return err } @@ -222,7 +222,7 @@ func WaitForDownloader(ctx context.Context, logPrefix string, histV3, blobs bool } if _, err := snapshotDownloader.ProhibitNewDownloads(ctx, &proto_downloader.ProhibitNewDownloadsRequest{ - Type: p.String(), + Type: p.Name(), }); err != nil { return err }