Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Blocks snaps - see 0 indices after reopen #10219

Merged
merged 1 commit into from
May 6, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions cmd/rpcdaemon/cli/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -384,7 +384,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger
allSnapshots.OptimisticReopenWithDB(db)
allBorSnapshots.OptimisticalyReopenWithDB(db)
allSnapshots.LogStat("remote")
allBorSnapshots.LogStat("remote")
allBorSnapshots.LogStat("bor:remote")

if agg, err = libstate.NewAggregator(ctx, cfg.Dirs.SnapHistory, cfg.Dirs.Tmp, config3.HistoryV3AggregationStep, db, logger); err != nil {
return nil, nil, nil, nil, nil, nil, nil, ff, nil, fmt.Errorf("create aggregator: %w", err)
Expand Down Expand Up @@ -413,7 +413,7 @@ func RemoteServices(ctx context.Context, cfg *httpcfg.HttpCfg, logger log.Logger
if err := allBorSnapshots.ReopenList(reply.BlocksFiles, true); err != nil {
logger.Error("[bor snapshots] reopen", "err", err)
} else {
allBorSnapshots.LogStat("reopen")
allBorSnapshots.LogStat("bor:reopen")
}

_ = reply.HistoryFiles
Expand Down
2 changes: 1 addition & 1 deletion turbo/app/snapshots_cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -409,7 +409,7 @@ func openSnaps(ctx context.Context, cfg ethconfig.BlocksFreezing, dirs datadir.D
return
}

borSnaps.LogStat("open")
borSnaps.LogStat("bor:open")
agg = openAgg(ctx, dirs, chainDB, logger)
err = chainDB.View(ctx, func(tx kv.Tx) error {
ac := agg.BeginFilesRo()
Expand Down
38 changes: 27 additions & 11 deletions turbo/snapshotsync/freezeblocks/block_snapshots.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"encoding/hex"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"reflect"
Expand Down Expand Up @@ -312,8 +311,8 @@ func (s *RoSnapshots) BlocksAvailable() uint64 {
func (s *RoSnapshots) LogStat(label string) {
var m runtime.MemStats
dbg.ReadMemStats(&m)
s.logger.Info(fmt.Sprintf("[snapshots:%s] Blocks Stat", label),
"blocks", fmt.Sprintf("%dk", (s.BlocksAvailable()+1)/1000),
s.logger.Info(fmt.Sprintf("[snapshots:%s] Stat", label),
"blocks", fmt.Sprintf("%dk", (s.SegmentsMax()+1)/1000),
"indices", fmt.Sprintf("%dk", (s.IndicesMax()+1)/1000),
"alloc", common2.ByteCount(m.Alloc), "sys", common2.ByteCount(m.Sys))
}
Expand Down Expand Up @@ -386,29 +385,46 @@ func (s *RoSnapshots) EnableMadvNormal() *RoSnapshots {
return s
}

// minimax of existing indices
func (s *RoSnapshots) idxAvailability() uint64 {
_max := make([]uint64, len(s.Types()))
i := 0
// Use-Cases:
// 1. developers can add new types in future. and users will not have files of this type
// 2. some types are network-specific. example: borevents exists only on Bor-consensus networks
// 3. user can manually remove 1 .idx file: `rm snapshots/v1-type1-0000-1000.idx`
// 4. user can manually remove all .idx files of given type: `rm snapshots/*type1*.idx`
// 5. file-types may have different height: 10 headers, 10 bodies, 9 trancasctions (for example if `kill -9` came during files building/merge). still need index all 3 types.
amount := 0
s.segments.Scan(func(segtype snaptype.Enum, value *segments) bool {
if len(value.segments) == 0 || !s.HasType(segtype.Type()) {
return true
}
amount++
return true
})

maximums := make([]uint64, amount)
var i int
s.segments.Scan(func(segtype snaptype.Enum, value *segments) bool {
if len(value.segments) == 0 || !s.HasType(segtype.Type()) {
return true
}

for _, seg := range value.segments {
if !seg.IsIndexed() {
break
}

_max[i] = seg.to - 1
maximums[i] = seg.to - 1
}

i++
return true
})

var _min uint64 = math.MaxUint64
for _, maxEl := range _max {
_min = cmp.Min(_min, maxEl)
if len(maximums) == 0 {
return 0
}

return _min
return slices.Min(maximums)
}

// OptimisticReopenWithDB - optimistically open snapshots (ignoring error), useful at App startup because:
Expand Down
Loading