Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

stagedSync: Optimize prune old chunks #10019

Merged
merged 7 commits into from
Apr 24, 2024
Merged

stagedSync: Optimize prune old chunks #10019

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
37cdfcd
Fix pruning again
somnathb1 Apr 21, 2024
caeb6e9
Cleanup
somnathb1 Apr 22, 2024
fb4bda8
Cleanup
somnathb1 Apr 22, 2024
d734b52
Cleanup
somnathb1 Apr 22, 2024
611d927
Add err check
somnathb1 Apr 23, 2024
f872c07
Switch to tx.Delete
somnathb1 Apr 23, 2024
790ebf3
Switch back to read cursor
somnathb1 Apr 23, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 6 additions & 14 deletions eth/stagedsync/stage_log_index.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -395,11 +395,8 @@ func pruneOldLogChunks(tx kv.RwTx, bucket string, inMem *etl.Collector, pruneTo
return err
}
var blockNum uint64
if bucket == kv.Log {
blockNum = binary.BigEndian.Uint64(k)
} else {
blockNum = uint64(binary.BigEndian.Uint32(k[len(key):]))
}
blockNum = uint64(binary.BigEndian.Uint32(k[len(key):]))

if !bytes.HasPrefix(k, key) || blockNum >= pruneTo {
break
}
Expand Down Expand Up @@ -434,11 +431,10 @@ func PruneLogIndex(s *PruneState, tx kv.RwTx, cfg LogIndexCfg, ctx context.Conte
}

pruneTo := cfg.prune.Receipts.PruneTo(s.ForwardProgress)
// s.PruneProgress
if err = pruneLogIndex(logPrefix, tx, cfg.tmpdir, s.PruneProgress, pruneTo, ctx, logger, cfg.noPruneContracts); err != nil {
return err
}
if err = s.Done(tx); err != nil {
if err = s.DoneAt(tx, pruneTo); err != nil {
return err
}

Expand All @@ -460,8 +456,6 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune
defer topics.Close()
addrs := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger)
defer addrs.Close()
pruneLogKeyCollector := etl.NewCollector(logPrefix, tmpDir, etl.NewOldestEntryBuffer(bufferSize), logger)
defer pruneLogKeyCollector.Close()

reader := bytes.NewReader(nil)
{
Expand All @@ -481,7 +475,7 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune
}
select {
case <-logEvery.C:
logger.Info(fmt.Sprintf("[%s]", logPrefix), "table", kv.Log, "block", blockNum)
logger.Info(fmt.Sprintf("[%s]", logPrefix), "table", kv.Log, "block", blockNum, "pruneFrom", pruneFrom, "pruneTo", pruneTo)
case <-ctx.Done():
return libcommon.ErrStopped
default:
Expand All @@ -502,6 +496,7 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune
break
}
}

if toPrune {
for _, l := range logs {
for _, topic := range l.Topics {
Expand All @@ -513,7 +508,7 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune
return err
}
}
if err := pruneLogKeyCollector.Collect(k, nil); err != nil {
if err := tx.Delete(kv.Log, k); err != nil {
return err
}
}
Expand All @@ -526,8 +521,5 @@ func pruneLogIndex(logPrefix string, tx kv.RwTx, tmpDir string, pruneFrom, prune
if err := pruneOldLogChunks(tx, kv.LogAddressIndex, addrs, pruneTo, ctx); err != nil {
return err
}
if err := pruneOldLogChunks(tx, kv.Log, pruneLogKeyCollector, pruneTo, ctx); err != nil {
return err
}
return nil
}
Loading