Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cmd, core, ethdb, node: rework ancient store #25487

Merged
merged 7 commits into from
Aug 8, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 22 additions & 31 deletions cmd/geth/dbcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"os"
"os/signal"
"path/filepath"
"sort"
"strconv"
"strings"
"syscall"
Expand Down Expand Up @@ -160,8 +159,8 @@ WARNING: This is a low-level operation which may cause database corruption!`,
dbDumpFreezerIndex = &cli.Command{
Action: freezerInspect,
Name: "freezer-index",
Usage: "Dump out the index of a given freezer type",
ArgsUsage: "<type> <start (int)> <end (int)>",
Usage: "Dump out the index of a specific freezer table",
ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
Flags: flags.Merge([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Expand Down Expand Up @@ -275,7 +274,7 @@ func inspect(ctx *cli.Context) error {
start []byte
)
if ctx.NArg() > 2 {
return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage)
return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage)
}
if ctx.NArg() >= 1 {
if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
Expand Down Expand Up @@ -536,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error {
}

func freezerInspect(ctx *cli.Context) error {
var (
start, end int64
disableSnappy bool
err error
)
if ctx.NArg() < 3 {
if ctx.NArg() < 4 {
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
}
kind := ctx.Args().Get(0)
if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok {
var options []string
for opt := range rawdb.FreezerNoSnappy {
options = append(options, opt)
}
sort.Strings(options)
return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options)
} else {
disableSnappy = noSnap
}
if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil {
log.Info("Could read start-param", "error", err)
var (
freezer = ctx.Args().Get(0)
table = ctx.Args().Get(1)
)
start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64)
if err != nil {
log.Info("Could not read start-param", "err", err)
return err
}
if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
log.Info("Could read count param", "error", err)
end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64)
if err != nil {
log.Info("Could not read count param", "err", err)
return err
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
log.Info("Opening freezer", "location", path, "name", kind)
if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil {

db := utils.MakeChainDatabase(ctx, stack, true)
defer db.Close()

ancient, err := db.AncientDatadir()
if err != nil {
log.Info("Failed to retrieve ancient root", "err", err)
return err
} else {
f.DumpIndex(start, end)
}
return nil
return rawdb.InspectFreezerTable(ancient, freezer, table, start, end)
}

func importLDBdata(ctx *cli.Context) error {
Expand Down
2 changes: 1 addition & 1 deletion cmd/utils/flags.go
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ var (
}
AncientFlag = &flags.DirectoryFlag{
Name: "datadir.ancient",
Usage: "Data directory for ancient chain segments (default = inside chaindata)",
Usage: "Root directory for ancient data (default = inside chaindata)",
Category: flags.EthCategory,
}
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
Expand Down
26 changes: 13 additions & 13 deletions core/rawdb/accessors_chain.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ import (
func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
var data []byte
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
data, _ = reader.Ancient(freezerHashTable, number)
data, _ = reader.Ancient(chainFreezerHashTable, number)
if len(data) == 0 {
// Get it by hash from leveldb
data, _ = db.Get(headerHashKey(number))
Expand Down Expand Up @@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
}
// read remaining from ancients
max := count * 700
data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max)
data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max)
if err == nil && uint64(len(data)) == count {
// the data is on the order [h, h+1, .., n] -- reordering needed
for i := range data {
Expand All @@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu
// First try to look up the data in ancient database. Extra hash
// comparison is necessary since ancient database only maintains
// the canonical data.
data, _ = reader.Ancient(freezerHeaderTable, number)
data, _ = reader.Ancient(chainFreezerHeaderTable, number)
if len(data) > 0 && crypto.Keccak256Hash(data) == hash {
return nil
}
Expand Down Expand Up @@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
// isCanon is an internal utility method, to check whether the given number/hash
// is part of the ancient (canon) set.
func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool {
h, err := reader.Ancient(freezerHashTable, number)
h, err := reader.Ancient(chainFreezerHashTable, number)
if err != nil {
return false
}
Expand All @@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
// Check if the data is in ancients
if isCanon(reader, number, hash) {
data, _ = reader.Ancient(freezerBodiesTable, number)
data, _ = reader.Ancient(chainFreezerBodiesTable, number)
return nil
}
// If not, try reading from leveldb
Expand All @@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
var data []byte
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
data, _ = reader.Ancient(freezerBodiesTable, number)
data, _ = reader.Ancient(chainFreezerBodiesTable, number)
if len(data) > 0 {
return nil
}
Expand Down Expand Up @@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
// Check if the data is in ancients
if isCanon(reader, number, hash) {
data, _ = reader.Ancient(freezerDifficultyTable, number)
data, _ = reader.Ancient(chainFreezerDifficultyTable, number)
return nil
}
// If not, try reading from leveldb
Expand Down Expand Up @@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
// Check if the data is in ancients
if isCanon(reader, number, hash) {
data, _ = reader.Ancient(freezerReceiptTable, number)
data, _ = reader.Ancient(chainFreezerReceiptTable, number)
return nil
}
// If not, try reading from leveldb
Expand Down Expand Up @@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts

func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
num := block.NumberU64()
if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil {
if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
return fmt.Errorf("can't add block %d hash: %v", num, err)
}
if err := op.Append(freezerHeaderTable, num, header); err != nil {
if err := op.Append(chainFreezerHeaderTable, num, header); err != nil {
return fmt.Errorf("can't append block header %d: %v", num, err)
}
if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil {
if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil {
return fmt.Errorf("can't append block body %d: %v", num, err)
}
if err := op.Append(freezerReceiptTable, num, receipts); err != nil {
if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil {
return fmt.Errorf("can't append block %d receipts: %v", num, err)
}
if err := op.Append(freezerDifficultyTable, num, td); err != nil {
if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil {
return fmt.Errorf("can't append block %d total difficulty: %v", num, err)
}
return nil
Expand Down
86 changes: 86 additions & 0 deletions core/rawdb/ancient_scheme.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,86 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.

package rawdb

import "fmt"

// The list of table names of chain freezer.
const (
// chainFreezerHeaderTable indicates the name of the freezer header table.
chainFreezerHeaderTable = "headers"

// chainFreezerHashTable indicates the name of the freezer canonical hash table.
chainFreezerHashTable = "hashes"

// chainFreezerBodiesTable indicates the name of the freezer block body table.
chainFreezerBodiesTable = "bodies"

// chainFreezerReceiptTable indicates the name of the freezer receipts table.
chainFreezerReceiptTable = "receipts"

// chainFreezerDifficultyTable indicates the name of the freezer total difficulty table.
chainFreezerDifficultyTable = "diffs"
)

// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
// Hashes and difficulties don't compress well.
var chainFreezerNoSnappy = map[string]bool{
chainFreezerHeaderTable: false,
chainFreezerHashTable: true,
chainFreezerBodiesTable: false,
chainFreezerReceiptTable: false,
chainFreezerDifficultyTable: true,
}

// The list of identifiers of ancient stores.
var (
chainFreezerName = "chain" // the folder name of chain segment ancient store.
)

// freezers the collections of all builtin freezers.
var freezers = []string{chainFreezerName}

// InspectFreezerTable dumps out the index of a specific freezer table. The passed
// ancient indicates the path of root ancient directory where the chain freezer can
// be opened. Start and end specify the range for dumping out indexes.
// Note this function can only be used for debugging purposes.
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
var (
path string
tables map[string]bool
)
switch freezerName {
case chainFreezerName:
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
default:
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
}
noSnappy, exist := tables[tableName]
if !exist {
var names []string
for name := range tables {
names = append(names, name)
}
return fmt.Errorf("unknown table, supported ones: %v", names)
}
table, err := newFreezerTable(path, tableName, noSnappy, true)
if err != nil {
return err
}
table.dumpIndexStdout(start, end)
return nil
}
10 changes: 5 additions & 5 deletions core/rawdb/chain_freezer.go
Original file line number Diff line number Diff line change
Expand Up @@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
}

// Write to the batch.
if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil {
if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil {
return fmt.Errorf("can't write hash to Freezer: %v", err)
}
if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil {
if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil {
return fmt.Errorf("can't write header to Freezer: %v", err)
}
if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil {
if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil {
return fmt.Errorf("can't write body to Freezer: %v", err)
}
if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil {
if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil {
return fmt.Errorf("can't write receipts to Freezer: %v", err)
}
if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil {
if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil {
return fmt.Errorf("can't write td to Freezer: %v", err)
}

Expand Down
2 changes: 1 addition & 1 deletion core/rawdb/chain_iterator.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
if i+count > frozen {
count = frozen - i
}
data, err := db.AncientRange(freezerHashTable, i, count, 32*count)
data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count)
if err != nil {
log.Crit("Failed to init database from freezer", "err", err)
}
Expand Down
Loading