From 0d64796408cc13e148964b35085664866acd4e03 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 9 May 2022 13:47:00 +0200 Subject: [PATCH 1/6] ethdb/remotedb, cmd: add support for remote (readonly) databases --- cmd/utils/flags.go | 9 ++ ethdb/remotedb/remotedb.go | 183 +++++++++++++++++++++++++++++++++++++ 2 files changed, 192 insertions(+) create mode 100644 ethdb/remotedb/remotedb.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 161130013f08..b0526cb30b22 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -50,6 +50,7 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/ethdb/remotedb" "github.com/ethereum/go-ethereum/ethstats" "github.com/ethereum/go-ethereum/graphql" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -114,6 +115,10 @@ var ( Usage: "Data directory for the databases and keystore", Value: DirectoryString(node.DefaultDataDir()), } + RemoteDBFlag = cli.StringFlag{ + Name: "remotedb", + Usage: "URL for remote database", + } AncientFlag = DirectoryFlag{ Name: "datadir.ancient", Usage: "Data directory for ancient chain segments (default = inside chaindata)", @@ -841,6 +846,7 @@ var ( DatabasePathFlags = []cli.Flag{ DataDirFlag, AncientFlag, + RemoteDBFlag, } ) @@ -1953,6 +1959,9 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. err error chainDb ethdb.Database ) + if ctx.GlobalIsSet(RemoteDBFlag.Name) { + chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name)) + } if ctx.GlobalString(SyncModeFlag.Name) == "light" { name := "lightchaindata" chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go new file mode 100644 index 000000000000..d75cc77929d5 --- /dev/null +++ b/ethdb/remotedb/remotedb.go @@ -0,0 +1,183 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Package remotedb implements the key-value database layer based on a remote geth +// node. Under the hood, it utilises the `debug_dbGet` method to implement a +// read-only database. +// There really are no guarantees in this database, since the local geth does not +// exclusive access, but it can be used for basic diagnostics of a remote node. +package remotedb + +import ( + "errors" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rpc" + "strings" +) + +// Database is a key-value lookup for a remote database via debug_dbGet. +type Database struct { + remote *rpc.Client +} + +func (db *Database) Has(key []byte) (bool, error) { + if _, err := db.Get(key); err != nil { + return true, nil + } + return false, nil +} + +func (db *Database) Get(key []byte) ([]byte, error) { + var resp hexutil.Bytes + err := db.remote.Call(&resp, "debug_dbGet", hexutil.Bytes(key)) + if err != nil { + return nil, err + } + return resp, nil +} + +func (db *Database) HasAncient(kind string, number uint64) (bool, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Ancient(kind string, number uint64) ([]byte, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Ancients() (uint64, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Tail() (uint64, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) AncientSize(kind string) (uint64, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Put(key []byte, value []byte) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) Delete(key []byte) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) TruncateHead(n uint64) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) TruncateTail(n uint64) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) Sync() error { + //TODO implement me + panic("implement me") +} + +func (db *Database) MigrateTable(s string, f func([]byte) ([]byte, error)) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) NewBatch() ethdb.Batch { + //TODO implement me + panic("implement me") +} + +func (db *Database) NewBatchWithSize(size int) ethdb.Batch { + //TODO implement me + panic("implement me") +} + +func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { + //TODO implement me + panic("implement me") +} + +func (db *Database) Stat(property string) (string, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) AncientDatadir() (string, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Compact(start []byte, limit []byte) error { + //TODO implement me + panic("implement me") +} + +func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { + //TODO implement me + panic("implement me") +} + +func (db *Database) Close() error { + //TODO implement me + panic("implement me") +} + +func dialRPC(endpoint string) (*rpc.Client, error) { + if endpoint == "" { + return nil, errors.New("endpoint must be specified") + } + if strings.HasPrefix(endpoint, "rpc:") || strings.HasPrefix(endpoint, "ipc:") { + // Backwards compatibility with geth < 1.5 which required + // these prefixes. + endpoint = endpoint[4:] + } + return rpc.Dial(endpoint) +} + +func New(endpoint string) (ethdb.Database, error) { + client, err := dialRPC(endpoint) + if err != nil { + return nil, err + } + return &Database{ + remote: client, + }, nil +} From 3b5e2e10fc36748dd824bc1062ac2f94d641ec0a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 9 May 2022 14:02:20 +0200 Subject: [PATCH 2/6] ethdb/remotedb: minor changes --- ethdb/remotedb/remotedb.go | 66 +++++++++++++------------------------- 1 file changed, 22 insertions(+), 44 deletions(-) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index d75cc77929d5..610e77ef16f6 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -51,113 +51,91 @@ func (db *Database) Get(key []byte) ([]byte, error) { } func (db *Database) HasAncient(kind string, number uint64) (bool, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Ancient(kind string, number uint64) ([]byte, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Ancients() (uint64, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Tail() (uint64, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) AncientSize(kind string) (uint64, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Put(key []byte, value []byte) error { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Delete(key []byte) error { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) ModifyAncients(f func(ethdb.AncientWriteOp) error) (int64, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) TruncateHead(n uint64) error { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) TruncateTail(n uint64) error { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Sync() error { - //TODO implement me - panic("implement me") + return nil } func (db *Database) MigrateTable(s string, f func([]byte) ([]byte, error)) error { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) NewBatch() ethdb.Batch { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) NewBatchWithSize(size int) ethdb.Batch { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) NewIterator(prefix []byte, start []byte) ethdb.Iterator { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Stat(property string) (string, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) AncientDatadir() (string, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Compact(start []byte, limit []byte) error { - //TODO implement me - panic("implement me") + return nil } func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { - //TODO implement me - panic("implement me") + panic("not supported") } func (db *Database) Close() error { - //TODO implement me - panic("implement me") + return nil } func dialRPC(endpoint string) (*rpc.Client, error) { From 1142cda232273c00d2e2ab81763ab7687bd64a3b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 9 May 2022 14:23:55 +0200 Subject: [PATCH 3/6] ethdb/remotedb: close the conn --- ethdb/remotedb/remotedb.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 610e77ef16f6..24d986c7dfd7 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -135,6 +135,7 @@ func (db *Database) NewSnapshot() (ethdb.Snapshot, error) { } func (db *Database) Close() error { + db.remote.Close() return nil } From 65865c090ba8c2b6ed8bfcb9d4ef90e6f1eca429 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 10 May 2022 13:51:26 +0200 Subject: [PATCH 4/6] cmd, ethdb: add rpc accessor for ancient data --- cmd/utils/flags.go | 15 +++++++-------- ethdb/remotedb/remotedb.go | 18 ++++++++++++++---- internal/ethapi/api.go | 9 --------- internal/ethapi/dbapi.go | 27 +++++++++++++++++++++++++++ internal/web3ext/web3ext.go | 10 ++++++++++ 5 files changed, 58 insertions(+), 21 deletions(-) create mode 100644 internal/ethapi/dbapi.go diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index b0526cb30b22..a8c8b937d11e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1959,15 +1959,14 @@ func MakeChainDatabase(ctx *cli.Context, stack *node.Node, readonly bool) ethdb. err error chainDb ethdb.Database ) - if ctx.GlobalIsSet(RemoteDBFlag.Name) { + switch { + case ctx.GlobalIsSet(RemoteDBFlag.Name): + log.Info("Using remote db", "url", ctx.GlobalString(RemoteDBFlag.Name)) chainDb, err = remotedb.New(ctx.GlobalString(RemoteDBFlag.Name)) - } - if ctx.GlobalString(SyncModeFlag.Name) == "light" { - name := "lightchaindata" - chainDb, err = stack.OpenDatabase(name, cache, handles, "", readonly) - } else { - name := "chaindata" - chainDb, err = stack.OpenDatabaseWithFreezer(name, cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) + case ctx.GlobalString(SyncModeFlag.Name) == "light": + chainDb, err = stack.OpenDatabase("lightchaindata", cache, handles, "", readonly) + default: + chainDb, err = stack.OpenDatabaseWithFreezer("chaindata", cache, handles, ctx.GlobalString(AncientFlag.Name), "", readonly) } if err != nil { Fatalf("Could not open database: %v", err) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 24d986c7dfd7..912da73d8ac1 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -51,11 +51,19 @@ func (db *Database) Get(key []byte) ([]byte, error) { } func (db *Database) HasAncient(kind string, number uint64) (bool, error) { - panic("not supported") + if _, err := db.Ancient(kind, number); err != nil { + return true, nil + } + return false, nil } func (db *Database) Ancient(kind string, number uint64) ([]byte, error) { - panic("not supported") + var resp hexutil.Bytes + err := db.remote.Call(&resp, "debug_dbAncient", kind, number) + if err != nil { + return nil, err + } + return resp, nil } func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) { @@ -63,7 +71,9 @@ func (db *Database) AncientRange(kind string, start, count, maxBytes uint64) ([] } func (db *Database) Ancients() (uint64, error) { - panic("not supported") + var resp uint64 + err := db.remote.Call(&resp, "debug_dbAncients") + return resp, err } func (db *Database) Tail() (uint64, error) { @@ -75,7 +85,7 @@ func (db *Database) AncientSize(kind string) (uint64, error) { } func (db *Database) ReadAncients(fn func(op ethdb.AncientReaderOp) error) (err error) { - panic("not supported") + return fn(db) } func (db *Database) Put(key []byte, value []byte) error { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 33842ff6c48e..d087673797dc 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1972,15 +1972,6 @@ func (api *PrivateDebugAPI) SetHead(number hexutil.Uint64) { api.b.SetHead(uint64(number)) } -// DbGet returns the raw value of a key stored in the database. -func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { - blob, err := common.ParseHexOrString(key) - if err != nil { - return nil, err - } - return api.b.ChainDb().Get(blob) -} - // PublicNetAPI offers network related RPC methods type PublicNetAPI struct { net *p2p.Server diff --git a/internal/ethapi/dbapi.go b/internal/ethapi/dbapi.go new file mode 100644 index 000000000000..12b48ccfd267 --- /dev/null +++ b/internal/ethapi/dbapi.go @@ -0,0 +1,27 @@ +package ethapi + +import ( + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" +) + +// DbGet returns the raw value of a key stored in the database. +func (api *PrivateDebugAPI) DbGet(key string) (hexutil.Bytes, error) { + blob, err := common.ParseHexOrString(key) + if err != nil { + return nil, err + } + return api.b.ChainDb().Get(blob) +} + +// DbAncient retrieves an ancient binary blob from the append-only immutable files. +// It is a mapping to the `AncientReaderOp.Ancient` method +func (api *PrivateDebugAPI) DbAncient(kind string, number uint64) (hexutil.Bytes, error) { + return api.b.ChainDb().Ancient(kind, number) +} + +// DbAncients returns the ancient item numbers in the ancient store. +// It is a mapping to the `AncientReaderOp.Ancients` method +func (api *PrivateDebugAPI) DbAncients() (uint64, error) { + return api.b.ChainDb().Ancients() +} diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 4b7a196e4726..372ca3969584 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -476,6 +476,16 @@ web3._extend({ call: 'debug_dbGet', params: 1 }), + new web3._extend.Method({ + name: 'dbAncient', + call: 'debug_dbAncient', + params: 2 + }), + new web3._extend.Method({ + name: 'dbAncients', + call: 'debug_dbAncients', + params: 0 + }), ], properties: [] }); From f27ba7fc4cabf7e4999e17922f2249f353f2ee57 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 10 May 2022 13:52:22 +0200 Subject: [PATCH 5/6] internal/ethapi: license --- internal/ethapi/dbapi.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/internal/ethapi/dbapi.go b/internal/ethapi/dbapi.go index 12b48ccfd267..33dca29d3c66 100644 --- a/internal/ethapi/dbapi.go +++ b/internal/ethapi/dbapi.go @@ -1,3 +1,19 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package ethapi import ( From 95f952fc58b381725a196cc646c98ee03cce425d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 17 May 2022 09:24:12 +0200 Subject: [PATCH 6/6] ethdb/remotedb: linter fixes --- ethdb/remotedb/remotedb.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ethdb/remotedb/remotedb.go b/ethdb/remotedb/remotedb.go index 912da73d8ac1..a645760b9672 100644 --- a/ethdb/remotedb/remotedb.go +++ b/ethdb/remotedb/remotedb.go @@ -23,10 +23,11 @@ package remotedb import ( "errors" + "strings" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rpc" - "strings" ) // Database is a key-value lookup for a remote database via debug_dbGet.