Skip to content

Commit

Permalink
Merge pull request #1414 from skalenetwork/feature/IS-686-new-debug-a…
Browse files Browse the repository at this point in the history 
…pi-call

Feature/is 686 new leveldb version
  • Loading branch information
@DmytroNazarenko
DmytroNazarenko authored May 3, 2023
2 parents 35e6c43 + 75b2fb9 commit 2cee896
Show file tree
Hide file tree
Showing 15 changed files with 153 additions and 85 deletions.
4 changes: 2 additions & 2 deletions CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ endif()
option( HUNTER_RUN_UPLOAD "Upload binaries to the cache server" ${run_upload} )

include( HunterGate )
HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) new leveldb
#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) #leveldb 1.22

set( CMAKE_CXX_STANDARD 17 )

Expand Down
19 changes: 14 additions & 5 deletions libdevcore/LevelDB.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -88,13 +88,16 @@ leveldb::ReadOptions LevelDB::defaultReadOptions() {
}

leveldb::WriteOptions LevelDB::defaultWriteOptions() {
return leveldb::WriteOptions();
leveldb::WriteOptions writeOptions = leveldb::WriteOptions();
writeOptions.sync = true;
return writeOptions;
}

leveldb::Options LevelDB::defaultDBOptions() {
leveldb::Options options;
options.create_if_missing = true;
options.max_open_files = c_maxOpenLeveldbFiles;
options.filter_policy = leveldb::NewBloomFilterPolicy( 10 );
return options;
}

Expand All @@ -103,15 +106,21 @@ LevelDB::LevelDB( boost::filesystem::path const& _path, leveldb::ReadOptions _re
: m_db( nullptr ),
m_readOptions( std::move( _readOptions ) ),
m_writeOptions( std::move( _writeOptions ) ),
m_options( std::move( _dbOptions ) ),
m_path( _path ) {
auto db = static_cast< leveldb::DB* >( nullptr );
auto const status = leveldb::DB::Open( _dbOptions, _path.string(), &db );
auto const status = leveldb::DB::Open( m_options, _path.string(), &db );
checkStatus( status, _path );

assert( db );
m_db.reset( db );
}

LevelDB::~LevelDB() {
if ( m_options.filter_policy )
delete m_options.filter_policy;
}

std::string LevelDB::lookup( Slice _key ) const {
leveldb::Slice const key( _key.data(), _key.size() );
std::string value;
Expand Down Expand Up @@ -227,9 +236,9 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
return hash;
}

// void LevelDB::doCompaction() const {
// m_db->CompactRange( NULL, NULL );
//}
void LevelDB::doCompaction() const {
m_db->CompactRange( nullptr, nullptr );
}

} // namespace db
} // namespace dev
6 changes: 5 additions & 1 deletion libdevcore/LevelDB.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
#include "db.h"

#include <leveldb/db.h>
#include <leveldb/filter_policy.h>
#include <leveldb/write_batch.h>
#include <boost/filesystem.hpp>

Expand All @@ -38,6 +39,8 @@ class LevelDB : public DatabaseFace {
leveldb::WriteOptions _writeOptions = defaultWriteOptions(),
leveldb::Options _dbOptions = defaultDBOptions() );

~LevelDB();

std::string lookup( Slice _key ) const override;
bool exists( Slice _key ) const override;
void insert( Slice _key, Slice _value ) override;
Expand All @@ -51,12 +54,13 @@ class LevelDB : public DatabaseFace {
h256 hashBase() const override;
h256 hashBaseWithPrefix( char _prefix ) const;

// void doCompaction() const;
void doCompaction() const;

private:
std::unique_ptr< leveldb::DB > m_db;
leveldb::ReadOptions const m_readOptions;
leveldb::WriteOptions const m_writeOptions;
leveldb::Options m_options;
boost::filesystem::path const m_path;
};

Expand Down
18 changes: 9 additions & 9 deletions libethereum/BlockChain.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,9 @@ void BlockChain::open( fs::path const& _path, bool _applyPatches, WithExisting _

try {
fs::create_directories( chainPath / fs::path( "blocks_and_extras" ) );
auto rotator = std::make_shared< batched_io::rotating_db_io >(
m_rotator = std::make_shared< batched_io::rotating_db_io >(
chainPath / fs::path( "blocks_and_extras" ), 5, chainParams().nodeInfo.archiveMode );
m_rotating_db = std::make_shared< db::ManuallyRotatingLevelDB >( rotator );
m_rotating_db = std::make_shared< db::ManuallyRotatingLevelDB >( m_rotator );
auto db = std::make_shared< batched_io::batched_db >();
db->open( m_rotating_db );
m_db = db;
Expand Down Expand Up @@ -1364,13 +1364,13 @@ void BlockChain::clearCaches() {
}
}

// void BlockChain::doLevelDbCompaction() const {
// for ( auto it = m_rotator->begin(); it != m_rotator->end(); ++it ) {
// dev::db::LevelDB* ldb = dynamic_cast< dev::db::LevelDB* >( it->get() );
// assert( ldb );
// ldb->doCompaction();
// }
//}
void BlockChain::doLevelDbCompaction() const {
for ( auto it = m_rotator->begin(); it != m_rotator->end(); ++it ) {
dev::db::LevelDB* ldb = dynamic_cast< dev::db::LevelDB* >( it->get() );
assert( ldb );
ldb->doCompaction();
}
}

void BlockChain::checkConsistency() {
DEV_WRITE_GUARDED( x_details ) { m_details.clear(); }
Expand Down
6 changes: 3 additions & 3 deletions libethereum/BlockChain.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,8 @@ class BlockChain {
void open( boost::filesystem::path const& _path, bool _applyPatches, WithExisting _we );
/// Finalise everything and close the database.
void close();
// /// compact db before snapshot
// void doLevelDbCompaction() const;
/// compact db before snapshot
void doLevelDbCompaction() const;

private:
bool rotateDBIfNeeded( uint64_t pieceUsageBytes );
Expand Down Expand Up @@ -589,7 +589,7 @@ class BlockChain {
uint64_t m_maxStorageUsage;

/// The disk DBs. Thread-safe, so no need for locks.
// std::shared_ptr< batched_io::rotating_db_io > m_rotator; // for compaction
std::shared_ptr< batched_io::rotating_db_io > m_rotator; // for compaction
std::shared_ptr< db::ManuallyRotatingLevelDB > m_rotating_db; // rotate()
std::shared_ptr< batched_io::db_face > m_db; // insert()/commit()
std::unique_ptr< batched_io::db_splitter > m_db_splitter; // new_interface()
Expand Down
4 changes: 2 additions & 2 deletions libethereum/Client.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,9 @@ class Client : public ClientBase, protected Worker {
return chainParams().sChain.nodeGroups[imaBLSPublicKeyGroupIndex].blsPublicKey;
}

// void doStateDbCompaction() const { m_state.getOriginalDb()->doCompaction(); }
void doStateDbCompaction() const { m_state.getOriginalDb()->doCompaction(); }

// void doBlocksDbCompaction() const { m_bc.doLevelDbCompaction(); }
void doBlocksDbCompaction() const { m_bc.doLevelDbCompaction(); }

std::pair< uint64_t, uint64_t > getBlocksDbUsage() const;

Expand Down
52 changes: 50 additions & 2 deletions libskale/State.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,52 @@ using dev::eth::TransactionReceipt;
#define ETH_VMTRACE 0
#endif

State::State( dev::u256 const& _accountStartNonce, boost::filesystem::path const& _dbPath,
dev::h256 const& _genesis, BaseState _bs, dev::u256 _initialFunds,
dev::s256 _contractStorageLimit )
: x_db_ptr( make_shared< boost::shared_mutex >() ),
m_storedVersion( make_shared< size_t >( 0 ) ),
m_currentVersion( *m_storedVersion ),
m_accountStartNonce( _accountStartNonce ),
m_initial_funds( _initialFunds ),
contractStorageLimit_( _contractStorageLimit )
#ifdef HISTORIC_STATE
,
m_historicState( _accountStartNonce,
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_STATE_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ),
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_ROOTS_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) )
#endif
{
m_db_ptr = make_shared< OverlayDB >( openDB( _dbPath, _genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) );

auto state = createStateReadOnlyCopy();
totalStorageUsed_ = state.storageUsedTotal();
#ifdef HISTORIC_STATE
m_historicState.setRootFromDB();
#endif
m_fs_ptr = state.fs();
if ( _bs == BaseState::PreExisting ) {
clog( VerbosityDebug, "statedb" ) << cc::debug( "Using existing database" );
} else if ( _bs == BaseState::Empty ) {
// Initialise to the state entailed by the genesis block; this guarantees the trie is built
// correctly.
m_db_ptr->clearDB();
} else {
throw std::logic_error( "Not implemented" );
}
}

State::State( u256 const& _accountStartNonce, OverlayDB const& _db,
#ifdef HISTORIC_STATE
dev::OverlayDB const& _historicDb, dev::OverlayDB const& _historicBlockToStateRootDb,
Expand Down Expand Up @@ -178,9 +224,9 @@ skale::OverlayDB State::openDB(

fs::path state_path = path / fs::path( "state" );
try {
std::shared_ptr< db::DatabaseFace > db( new db::DBImpl( state_path ) );
m_orig_db.reset( new db::DBImpl( state_path ) );
std::unique_ptr< batched_io::batched_db > bdb = make_unique< batched_io::batched_db >();
bdb->open( db );
bdb->open( m_orig_db );
assert( bdb->is_open() );
clog( VerbosityDebug, "statedb" ) << cc::success( "Opened state DB." );
return OverlayDB( std::move( bdb ) );
Expand Down Expand Up @@ -213,6 +259,7 @@ State::State( const State& _s )
std::logic_error( "Can't copy locked for writing state object" );
}
m_db_ptr = _s.m_db_ptr;
m_orig_db = _s.m_orig_db;
m_storedVersion = _s.m_storedVersion;
m_currentVersion = _s.m_currentVersion;
m_cache = _s.m_cache;
Expand All @@ -234,6 +281,7 @@ State& State::operator=( const State& _s ) {
std::logic_error( "Can't copy locked for writing state object" );
}
m_db_ptr = _s.m_db_ptr;
m_orig_db = _s.m_orig_db;
m_storedVersion = _s.m_storedVersion;
m_currentVersion = _s.m_currentVersion;
m_cache = _s.m_cache;
Expand Down
35 changes: 7 additions & 28 deletions libskale/State.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -179,29 +179,8 @@ class State {
// This is called once in the client during the client creation
explicit State( dev::u256 const& _accountStartNonce, boost::filesystem::path const& _dbPath,
dev::h256 const& _genesis, BaseState _bs = BaseState::PreExisting,
dev::u256 _initialFunds = 0, dev::s256 _contractStorageLimit = 32 )
: State( _accountStartNonce,
openDB( _dbPath, _genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
#ifdef HISTORIC_STATE
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_STATE_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_ROOTS_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
#endif /// which uses it. If you have no preexisting database then set BaseState to something other
_bs, _initialFunds, _contractStorageLimit ) {
}
dev::u256 _initialFunds = 0, dev::s256 _contractStorageLimit = 32 );
/// which uses it. If you have no preexisting database then set BaseState to something other

State()
: State( dev::Invalid256, skale::OverlayDB(),
Expand Down Expand Up @@ -394,7 +373,7 @@ class State {
/// Check if state is empty
bool empty() const;

// const dev::db::DBImpl* getOriginalDb() const { return m_orig_db.get(); }
const dev::db::DBImpl* getOriginalDb() const { return m_orig_db.get(); }

void resetStorageChanges() {
storageUsage.clear();
Expand Down Expand Up @@ -425,7 +404,7 @@ class State {

/// Open a DB - useful for passing into the constructor & keeping for other states that are
/// necessary.
static OverlayDB openDB( boost::filesystem::path const& _path, dev::h256 const& _genesisHash,
OverlayDB openDB( boost::filesystem::path const& _path, dev::h256 const& _genesisHash,
dev::WithExisting _we = dev::WithExisting::Trust );

/// Turns all "touched" empty accounts into non-alive accounts.
Expand Down Expand Up @@ -482,9 +461,9 @@ class State {
std::shared_ptr< boost::shared_mutex > x_db_ptr;
std::shared_ptr< OverlayDB > m_db_ptr; ///< Our overlay for the state.
std::shared_ptr< OverlayFS > m_fs_ptr; ///< Our overlay for the file system operations.
// // HACK
// // TODO Implement DB-registry, remove it!
// std::shared_ptr< dev::db::DBImpl > m_orig_db;
// HACK
// TODO Implement DB-registry, remove it!
std::shared_ptr< dev::db::DBImpl > m_orig_db;
std::shared_ptr< size_t > m_storedVersion;
size_t m_currentVersion;
mutable std::unordered_map< dev::Address, dev::eth::Account > m_cache; ///< Our address cache.
Expand Down
24 changes: 12 additions & 12 deletions libweb3jsonrpc/Debug.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -299,18 +299,18 @@ uint64_t Debug::debug_getSnapshotHashCalculationTime() {
return m_eth.getSnapshotHashCalculationTime();
}

// uint64_t Debug::debug_doStateDbCompaction() {
// auto t1 = boost::chrono::high_resolution_clock::now();
// m_eth.doStateDbCompaction();
// auto t2 = boost::chrono::high_resolution_clock::now();
uint64_t Debug::debug_doStateDbCompaction() {
auto t1 = boost::chrono::high_resolution_clock::now();
m_eth.doStateDbCompaction();
auto t2 = boost::chrono::high_resolution_clock::now();

// return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
//}
return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
}

// uint64_t Debug::debug_doBlocksDbCompaction() {
// auto t1 = boost::chrono::high_resolution_clock::now();
// m_eth.doBlocksDbCompaction();
// auto t2 = boost::chrono::high_resolution_clock::now();
uint64_t Debug::debug_doBlocksDbCompaction() {
auto t1 = boost::chrono::high_resolution_clock::now();
m_eth.doBlocksDbCompaction();
auto t2 = boost::chrono::high_resolution_clock::now();

// return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
//}
return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
}
4 changes: 2 additions & 2 deletions libweb3jsonrpc/Debug.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ class Debug : public DebugFace {
virtual uint64_t debug_getSnapshotCalculationTime() override;
virtual uint64_t debug_getSnapshotHashCalculationTime() override;

// virtual uint64_t debug_doStateDbCompaction() override;
// virtual uint64_t debug_doBlocksDbCompaction() override;
virtual uint64_t debug_doStateDbCompaction() override;
virtual uint64_t debug_doBlocksDbCompaction() override;

private:
eth::Client const& m_eth;
Expand Down
Loading

0 comments on commit 2cee896

Please sign in to comment.