Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "Feature/is 686 new leveldb version" #1517

Merged
merged 1 commit into from
May 5, 2023
Merged

Revert "Feature/is 686 new leveldb version" #1517

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,8 +96,8 @@ endif()
option( HUNTER_RUN_UPLOAD "Upload binaries to the cache server" ${run_upload} )

include( HunterGate )
#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) #leveldb 1.22
HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.76.tar.gz" SHA1 "c7b60993e841850e2c449afd454f5d5aa4ec04e4" LOCAL )
#HunterGate( URL "https://github.com/ruslo/hunter/archive/v0.23.214.tar.gz" SHA1 "e14bc153a7f16d6a5eeec845fb0283c8fad8c358" LOCAL ) new leveldb

set( CMAKE_CXX_STANDARD 17 )

Expand Down
19 changes: 5 additions & 14 deletions libdevcore/LevelDB.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,16 +88,13 @@ leveldb::ReadOptions LevelDB::defaultReadOptions() {
}

leveldb::WriteOptions LevelDB::defaultWriteOptions() {
leveldb::WriteOptions writeOptions = leveldb::WriteOptions();
writeOptions.sync = true;
return writeOptions;
return leveldb::WriteOptions();
}

leveldb::Options LevelDB::defaultDBOptions() {
leveldb::Options options;
options.create_if_missing = true;
options.max_open_files = c_maxOpenLeveldbFiles;
options.filter_policy = leveldb::NewBloomFilterPolicy( 10 );
return options;
}

Expand All @@ -106,21 +103,15 @@ LevelDB::LevelDB( boost::filesystem::path const& _path, leveldb::ReadOptions _re
: m_db( nullptr ),
m_readOptions( std::move( _readOptions ) ),
m_writeOptions( std::move( _writeOptions ) ),
m_options( std::move( _dbOptions ) ),
m_path( _path ) {
auto db = static_cast< leveldb::DB* >( nullptr );
auto const status = leveldb::DB::Open( m_options, _path.string(), &db );
auto const status = leveldb::DB::Open( _dbOptions, _path.string(), &db );
checkStatus( status, _path );

assert( db );
m_db.reset( db );
}

LevelDB::~LevelDB() {
if ( m_options.filter_policy )
delete m_options.filter_policy;
}

std::string LevelDB::lookup( Slice _key ) const {
leveldb::Slice const key( _key.data(), _key.size() );
std::string value;
Expand Down Expand Up @@ -236,9 +227,9 @@ h256 LevelDB::hashBaseWithPrefix( char _prefix ) const {
return hash;
}

void LevelDB::doCompaction() const {
m_db->CompactRange( nullptr, nullptr );
}
// void LevelDB::doCompaction() const {
// m_db->CompactRange( NULL, NULL );
//}

} // namespace db
} // namespace dev
6 changes: 1 addition & 5 deletions libdevcore/LevelDB.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@
#include "db.h"

#include <leveldb/db.h>
#include <leveldb/filter_policy.h>
#include <leveldb/write_batch.h>
#include <boost/filesystem.hpp>

Expand All @@ -39,8 +38,6 @@ class LevelDB : public DatabaseFace {
leveldb::WriteOptions _writeOptions = defaultWriteOptions(),
leveldb::Options _dbOptions = defaultDBOptions() );

~LevelDB();

std::string lookup( Slice _key ) const override;
bool exists( Slice _key ) const override;
void insert( Slice _key, Slice _value ) override;
Expand All @@ -54,13 +51,12 @@ class LevelDB : public DatabaseFace {
h256 hashBase() const override;
h256 hashBaseWithPrefix( char _prefix ) const;

void doCompaction() const;
// void doCompaction() const;

private:
std::unique_ptr< leveldb::DB > m_db;
leveldb::ReadOptions const m_readOptions;
leveldb::WriteOptions const m_writeOptions;
leveldb::Options m_options;
boost::filesystem::path const m_path;
};

Expand Down
18 changes: 9 additions & 9 deletions libethereum/BlockChain.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,9 @@ void BlockChain::open( fs::path const& _path, bool _applyPatches, WithExisting _

try {
fs::create_directories( chainPath / fs::path( "blocks_and_extras" ) );
m_rotator = std::make_shared< batched_io::rotating_db_io >(
auto rotator = std::make_shared< batched_io::rotating_db_io >(
chainPath / fs::path( "blocks_and_extras" ), 5, chainParams().nodeInfo.archiveMode );
m_rotating_db = std::make_shared< db::ManuallyRotatingLevelDB >( m_rotator );
m_rotating_db = std::make_shared< db::ManuallyRotatingLevelDB >( rotator );
auto db = std::make_shared< batched_io::batched_db >();
db->open( m_rotating_db );
m_db = db;
Expand Down Expand Up @@ -1364,13 +1364,13 @@ void BlockChain::clearCaches() {
}
}

void BlockChain::doLevelDbCompaction() const {
for ( auto it = m_rotator->begin(); it != m_rotator->end(); ++it ) {
dev::db::LevelDB* ldb = dynamic_cast< dev::db::LevelDB* >( it->get() );
assert( ldb );
ldb->doCompaction();
}
}
// void BlockChain::doLevelDbCompaction() const {
// for ( auto it = m_rotator->begin(); it != m_rotator->end(); ++it ) {
// dev::db::LevelDB* ldb = dynamic_cast< dev::db::LevelDB* >( it->get() );
// assert( ldb );
// ldb->doCompaction();
// }
//}

void BlockChain::checkConsistency() {
DEV_WRITE_GUARDED( x_details ) { m_details.clear(); }
Expand Down
6 changes: 3 additions & 3 deletions libethereum/BlockChain.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -470,8 +470,8 @@ class BlockChain {
void open( boost::filesystem::path const& _path, bool _applyPatches, WithExisting _we );
/// Finalise everything and close the database.
void close();
/// compact db before snapshot
void doLevelDbCompaction() const;
// /// compact db before snapshot
// void doLevelDbCompaction() const;

private:
bool rotateDBIfNeeded( uint64_t pieceUsageBytes );
Expand Down Expand Up @@ -589,7 +589,7 @@ class BlockChain {
uint64_t m_maxStorageUsage;

/// The disk DBs. Thread-safe, so no need for locks.
std::shared_ptr< batched_io::rotating_db_io > m_rotator; // for compaction
// std::shared_ptr< batched_io::rotating_db_io > m_rotator; // for compaction
std::shared_ptr< db::ManuallyRotatingLevelDB > m_rotating_db; // rotate()
std::shared_ptr< batched_io::db_face > m_db; // insert()/commit()
std::unique_ptr< batched_io::db_splitter > m_db_splitter; // new_interface()
Expand Down
4 changes: 2 additions & 2 deletions libethereum/Client.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,9 +309,9 @@ class Client : public ClientBase, protected Worker {
return chainParams().sChain.nodeGroups[imaBLSPublicKeyGroupIndex].blsPublicKey;
}

void doStateDbCompaction() const { m_state.getOriginalDb()->doCompaction(); }
// void doStateDbCompaction() const { m_state.getOriginalDb()->doCompaction(); }

void doBlocksDbCompaction() const { m_bc.doLevelDbCompaction(); }
// void doBlocksDbCompaction() const { m_bc.doLevelDbCompaction(); }

std::pair< uint64_t, uint64_t > getBlocksDbUsage() const;

Expand Down
52 changes: 2 additions & 50 deletions libskale/State.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,52 +66,6 @@ using dev::eth::TransactionReceipt;
#define ETH_VMTRACE 0
#endif

State::State( dev::u256 const& _accountStartNonce, boost::filesystem::path const& _dbPath,
dev::h256 const& _genesis, BaseState _bs, dev::u256 _initialFunds,
dev::s256 _contractStorageLimit )
: x_db_ptr( make_shared< boost::shared_mutex >() ),
m_storedVersion( make_shared< size_t >( 0 ) ),
m_currentVersion( *m_storedVersion ),
m_accountStartNonce( _accountStartNonce ),
m_initial_funds( _initialFunds ),
contractStorageLimit_( _contractStorageLimit )
#ifdef HISTORIC_STATE
,
m_historicState( _accountStartNonce,
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_STATE_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ),
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_ROOTS_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) )
#endif
{
m_db_ptr = make_shared< OverlayDB >( openDB( _dbPath, _genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust : dev::WithExisting::Kill ) );

auto state = createStateReadOnlyCopy();
totalStorageUsed_ = state.storageUsedTotal();
#ifdef HISTORIC_STATE
m_historicState.setRootFromDB();
#endif
m_fs_ptr = state.fs();
if ( _bs == BaseState::PreExisting ) {
clog( VerbosityDebug, "statedb" ) << cc::debug( "Using existing database" );
} else if ( _bs == BaseState::Empty ) {
// Initialise to the state entailed by the genesis block; this guarantees the trie is built
// correctly.
m_db_ptr->clearDB();
} else {
throw std::logic_error( "Not implemented" );
}
}

State::State( u256 const& _accountStartNonce, OverlayDB const& _db,
#ifdef HISTORIC_STATE
dev::OverlayDB const& _historicDb, dev::OverlayDB const& _historicBlockToStateRootDb,
Expand Down Expand Up @@ -224,9 +178,9 @@ skale::OverlayDB State::openDB(

fs::path state_path = path / fs::path( "state" );
try {
m_orig_db.reset( new db::DBImpl( state_path ) );
std::shared_ptr< db::DatabaseFace > db( new db::DBImpl( state_path ) );
std::unique_ptr< batched_io::batched_db > bdb = make_unique< batched_io::batched_db >();
bdb->open( m_orig_db );
bdb->open( db );
assert( bdb->is_open() );
clog( VerbosityDebug, "statedb" ) << cc::success( "Opened state DB." );
return OverlayDB( std::move( bdb ) );
Expand Down Expand Up @@ -259,7 +213,6 @@ State::State( const State& _s )
std::logic_error( "Can't copy locked for writing state object" );
}
m_db_ptr = _s.m_db_ptr;
m_orig_db = _s.m_orig_db;
m_storedVersion = _s.m_storedVersion;
m_currentVersion = _s.m_currentVersion;
m_cache = _s.m_cache;
Expand All @@ -281,7 +234,6 @@ State& State::operator=( const State& _s ) {
std::logic_error( "Can't copy locked for writing state object" );
}
m_db_ptr = _s.m_db_ptr;
m_orig_db = _s.m_orig_db;
m_storedVersion = _s.m_storedVersion;
m_currentVersion = _s.m_currentVersion;
m_cache = _s.m_cache;
Expand Down
35 changes: 28 additions & 7 deletions libskale/State.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,8 +179,29 @@ class State {
// This is called once in the client during the client creation
explicit State( dev::u256 const& _accountStartNonce, boost::filesystem::path const& _dbPath,
dev::h256 const& _genesis, BaseState _bs = BaseState::PreExisting,
dev::u256 _initialFunds = 0, dev::s256 _contractStorageLimit = 32 );
/// which uses it. If you have no preexisting database then set BaseState to something other
dev::u256 _initialFunds = 0, dev::s256 _contractStorageLimit = 32 )
: State( _accountStartNonce,
openDB( _dbPath, _genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
#ifdef HISTORIC_STATE
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_STATE_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
dev::eth::HistoricState::openDB(
boost::filesystem::path( std::string( _dbPath.string() )
.append( "/" )
.append( dev::eth::HISTORIC_ROOTS_DIR ) ),
_genesis,
_bs == BaseState::PreExisting ? dev::WithExisting::Trust :
dev::WithExisting::Kill ),
#endif /// which uses it. If you have no preexisting database then set BaseState to something other
_bs, _initialFunds, _contractStorageLimit ) {
}

State()
: State( dev::Invalid256, skale::OverlayDB(),
Expand Down Expand Up @@ -373,7 +394,7 @@ class State {
/// Check if state is empty
bool empty() const;

const dev::db::DBImpl* getOriginalDb() const { return m_orig_db.get(); }
// const dev::db::DBImpl* getOriginalDb() const { return m_orig_db.get(); }

void resetStorageChanges() {
storageUsage.clear();
Expand Down Expand Up @@ -404,7 +425,7 @@ class State {

/// Open a DB - useful for passing into the constructor & keeping for other states that are
/// necessary.
OverlayDB openDB( boost::filesystem::path const& _path, dev::h256 const& _genesisHash,
static OverlayDB openDB( boost::filesystem::path const& _path, dev::h256 const& _genesisHash,
dev::WithExisting _we = dev::WithExisting::Trust );

/// Turns all "touched" empty accounts into non-alive accounts.
Expand Down Expand Up @@ -461,9 +482,9 @@ class State {
std::shared_ptr< boost::shared_mutex > x_db_ptr;
std::shared_ptr< OverlayDB > m_db_ptr; ///< Our overlay for the state.
std::shared_ptr< OverlayFS > m_fs_ptr; ///< Our overlay for the file system operations.
// HACK
// TODO Implement DB-registry, remove it!
std::shared_ptr< dev::db::DBImpl > m_orig_db;
// // HACK
// // TODO Implement DB-registry, remove it!
// std::shared_ptr< dev::db::DBImpl > m_orig_db;
std::shared_ptr< size_t > m_storedVersion;
size_t m_currentVersion;
mutable std::unordered_map< dev::Address, dev::eth::Account > m_cache; ///< Our address cache.
Expand Down
24 changes: 12 additions & 12 deletions libweb3jsonrpc/Debug.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,18 +299,18 @@ uint64_t Debug::debug_getSnapshotHashCalculationTime() {
return m_eth.getSnapshotHashCalculationTime();
}

uint64_t Debug::debug_doStateDbCompaction() {
auto t1 = boost::chrono::high_resolution_clock::now();
m_eth.doStateDbCompaction();
auto t2 = boost::chrono::high_resolution_clock::now();
// uint64_t Debug::debug_doStateDbCompaction() {
// auto t1 = boost::chrono::high_resolution_clock::now();
// m_eth.doStateDbCompaction();
// auto t2 = boost::chrono::high_resolution_clock::now();

return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
}
// return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
//}

uint64_t Debug::debug_doBlocksDbCompaction() {
auto t1 = boost::chrono::high_resolution_clock::now();
m_eth.doBlocksDbCompaction();
auto t2 = boost::chrono::high_resolution_clock::now();
// uint64_t Debug::debug_doBlocksDbCompaction() {
// auto t1 = boost::chrono::high_resolution_clock::now();
// m_eth.doBlocksDbCompaction();
// auto t2 = boost::chrono::high_resolution_clock::now();

return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
}
// return boost::chrono::duration_cast< boost::chrono::milliseconds >( t2 - t1 ).count();
//}
4 changes: 2 additions & 2 deletions libweb3jsonrpc/Debug.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,8 @@ class Debug : public DebugFace {
virtual uint64_t debug_getSnapshotCalculationTime() override;
virtual uint64_t debug_getSnapshotHashCalculationTime() override;

virtual uint64_t debug_doStateDbCompaction() override;
virtual uint64_t debug_doBlocksDbCompaction() override;
// virtual uint64_t debug_doStateDbCompaction() override;
// virtual uint64_t debug_doBlocksDbCompaction() override;

private:
eth::Client const& m_eth;
Expand Down
Loading