From 637ae0d422c89450dba400a0967062cfad49c965 Mon Sep 17 00:00:00 2001 From: Kevin Heifner Date: Fri, 29 Sep 2017 14:37:19 -0500 Subject: [PATCH] Skeleton of db_plugin created #172 --- libraries/chain/chain_controller.cpp | 16 +- .../include/eos/chain/chain_controller.hpp | 14 +- plugins/CMakeLists.txt | 1 + plugins/chain_plugin/CMakeLists.txt | 2 +- plugins/chain_plugin/chain_plugin.cpp | 15 +- plugins/db_plugin/CMakeLists.txt | 16 ++ plugins/db_plugin/db_plugin.cpp | 239 ++++++++++++++++++ .../include/eos/db_plugin/db_plugin.hpp | 36 +++ programs/eosd/CMakeLists.txt | 2 +- programs/eosd/main.cpp | 2 + 10 files changed, 338 insertions(+), 5 deletions(-) create mode 100644 plugins/db_plugin/CMakeLists.txt create mode 100644 plugins/db_plugin/db_plugin.cpp create mode 100644 plugins/db_plugin/include/eos/db_plugin/db_plugin.hpp diff --git a/libraries/chain/chain_controller.cpp b/libraries/chain/chain_controller.cpp index aa4757123f6..85acfd2753f 100644 --- a/libraries/chain/chain_controller.cpp +++ b/libraries/chain/chain_controller.cpp @@ -723,6 +723,9 @@ void chain_controller::_apply_block(const signed_block& next_block) // notify observers that the block has been applied // TODO: do this outside the write lock...? applied_block( next_block ); //emit + if (_currently_replaying_blocks) + applied_irreversible_block(next_block); + } FC_CAPTURE_AND_RETHROW( (next_block.block_num()) ) } @@ -1167,9 +1170,13 @@ void chain_controller::initialize_chain(chain_initializer_interface& starter) } FC_CAPTURE_AND_RETHROW() } chain_controller::chain_controller(database& database, fork_database& fork_db, block_log& blocklog, - chain_initializer_interface& starter, unique_ptr admin) + chain_initializer_interface& starter, unique_ptr admin, + const applied_irreverisable_block_func& applied_func) : _db(database), _fork_db(fork_db), _block_log(blocklog), _admin(std::move(admin)) { + if (applied_func) + applied_irreversible_block.connect(*applied_func); + initialize_indexes(); starter.register_types(*this, _db); @@ -1194,6 +1201,12 @@ chain_controller::~chain_controller() { void chain_controller::replay() { ilog("Replaying blockchain"); auto start = fc::time_point::now(); + + auto on_exit = fc::make_scoped_exit([this](){ + _currently_replaying_blocks = false; + }); + _currently_replaying_blocks = true; + auto last_block = _block_log.read_head(); if (!last_block) { elog("No blocks in block log; skipping replay"); @@ -1353,6 +1366,7 @@ void chain_controller::update_last_irreversible_block() auto block = fetch_block_by_number(block_to_write); assert(block); _block_log.append(*block); + applied_irreversible_block(*block); } // Trim fork_database and undo histories diff --git a/libraries/chain/include/eos/chain/chain_controller.hpp b/libraries/chain/include/eos/chain/chain_controller.hpp index 6b1274e4382..90a424a0663 100644 --- a/libraries/chain/include/eos/chain/chain_controller.hpp +++ b/libraries/chain/include/eos/chain/chain_controller.hpp @@ -47,6 +47,7 @@ namespace eos { namespace chain { using database = chainbase::database; using boost::signals2::signal; + using applied_irreverisable_block_func = fc::optional::slot_type>; struct path_cons_list; /** @@ -56,7 +57,8 @@ namespace eos { namespace chain { class chain_controller { public: chain_controller(database& database, fork_database& fork_db, block_log& blocklog, - chain_initializer_interface& starter, unique_ptr admin); + chain_initializer_interface& starter, unique_ptr admin, + const applied_irreverisable_block_func& applied_func = {}); chain_controller(chain_controller&&) = default; ~chain_controller(); @@ -70,6 +72,15 @@ namespace eos { namespace chain { */ signal applied_block; + /** + * This signal is emitted after irreversible block is written to disk. + * + * You may not yield from this callback because the blockchain is holding + * the write lock and may be in an "inconstant state" until after it is + * released. + */ + signal applied_irreversible_block; + /** * This signal is emitted any time a new transaction is added to the pending * block state. @@ -377,6 +388,7 @@ namespace eos { namespace chain { deque _pending_transactions; bool _currently_applying_block = false; + bool _currently_replaying_blocks = false; uint64_t _skip_flags = 0; flat_map _checkpoints; diff --git a/plugins/CMakeLists.txt b/plugins/CMakeLists.txt index b05c67ca527..05f5c08ebeb 100644 --- a/plugins/CMakeLists.txt +++ b/plugins/CMakeLists.txt @@ -2,6 +2,7 @@ add_subdirectory(net_plugin) #add_subdirectory(p2p_plugin) add_subdirectory(http_plugin) add_subdirectory(database_plugin) +add_subdirectory(db_plugin) add_subdirectory(chain_plugin) add_subdirectory(chain_api_plugin) add_subdirectory(producer_plugin) diff --git a/plugins/chain_plugin/CMakeLists.txt b/plugins/chain_plugin/CMakeLists.txt index 5e0f33ef551..6ac16f0ae53 100644 --- a/plugins/chain_plugin/CMakeLists.txt +++ b/plugins/chain_plugin/CMakeLists.txt @@ -3,7 +3,7 @@ add_library( chain_plugin chain_plugin.cpp ${HEADERS} ) -target_link_libraries( chain_plugin database_plugin eos_native_contract eos_chain appbase ) +target_link_libraries( chain_plugin database_plugin db_plugin eos_native_contract eos_chain appbase ) target_include_directories( chain_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) install( TARGETS diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index f25847cc0c6..881f3fbfe4e 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -6,6 +6,8 @@ #include #include +#include + #include #include #include @@ -108,10 +110,16 @@ void chain_plugin::plugin_initialize(const variables_map& options) { if (options.at("replay-blockchain").as()) { ilog("Replay requested: wiping database"); app().get_plugin().wipe_database(); + if (db_plugin* db = app().find_plugin()) { + db->wipe_database(); + } } if (options.at("resync-blockchain").as()) { ilog("Resync requested: wiping blocks"); app().get_plugin().wipe_database(); + if (db_plugin* db = app().find_plugin()) { + db->wipe_database(); + } fc::remove_all(my->block_log_dir); } if (options.at("skip-transaction-signatures").as()) { @@ -144,6 +152,11 @@ void chain_plugin::plugin_initialize(const variables_map& options) { void chain_plugin::plugin_startup() { try { auto& db = app().get_plugin().db(); + eos::chain::applied_irreverisable_block_func applied_func; + if (db_plugin* plugin = app().find_plugin()) { + ilog("Blockchain configured with external database."); + applied_func = [plugin](const chain::signed_block& b) { plugin->applied_irreversible_block(b); }; + } FC_ASSERT( fc::exists( my->genesis_file ), "unable to find genesis file '${f}', check --genesis-json argument", @@ -160,7 +173,7 @@ void chain_plugin::plugin_startup() my->block_logger = block_log(my->block_log_dir); my->chain_id = genesis.compute_chain_id(); my->chain = chain_controller(db, *my->fork_db, *my->block_logger, - initializer, native_contract::make_administrator()); + initializer, native_contract::make_administrator(), applied_func); if(!my->readonly) { ilog("starting chain in read/write mode"); diff --git a/plugins/db_plugin/CMakeLists.txt b/plugins/db_plugin/CMakeLists.txt new file mode 100644 index 00000000000..ffd873c05f6 --- /dev/null +++ b/plugins/db_plugin/CMakeLists.txt @@ -0,0 +1,16 @@ +file(GLOB HEADERS "include/eos/db_plugin/*.hpp") +add_library( db_plugin + db_plugin.cpp + ${HEADERS} ) + +target_link_libraries( db_plugin chain_plugin eos_chain appbase ) +target_include_directories( db_plugin PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/include" ) + +install( TARGETS + db_plugin + + RUNTIME DESTINATION bin + LIBRARY DESTINATION lib + ARCHIVE DESTINATION lib +) +install( FILES ${HEADERS} DESTINATION "include/eos/db_plugin" ) diff --git a/plugins/db_plugin/db_plugin.cpp b/plugins/db_plugin/db_plugin.cpp new file mode 100644 index 00000000000..8024d60e270 --- /dev/null +++ b/plugins/db_plugin/db_plugin.cpp @@ -0,0 +1,239 @@ +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +namespace fc { class variant; } + +namespace eos { + +using chain::AccountName; +using chain::block_id_type; +using chain::PermissionName; +using chain::ProcessedTransaction; +using chain::signed_block; +using boost::multi_index_container; +using chain::transaction_id_type; +using namespace boost::multi_index; + +class db_plugin_impl { +public: + void applied_irreversible_block(const signed_block&); + + chain_plugin* chain_plug; + std::set filter_on; + +private: + struct block_comp + { + bool operator()(const block_id_type& a, const block_id_type& b) const + { + return chain::block_header::num_from_id(a) > chain::block_header::num_from_id(b); + } + }; + typedef std::multimap block_transaction_id_map; + + bool is_scope_relevant(const eos::types::Vector& scope); + static void add(chainbase::database& db, const vector& keys, const AccountName& account_name, const PermissionName& permission); + + template + static void remove(chainbase::database& db, const AccountName& account_name, const PermissionName& permission) + { + const auto& idx = db.get_index(); + auto& mutatable_idx = db.get_mutable_index(); + auto range = idx.equal_range( boost::make_tuple( account_name, permission ) ); + + for (auto acct_perm = range.first; acct_perm != range.second; ++acct_perm) + { + mutatable_idx.remove(*acct_perm); + } + } + + static void add(chainbase::database& db, const vector& controlling_accounts, const AccountName& account_name, const PermissionName& permission); + static const AccountName NEW_ACCOUNT; + static const AccountName UPDATE_AUTH; + static const AccountName DELETE_AUTH; + static const PermissionName OWNER; + static const PermissionName ACTIVE; + static const PermissionName RECOVERY; +}; +const AccountName db_plugin_impl::NEW_ACCOUNT = "newaccount"; +const AccountName db_plugin_impl::UPDATE_AUTH = "updateauth"; +const AccountName db_plugin_impl::DELETE_AUTH = "deleteauth"; +const PermissionName db_plugin_impl::OWNER = "owner"; +const PermissionName db_plugin_impl::ACTIVE = "active"; +const PermissionName db_plugin_impl::RECOVERY = "recovery"; + +void db_plugin_impl::applied_irreversible_block(const signed_block& block) +{ + const auto block_id = block.id(); + ilog("block ${bid}", ("bid", block_id)); +// auto& db = chain_plug->chain().get_mutable_database(); + const bool check_relevance = filter_on.size(); + for (const auto& cycle : block.cycles) + { + for (const auto& thread : cycle) + { + for (const auto& trx : thread.user_input) + { + if (check_relevance && !is_scope_relevant(trx.scope)) + continue; + + ilog("block ${bid} : ${tid}", ("bid", block_id)("tid", trx.id())); + +// db.create([&block_id,&trx](transaction_history_object& transaction_history) { +// transaction_history.block_id = block_id; +// transaction_history.transaction_id = trx.id(); +// }); + + for (const auto& account_name : trx.scope) + { +// db.create([&trx,&account_name](account_transaction_history_object& account_transaction_history) { +// account_transaction_history.account_name = account_name; +// account_transaction_history.transaction_id = trx.id(); +// }); + } + + for (const chain::Message& msg : trx.messages) + { + if (msg.code == config::EosContractName) + { + if (msg.type == NEW_ACCOUNT) + { + const auto create = msg.as(); +// add(db, create.owner.keys, create.name, OWNER); +// add(db, create.active.keys, create.name, ACTIVE); +// add(db, create.recovery.keys, create.name, RECOVERY); +// +// add(db, create.owner.accounts, create.name, OWNER); +// add(db, create.active.accounts, create.name, ACTIVE); +// add(db, create.recovery.accounts, create.name, RECOVERY); + } + else if (msg.type == UPDATE_AUTH) + { + const auto update = msg.as(); +// remove(db, update.account, update.permission); +// add(db, update.authority.keys, update.account, update.permission); +// +// remove(db, update.account, update.permission); +// add(db, update.authority.accounts, update.account, update.permission); + } + else if (msg.type == DELETE_AUTH) + { + const auto del = msg.as(); +// remove(db, del.account, del.permission); +// +// remove(db, del.account, del.permission); + } + } + } + } + } + } +} + +void db_plugin_impl::add(chainbase::database& db, const vector& keys, const AccountName& account_name, const PermissionName& permission) +{ + for (auto pub_key_weight : keys ) + { +// db.create([&](public_key_history_object& obj) { +// obj.public_key = pub_key_weight.key; +// obj.account_name = account_name; +// obj.permission = permission; +// }); + } +} + +void db_plugin_impl::add(chainbase::database& db, const vector& controlling_accounts, const AccountName& account_name, const PermissionName& permission) +{ + for (auto controlling_account : controlling_accounts ) + { +// db.create([&](account_control_history_object& obj) { +// obj.controlled_account = account_name; +// obj.controlled_permission = permission; +// obj.controlling_account = controlling_account.permission.account; +// }); + } +} + +bool db_plugin_impl::is_scope_relevant(const eos::types::Vector& scope) +{ + for (const AccountName& account_name : scope) + if (filter_on.count(account_name)) + return true; + + return false; +} + + +db_plugin::db_plugin() +:my(new db_plugin_impl()) +{ +} + +db_plugin::~db_plugin() +{ +} + +void db_plugin::set_program_options(options_description& cli, options_description& cfg) +{ + cfg.add_options() + ("filter_on_accounts,f", bpo::value>()->composing(), + "Track only transactions whose scopes involve the listed accounts. Default is to track all transactions.") + ; +} + +void db_plugin::wipe_database() { + if (true) + elog("TODO: db wipe_database"); + else + elog("ERROR: db_plugin::wipe_database() called before configuration or after startup. Ignoring."); +} + +void db_plugin::applied_irreversible_block(const signed_block& block) { + my->applied_irreversible_block(block); +} + + +void db_plugin::plugin_initialize(const variables_map& options) +{ + ilog("initializing db plugin"); + if(options.count("filter_on_accounts")) { + auto foa = options.at("filter_on_accounts").as>(); + for(auto filter_account : foa) + my->filter_on.emplace(filter_account); + } +} + +void db_plugin::plugin_startup() +{ + ilog("starting db plugin"); + // TODO: during chain startup we want to pushback on apply so that chainbase has to wait for db. + // TODO: assert that last irreversible in db is one less than received (on startup only?, every so often?) + my->chain_plug = app().find_plugin(); +// auto& db = my->chain_plug->chain().get_mutable_database(); +// db.add_index(); +// db.add_index(); +// db.add_index(); +// db.add_index(); + +} + +void db_plugin::plugin_shutdown() +{ +} + +} // namespace eos diff --git a/plugins/db_plugin/include/eos/db_plugin/db_plugin.hpp b/plugins/db_plugin/include/eos/db_plugin/db_plugin.hpp new file mode 100644 index 00000000000..88838719a2d --- /dev/null +++ b/plugins/db_plugin/include/eos/db_plugin/db_plugin.hpp @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include +#include + +namespace fc { class variant; } + +namespace eos { + + typedef std::shared_ptr db_plugin_impl_ptr; + +class db_plugin : public plugin { +public: + APPBASE_PLUGIN_REQUIRES((chain_plugin)) + + db_plugin(); + virtual ~db_plugin(); + + virtual void set_program_options(options_description& cli, options_description& cfg) override; + + // This may only be called after plugin_initialize() and before plugin_startup()! + void wipe_database(); + void applied_irreversible_block(const chain::signed_block& block); + + void plugin_initialize(const variables_map& options); + void plugin_startup(); + void plugin_shutdown(); + +private: + db_plugin_impl_ptr my; +}; + +} + diff --git a/programs/eosd/CMakeLists.txt b/programs/eosd/CMakeLists.txt index 217c61d289e..e8ba9d7f775 100644 --- a/programs/eosd/CMakeLists.txt +++ b/programs/eosd/CMakeLists.txt @@ -11,7 +11,7 @@ endif() target_link_libraries( eosd PRIVATE appbase - PRIVATE account_history_api_plugin account_history_plugin + PRIVATE account_history_api_plugin account_history_plugin db_plugin PRIVATE chain_api_plugin producer_plugin chain_plugin wallet_api_plugin PRIVATE net_plugin http_plugin PRIVATE eos_chain fc ${CMAKE_DL_LIBS} ${PLATFORM_SPECIFIC_LIBS} ) diff --git a/programs/eosd/main.cpp b/programs/eosd/main.cpp index ea54807f0c9..4cf9c9df03e 100644 --- a/programs/eosd/main.cpp +++ b/programs/eosd/main.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -24,6 +25,7 @@ int main(int argc, char** argv) app().register_plugin(); app().register_plugin(); app().register_plugin(); + app().register_plugin(); app().register_plugin(); if(!app().initialize(argc, argv)) return -1;