diff --git a/libraries/chain/block_log.cpp b/libraries/chain/block_log.cpp index 9336fa92dd6..11aafc167cb 100644 --- a/libraries/chain/block_log.cpp +++ b/libraries/chain/block_log.cpp @@ -262,4 +262,113 @@ namespace eosio { namespace chain { } } // construct_index + void block_log::repair_log( const fc::path& data_dir ) { + ilog("Recovering Block Log..."); + FC_ASSERT( fc::is_directory(data_dir) && fc::is_regular_file(data_dir / "blocks.log"), + "Block log not found in '${blocks_dir}'", ("blocks_dir", data_dir) ); + + auto now = fc::time_point::now(); + + auto blocks_dir = fc::canonical( data_dir ); + auto backup_dir = blocks_dir.parent_path(); + auto blocks_dir_name = blocks_dir.filename(); + if( blocks_dir_name.generic_string() == "." ) { + blocks_dir_name = backup_dir.filename(); + backup_dir = backup_dir.parent_path(); + FC_ASSERT( blocks_dir_name.generic_string() != ".", "Invalid path to blocks directory" ); + } + backup_dir = backup_dir / blocks_dir_name.generic_string().append("-").append( now ); + + FC_ASSERT( !fc::exists(backup_dir), + "Cannot move existing blocks directory to already existing directory '${new_blocks_dir}'", + ("new_blocks_dir", backup_dir) ); + + fc::rename( blocks_dir, backup_dir ); + ilog( "Moved existing blocks directory to backup location: '${new_blocks_dir}'", ("new_blocks_dir", backup_dir) ); + + fc::create_directories(blocks_dir); + auto block_log_path = blocks_dir / "blocks.log"; + + ilog( "Reconstructing '${new_block_log}' from backed up block log", ("new_block_log", block_log_path) ); + + std::fstream old_block_stream; + std::fstream new_block_stream; + + old_block_stream.open( (backup_dir / "blocks.log").generic_string().c_str(), LOG_READ ); + new_block_stream.open( block_log_path.generic_string().c_str(), LOG_WRITE ); + + uint64_t pos = 0; + uint64_t end_pos = old_block_stream.tellg(); + old_block_stream.seekg( 0, std::ios::end ); + end_pos = static_cast(old_block_stream.tellg()) - end_pos; + old_block_stream.seekg( 0 ); + + std::exception_ptr except_ptr; + vector incomplete_block_data; + optional bad_block; + uint32_t block_num = 0; + + while( pos < end_pos ) { + signed_block tmp; + + try { + fc::raw::unpack(old_block_stream, tmp); + } catch( ... ) { + except_ptr = std::current_exception(); + incomplete_block_data.resize( end_pos - pos ); + old_block_stream.read( incomplete_block_data.data(), incomplete_block_data.size() ); + break; + } + + uint64_t tmp_pos = std::numeric_limits::max(); + if( (static_cast(old_block_stream.tellg()) + sizeof(pos)) <= end_pos ) { + old_block_stream.read( reinterpret_cast(&tmp_pos), sizeof(tmp_pos) ); + } + if( pos != tmp_pos ) { + bad_block = tmp; + break; + } + + auto data = fc::raw::pack(tmp); + new_block_stream.write( data.data(), data.size() ); + new_block_stream.write( reinterpret_cast(&pos), sizeof(pos) ); + block_num = tmp.block_num(); + pos = new_block_stream.tellp(); + } + + if( bad_block.valid() ) { + ilog( "Recovered only up to block number ${num}. Last block in block log was not properly committed:\n${last_block}", + ("num", block_num)("last_block", *bad_block) ); + } else if( except_ptr ) { + std::string error_msg; + + try { + std::rethrow_exception(except_ptr); + } catch( const fc::exception& e ) { + error_msg = e.what(); + } catch( const std::exception& e ) { + error_msg = e.what(); + } catch( ... ) { + error_msg = "unrecognized exception"; + } + + ilog( "Recovered only up to block number ${num}. " + "The block ${next_num} could not be deserialized from the block log due to error:\n${error_msg}", + ("num", block_num)("next_num", block_num+1)("error_msg", error_msg) ); + + auto tail_path = blocks_dir / std::string("blocks-bad-tail-").append( now ).append(".log"); + if( !fc::exists(tail_path) && incomplete_block_data.size() > 0 ) { + std::fstream tail_stream; + tail_stream.open( tail_path.generic_string().c_str(), LOG_WRITE ); + tail_stream.write( incomplete_block_data.data(), incomplete_block_data.size() ); + + ilog( "Data at tail end of block log which should contain the (incomplete) serialization of block ${num} " + "has been written out to '${tail_path}'.", + ("num", block_num+1)("tail_path", tail_path) ); + } + } else { + ilog( "Existing block log was undamaged. Recovered all irreversible blocks up to block number ${num}.", ("num", block_num) ); + } + } + } } /// eosio::chain diff --git a/libraries/chain/include/eosio/chain/block_log.hpp b/libraries/chain/include/eosio/chain/block_log.hpp index 0ebca3e536b..8b3c8ff6e1d 100644 --- a/libraries/chain/include/eosio/chain/block_log.hpp +++ b/libraries/chain/include/eosio/chain/block_log.hpp @@ -58,6 +58,8 @@ namespace eosio { namespace chain { static const uint64_t npos = std::numeric_limits::max(); + static void repair_log( const fc::path& data_dir ); + private: void open(const fc::path& data_dir); void construct_index(); diff --git a/plugins/chain_plugin/chain_plugin.cpp b/plugins/chain_plugin/chain_plugin.cpp index 9eb0efbd7bd..7db34c316aa 100644 --- a/plugins/chain_plugin/chain_plugin.cpp +++ b/plugins/chain_plugin/chain_plugin.cpp @@ -49,7 +49,7 @@ class chain_plugin_impl { ,incoming_block_sync_method(app().get_method()) ,incoming_transaction_sync_method(app().get_method()) {} - + bfs::path block_log_dir; bfs::path genesis_file; time_point genesis_timestamp; @@ -116,6 +116,8 @@ void chain_plugin::set_program_options(options_description& cli, options_descrip cli.add_options() ("replay-blockchain", bpo::bool_switch()->default_value(false), "clear chain database and replay all blocks") + ("hard-replay-blockchain", bpo::bool_switch()->default_value(false), + "clear chain database, recover as many blocks as possible from the block log, and then replay those blocks") ("resync-blockchain", bpo::bool_switch()->default_value(false), "clear chain database and block log") ; @@ -158,16 +160,20 @@ void chain_plugin::plugin_initialize(const variables_map& options) { my->shared_memory_size = options.at("shared-memory-size-mb").as() * 1024 * 1024; } - if (options.at("replay-blockchain").as()) { - ilog("Replay requested: wiping database"); - fc::remove_all(app().data_dir() / default_shared_memory_dir); - } - if (options.at("resync-blockchain").as()) { + if( options.at("resync-blockchain").as() ) { ilog("Resync requested: wiping database and blocks"); fc::remove_all(app().data_dir() / default_shared_memory_dir); fc::remove_all(my->block_log_dir); + } else if( options.at("hard-replay-blockchain").as() ) { + ilog("Hard replay requested: wiping database"); + fc::remove_all(app().data_dir() / default_shared_memory_dir); + block_log::repair_log( my->block_log_dir ); + } else if( options.at("replay-blockchain").as() ) { + ilog("Replay requested: wiping database"); + fc::remove_all(app().data_dir() / default_shared_memory_dir); } + if(options.count("checkpoint")) { auto cps = options.at("checkpoint").as>(); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index a747d6401d5..92d66564f45 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,7 +22,7 @@ target_include_directories( plugin_test PUBLIC ${CMAKE_SOURCE_DIR}/plugins/net_p add_dependencies(plugin_test asserter test_api test_api_mem test_api_db test_api_multi_index exchange proxy identity identity_test stltest infinite eosio.system eosio.token eosio.bios test.inline multi_index_test noop dice eosio.msig) # -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/core_symbol.py.in ${CMAKE_CURRENT_SOURCE_DIR}/core_symbol.py) +configure_file(${CMAKE_CURRENT_SOURCE_DIR}/core_symbol.py.in ${CMAKE_CURRENT_BINARY_DIR}/core_symbol.py) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/sync/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/sync/test.sh COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/p2p_tests/dawn_515/test.sh ${CMAKE_CURRENT_BINARY_DIR}/p2p_tests/dawn_515/test.sh COPYONLY) @@ -32,7 +32,6 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/distributed-transactions-remote-test. configure_file(${CMAKE_CURRENT_SOURCE_DIR}/sample-cluster-map.json ${CMAKE_CURRENT_BINARY_DIR}/sample-cluster-map.json COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/restart-scenarios-test.py ${CMAKE_CURRENT_BINARY_DIR}/restart-scenarios-test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/testUtils.py ${CMAKE_CURRENT_BINARY_DIR}/testUtils.py COPYONLY) -configure_file(${CMAKE_CURRENT_SOURCE_DIR}/core_symbol.py ${CMAKE_CURRENT_BINARY_DIR}/core_symbol.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/nodeos_run_remote_test.py ${CMAKE_CURRENT_BINARY_DIR}/nodeos_run_remote_test.py COPYONLY) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/consensus-validation-malicious-producers.py ${CMAKE_CURRENT_BINARY_DIR}/consensus-validation-malicious-producers.py COPYONLY)