From dc68f6cd274bb95b432157613313d1e406f6c0de Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 16 Jul 2018 16:27:52 +0100 Subject: [PATCH 01/24] blockchain_ancestry: finds all ancestors of a tx, block, or chain /monero#4147 --- src/blockchain_utilities/CMakeLists.txt | 32 + .../blockchain_ancestry.cpp | 634 ++++++++++++++++++ 2 files changed, 666 insertions(+) create mode 100644 src/blockchain_utilities/blockchain_ancestry.cpp diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index d74980f2fc..c4f8cacb6b 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -91,6 +91,17 @@ monero_private_headers(blockchain_usage +set(blockchain_ancestry_sources + blockchain_ancestry.cpp + ) + +set(blockchain_ancestry_private_headers) + +monero_private_headers(blockchain_ancestry + ${blockchain_ancestry_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -183,3 +194,24 @@ set_property(TARGET blockchain_usage OUTPUT_NAME "aeon-blockchain-usage") install(TARGETS blockchain_usage DESTINATION bin) +monero_add_executable(blockchain_ancestry + ${blockchain_ancestry_sources} + ${blockchain_ancestry_private_headers}) + +target_link_libraries(blockchain_ancestry + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_ancestry + PROPERTY + OUTPUT_NAME "monero-blockchain-ancestry") +install(TARGETS blockchain_ancestry DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp new file mode 100644 index 0000000000..0de1c69df2 --- /dev/null +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -0,0 +1,634 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include +#include +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +struct ancestor +{ + uint64_t amount; + uint64_t offset; + + bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } +}; + +namespace std +{ + template<> struct hash + { + size_t operator()(const ancestor &a) const + { + crypto::hash h; + crypto::cn_fast_hash(&a, sizeof(a), h); + return reinterpret_cast(h); + } + }; +} + +static void add_ancestor(std::unordered_map &ancestry, uint64_t amount, uint64_t offset) +{ + std::pair::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); + if (!p.second) + { + ++p.first->second; + } +} + +static size_t get_full_ancestry(const std::unordered_map &ancestry) +{ + size_t count = 0; + for (const auto &i: ancestry) + count += i.second; + return count; +} + +static size_t get_deduplicated_ancestry(const std::unordered_map &ancestry) +{ + return ancestry.size(); +} + +static void add_ancestry(std::unordered_map> &ancestry, const crypto::hash &txid, const std::unordered_set &ancestors) +{ + std::pair>::iterator, bool> p = ancestry.insert(std::make_pair(txid, ancestors)); + if (!p.second) + { + for (const auto &e: ancestors) + p.first->second.insert(e); + } +} + +static void add_ancestry(std::unordered_map> &ancestry, const crypto::hash &txid, const ancestor &new_ancestor) +{ + std::pair>::iterator, bool> p = ancestry.insert(std::make_pair(txid, std::unordered_set())); + p.first->second.insert(new_ancestor); +} + +static std::unordered_set get_ancestry(const std::unordered_map> &ancestry, const crypto::hash &txid) +{ + std::unordered_map>::const_iterator i = ancestry.find(txid); + if (i == ancestry.end()) + { + //MERROR("txid ancestry not found: " << txid); + //throw std::runtime_error("txid ancestry not found"); + return std::unordered_set(); + } + return i->second; +} + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_txid = {"txid", "Get ancestry for this txid", ""}; + const command_line::arg_descriptor arg_height = {"height", "Get ancestry for all txes at this height", 0}; + const command_line::arg_descriptor arg_all = {"all", "Include the whole chain", false}; + const command_line::arg_descriptor arg_cache_outputs = {"cache-outputs", "Cache outputs (memory hungry)", false}; + const command_line::arg_descriptor arg_cache_txes = {"cache-txes", "Cache txes (memory hungry)", false}; + const command_line::arg_descriptor arg_cache_blocks = {"cache-blocks", "Cache blocks (memory hungry)", false}; + const command_line::arg_descriptor arg_include_coinbase = {"include-coinbase", "Including coinbase tx", false}; + const command_line::arg_descriptor arg_show_cache_stats = {"show-cache-stats", "Show cache statistics", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_all); + command_line::add_arg(desc_cmd_sett, arg_cache_outputs); + command_line::add_arg(desc_cmd_sett, arg_cache_txes); + command_line::add_arg(desc_cmd_sett, arg_cache_blocks); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_sett, arg_show_cache_stats); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_all = command_line::get_arg(vm, arg_all); + bool opt_cache_outputs = command_line::get_arg(vm, arg_cache_outputs); + bool opt_cache_txes = command_line::get_arg(vm, arg_cache_txes); + bool opt_cache_blocks = command_line::get_arg(vm, arg_cache_blocks); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + bool opt_show_cache_stats = command_line::get_arg(vm, arg_show_cache_stats); + + if ((!opt_txid_string.empty()) + !!opt_height + !!opt_all > 1) + { + std::cerr << "Only one of --txid, --height and --all can be given" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector start_txids; + + // forward method + if (opt_all) + { + uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; + std::unordered_map> ancestry; + std::unordered_map output_cache; + std::unordered_map tx_cache; + std::map block_cache; + + const uint64_t db_height = db->height(); + for (uint64_t h = 0; h < db_height; ++h) + { + size_t block_ancestry_size = 0; + const crypto::hash block_hash = db->get_block_hash_from_height(h); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + ++total_blocks; + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + block_cache.insert(std::make_pair(h, b)); + std::vector txids; + txids.reserve(1 + b.tx_hashes.size()); + if (opt_include_coinbase) + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + for (const auto &h: b.tx_hashes) + txids.push_back(h); + for (const crypto::hash &txid: txids) + { + printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); + fflush(stdout); + cryptonote::transaction tx; + std::unordered_map::const_iterator i = tx_cache.find(txid); + ++total_txes; + if (i != tx_cache.end()) + { + ++cached_txes; + tx = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + if (opt_cache_txes) + tx_cache.insert(std::make_pair(txid, tx)); + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + { + add_ancestry(ancestry, txid, std::unordered_set()); + } + else + { + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + add_ancestry(ancestry, txid, ancestor{amount, offset}); + cryptonote::block b; + auto iblock = block_cache.find(od.height); + ++total_blocks; + if (iblock != block_cache.end()) + { + ++cached_blocks; + b = iblock->second; + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + cryptonote::blobdata bd = db->get_block_blob(block_hash); + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + if (opt_cache_blocks) + block_cache.insert(std::make_pair(od.height, b)); + } + // find the tx which created this output + bool found = false; + std::unordered_map::const_iterator i = output_cache.find({amount, offset}); + ++total_outputs; + if (i != output_cache.end()) + { + ++cached_outputs; + add_ancestry(ancestry, txid, get_ancestry(ancestry, i->second)); + found = true; + } + else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(ancestry, txid, get_ancestry(ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + if (opt_cache_outputs) + output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + cryptonote::transaction tx2; + std::unordered_map::const_iterator i = tx_cache.find(block_txid); + ++total_txes; + if (i != tx_cache.end()) + { + ++cached_txes; + tx2 = i->second; + } + else + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + if (opt_cache_txes) + tx_cache.insert(std::make_pair(block_txid, tx2)); + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + add_ancestry(ancestry, txid, get_ancestry(ancestry, block_txid)); + if (opt_cache_outputs) + output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + const size_t ancestry_size = get_ancestry(ancestry, txid).size(); + block_ancestry_size += ancestry_size; + MINFO(txid << ": " << ancestry_size); + } + if (!txids.empty()) + { + std::string stats_msg; + if (opt_show_cache_stats) + stats_msg = std::string(", cache: txes ") + std::to_string(cached_txes*100./total_txes) + + ", blocks " + std::to_string(cached_blocks*100./total_blocks) + ", outputs " + + std::to_string(cached_outputs*100./total_outputs); + MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); + } + } + goto done; + } + + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + for (const crypto::hash &start_txid: start_txids) + { + LOG_PRINT_L0("Checking ancestry for txid " << start_txid); + + std::unordered_map ancestry; + + std::list txids; + txids.push_back(start_txid); + while (!txids.empty()) + { + const crypto::hash txid = txids.front(); + txids.pop_front(); + + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (coinbase) + continue; + + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + add_ancestor(ancestry, amount, offset); + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + + MINFO("Ancestry for " << start_txid << ": " << get_deduplicated_ancestry(ancestry) << " / " << get_full_ancestry(ancestry)); + for (const auto &i: ancestry) + { + MINFO(cryptonote::print_money(i.first.amount) << "/" << i.first.offset << ": " << i.second); + } + } + +done: + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} From 325b876043bd9e22d8ce2aafde8f67d7eca4d64a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 28 Jul 2018 00:56:37 +0100 Subject: [PATCH 02/24] blockchain_ancestry: add an incremental mode /monero#4147 --- .../blockchain_ancestry.cpp | 124 ++++++++++++++---- 1 file changed, 99 insertions(+), 25 deletions(-) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 0de1c69df2..c988b72503 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -30,8 +30,12 @@ #include #include #include +#include +#include +#include "common/unordered_containers_boost_serialization.h" #include "common/command_line.h" #include "common/varint.h" +#include "cryptonote_basic/cryptonote_boost_serialization.h" #include "cryptonote_core/tx_pool.h" #include "cryptonote_core/cryptonote_core.h" #include "cryptonote_core/blockchain.h" @@ -46,13 +50,22 @@ namespace po = boost::program_options; using namespace epee; using namespace cryptonote; +static bool stop_requested = false; + struct ancestor { uint64_t amount; uint64_t offset; bool operator==(const ancestor &other) const { return amount == other.amount && offset == other.offset; } + + template void serialize(t_archive &a, const unsigned int ver) + { + a & amount; + a & offset; + } }; +BOOST_CLASS_VERSION(ancestor, 0) namespace std { @@ -67,6 +80,25 @@ namespace std }; } +struct ancestry_state_t +{ + uint64_t height; + std::unordered_map> ancestry; + std::unordered_map output_cache; + std::unordered_map tx_cache; + std::unordered_map block_cache; + + template void serialize(t_archive &a, const unsigned int ver) + { + a & height; + a & ancestry; + a & output_cache; + a & tx_cache; + a & block_cache; + } +}; +BOOST_CLASS_VERSION(ancestry_state_t, 0) + static void add_ancestor(std::unordered_map &ancestry, uint64_t amount, uint64_t offset) { std::pair::iterator, bool> p = ancestry.insert(std::make_pair(ancestor{amount, offset}, 1)); @@ -274,13 +306,34 @@ int main(int argc, char* argv[]) if (opt_all) { uint64_t cached_txes = 0, cached_blocks = 0, cached_outputs = 0, total_txes = 0, total_blocks = 0, total_outputs = 0; - std::unordered_map> ancestry; - std::unordered_map output_cache; - std::unordered_map tx_cache; - std::map block_cache; + ancestry_state_t state; + + const std::string state_file_path = (boost::filesystem::path(opt_data_dir) / "ancestry-state.bin").string(); + LOG_PRINT_L0("Loading state data from " << state_file_path); + std::ifstream state_data_in; + state_data_in.open(state_file_path, std::ios_base::binary | std::ios_base::in); + if (!state_data_in.fail()) + { + try + { + boost::archive::portable_binary_iarchive a(state_data_in); + a >> state; + } + catch (const std::exception &e) + { + MERROR("Failed to load state data from " << state_file_path << ", restarting from scratch"); + state = ancestry_state_t(); + } + state_data_in.close(); + } + tools::signal_handler::install([](int type) { + stop_requested = true; + }); + + MINFO("Starting from height " << state.height); const uint64_t db_height = db->height(); - for (uint64_t h = 0; h < db_height; ++h) + for (uint64_t h = state.height; h < db_height; ++h) { size_t block_ancestry_size = 0; const crypto::hash block_hash = db->get_block_hash_from_height(h); @@ -293,7 +346,7 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_blocks) - block_cache.insert(std::make_pair(h, b)); + state.block_cache.insert(std::make_pair(h, b)); std::vector txids; txids.reserve(1 + b.tx_hashes.size()); if (opt_include_coinbase) @@ -305,9 +358,9 @@ int main(int argc, char* argv[]) printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); fflush(stdout); cryptonote::transaction tx; - std::unordered_map::const_iterator i = tx_cache.find(txid); + std::unordered_map::const_iterator i = state.tx_cache.find(txid); ++total_txes; - if (i != tx_cache.end()) + if (i != state.tx_cache.end()) { ++cached_txes; tx = i->second; @@ -326,12 +379,12 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_txes) - tx_cache.insert(std::make_pair(txid, tx)); + state.tx_cache.insert(std::make_pair(txid, tx)); } const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); if (coinbase) { - add_ancestry(ancestry, txid, std::unordered_set()); + add_ancestry(state.ancestry, txid, std::unordered_set()); } else { @@ -345,11 +398,11 @@ int main(int argc, char* argv[]) for (uint64_t offset: absolute_offsets) { const output_data_t od = db->get_output_key(amount, offset); - add_ancestry(ancestry, txid, ancestor{amount, offset}); + add_ancestry(state.ancestry, txid, ancestor{amount, offset}); cryptonote::block b; - auto iblock = block_cache.find(od.height); + auto iblock = state.block_cache.find(od.height); ++total_blocks; - if (iblock != block_cache.end()) + if (iblock != state.block_cache.end()) { ++cached_blocks; b = iblock->second; @@ -364,16 +417,16 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_blocks) - block_cache.insert(std::make_pair(od.height, b)); + state.block_cache.insert(std::make_pair(od.height, b)); } // find the tx which created this output bool found = false; - std::unordered_map::const_iterator i = output_cache.find({amount, offset}); + std::unordered_map::const_iterator i = state.output_cache.find({amount, offset}); ++total_outputs; - if (i != output_cache.end()) + if (i != state.output_cache.end()) { ++cached_outputs; - add_ancestry(ancestry, txid, get_ancestry(ancestry, i->second)); + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, i->second)); found = true; } else for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) @@ -384,9 +437,9 @@ int main(int argc, char* argv[]) if (txout.key == od.pubkey) { found = true; - add_ancestry(ancestry, txid, get_ancestry(ancestry, cryptonote::get_transaction_hash(b.miner_tx))); + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, cryptonote::get_transaction_hash(b.miner_tx))); if (opt_cache_outputs) - output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, cryptonote::get_transaction_hash(b.miner_tx))); break; } } @@ -401,9 +454,9 @@ int main(int argc, char* argv[]) if (found) break; cryptonote::transaction tx2; - std::unordered_map::const_iterator i = tx_cache.find(block_txid); + std::unordered_map::const_iterator i = state.tx_cache.find(block_txid); ++total_txes; - if (i != tx_cache.end()) + if (i != state.tx_cache.end()) { ++cached_txes; tx2 = i->second; @@ -422,7 +475,7 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_txes) - tx_cache.insert(std::make_pair(block_txid, tx2)); + state.tx_cache.insert(std::make_pair(block_txid, tx2)); } for (size_t out = 0; out < tx2.vout.size(); ++out) { @@ -432,9 +485,9 @@ int main(int argc, char* argv[]) if (txout.key == od.pubkey) { found = true; - add_ancestry(ancestry, txid, get_ancestry(ancestry, block_txid)); + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); if (opt_cache_outputs) - output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); break; } } @@ -459,7 +512,7 @@ int main(int argc, char* argv[]) } } } - const size_t ancestry_size = get_ancestry(ancestry, txid).size(); + const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); block_ancestry_size += ancestry_size; MINFO(txid << ": " << ancestry_size); } @@ -472,7 +525,28 @@ int main(int argc, char* argv[]) + std::to_string(cached_outputs*100./total_outputs); MINFO("Height " << h << ": " << (block_ancestry_size / txids.size()) << " average over " << txids.size() << stats_msg); } + state.height = h; + if (stop_requested) + break; } + + LOG_PRINT_L0("Saving state data to " << state_file_path); + std::ofstream state_data_out; + state_data_out.open(state_file_path, std::ios_base::binary | std::ios_base::out | std::ios::trunc); + if (!state_data_out.fail()) + { + try + { + boost::archive::portable_binary_oarchive a(state_data_out); + a << state; + } + catch (const std::exception &e) + { + MERROR("Failed to save state data to " << state_file_path); + } + state_data_out.close(); + } + goto done; } From 308c0efc7f7272f0b747f750a87a63178ccd8edd Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 30 Jul 2018 21:36:38 +0100 Subject: [PATCH 03/24] blockchain_ancestry: faster and uses less memory /monero#4147 --- .../blockchain_ancestry.cpp | 164 ++++++++++++------ 1 file changed, 114 insertions(+), 50 deletions(-) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index c988b72503..2f0bbffd64 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -73,31 +73,101 @@ namespace std { size_t operator()(const ancestor &a) const { - crypto::hash h; - crypto::cn_fast_hash(&a, sizeof(a), h); - return reinterpret_cast(h); + return a.amount ^ a.offset; // not that bad, since amount almost always have a high bit set, and offset doesn't } }; } +struct tx_data_t +{ + std::vector>> vin; + std::vector vout; + bool coinbase; + + tx_data_t(): coinbase(false) {} + tx_data_t(const cryptonote::transaction &tx) + { + coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); + if (!coinbase) + { + vin.reserve(tx.vin.size()); + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + vin.push_back(std::make_pair(txin.amount, cryptonote::relative_output_offsets_to_absolute(txin.key_offsets))); + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vin type"); + } + } + } + vout.reserve(tx.vout.size()); + for (size_t out = 0; out < tx.vout.size(); ++out) + { + if (tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx.vout[out].target); + vout.push_back(txout.key); + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << get_transaction_hash(tx)); + throw std::runtime_error("Bad vout type"); + } + } + } + + template void serialize(t_archive &a, const unsigned int ver) + { + a & coinbase; + a & vin; + a & vout; + } +}; + struct ancestry_state_t { uint64_t height; std::unordered_map> ancestry; std::unordered_map output_cache; - std::unordered_map tx_cache; - std::unordered_map block_cache; + std::unordered_map tx_cache; + std::vector block_cache; template void serialize(t_archive &a, const unsigned int ver) { a & height; a & ancestry; a & output_cache; - a & tx_cache; - a & block_cache; + if (ver < 1) + { + std::unordered_map old_tx_cache; + a & old_tx_cache; + for (const auto i: old_tx_cache) + tx_cache.insert(std::make_pair(i.first, ::tx_data_t(i.second))); + } + else + { + a & tx_cache; + } + if (ver < 2) + { + std::unordered_map old_block_cache; + a & old_block_cache; + block_cache.resize(old_block_cache.size()); + for (const auto i: old_block_cache) + block_cache[i.first] = i.second; + } + else + { + a & block_cache; + } } }; -BOOST_CLASS_VERSION(ancestry_state_t, 0) +BOOST_CLASS_VERSION(ancestry_state_t, 2) static void add_ancestor(std::unordered_map &ancestry, uint64_t amount, uint64_t offset) { @@ -333,6 +403,7 @@ int main(int argc, char* argv[]) MINFO("Starting from height " << state.height); const uint64_t db_height = db->height(); + state.block_cache.reserve(db_height); for (uint64_t h = state.height; h < db_height; ++h) { size_t block_ancestry_size = 0; @@ -346,7 +417,10 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_blocks) - state.block_cache.insert(std::make_pair(h, b)); + { + state.block_cache.resize(h + 1); + state.block_cache[h] = b; + } std::vector txids; txids.reserve(1 + b.tx_hashes.size()); if (opt_include_coinbase) @@ -357,13 +431,13 @@ int main(int argc, char* argv[]) { printf("%lu/%lu \r", (unsigned long)h, (unsigned long)db_height); fflush(stdout); - cryptonote::transaction tx; - std::unordered_map::const_iterator i = state.tx_cache.find(txid); + ::tx_data_t tx_data; + std::unordered_map::const_iterator i = state.tx_cache.find(txid); ++total_txes; if (i != state.tx_cache.end()) { ++cached_txes; - tx = i->second; + tx_data = i->second; } else { @@ -373,39 +447,38 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Failed to get txid " << txid << " from db"); return 1; } + cryptonote::transaction tx; if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) { LOG_PRINT_L0("Bad tx: " << txid); return 1; } + tx_data = ::tx_data_t(tx); if (opt_cache_txes) - state.tx_cache.insert(std::make_pair(txid, tx)); + state.tx_cache.insert(std::make_pair(txid, tx_data)); } - const bool coinbase = tx.vin.size() == 1 && tx.vin[0].type() == typeid(cryptonote::txin_gen); - if (coinbase) + if (tx_data.coinbase) { add_ancestry(state.ancestry, txid, std::unordered_set()); } else { - for (size_t ring = 0; ring < tx.vin.size(); ++ring) + for (size_t ring = 0; ring < tx_data.vin.size(); ++ring) { - if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + if (1) { - const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); - const uint64_t amount = txin.amount; - auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + const uint64_t amount = tx_data.vin[ring].first; + const std::vector &absolute_offsets = tx_data.vin[ring].second; for (uint64_t offset: absolute_offsets) { const output_data_t od = db->get_output_key(amount, offset); add_ancestry(state.ancestry, txid, ancestor{amount, offset}); cryptonote::block b; - auto iblock = state.block_cache.find(od.height); ++total_blocks; - if (iblock != state.block_cache.end()) + if (state.block_cache.size() > od.height && !state.block_cache[od.height].miner_tx.vin.empty()) { ++cached_blocks; - b = iblock->second; + b = state.block_cache[od.height]; } else { @@ -417,7 +490,10 @@ int main(int argc, char* argv[]) return 1; } if (opt_cache_blocks) - state.block_cache.insert(std::make_pair(od.height, b)); + { + state.block_cache.resize(od.height + 1); + state.block_cache[od.height] = b; + } } // find the tx which created this output bool found = false; @@ -453,13 +529,13 @@ int main(int argc, char* argv[]) { if (found) break; - cryptonote::transaction tx2; - std::unordered_map::const_iterator i = state.tx_cache.find(block_txid); + ::tx_data_t tx_data2; + std::unordered_map::const_iterator i = state.tx_cache.find(block_txid); ++total_txes; if (i != state.tx_cache.end()) { ++cached_txes; - tx2 = i->second; + tx_data2 = i->second; } else { @@ -469,32 +545,25 @@ int main(int argc, char* argv[]) LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); return 1; } - if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) { LOG_PRINT_L0("Bad tx: " << block_txid); return 1; } + tx_data2 = ::tx_data_t(tx); if (opt_cache_txes) - state.tx_cache.insert(std::make_pair(block_txid, tx2)); + state.tx_cache.insert(std::make_pair(block_txid, tx_data2)); } - for (size_t out = 0; out < tx2.vout.size(); ++out) + for (size_t out = 0; out < tx_data2.vout.size(); ++out) { - if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + if (tx_data2.vout[out] == od.pubkey) { - const auto &txout = boost::get(tx2.vout[out].target); - if (txout.key == od.pubkey) - { - found = true; - add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); - if (opt_cache_outputs) - state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); - break; - } - } - else - { - LOG_PRINT_L0("Bad vout type in txid " << block_txid); - return 1; + found = true; + add_ancestry(state.ancestry, txid, get_ancestry(state.ancestry, block_txid)); + if (opt_cache_outputs) + state.output_cache.insert(std::make_pair(ancestor{amount, offset}, block_txid)); + break; } } } @@ -505,11 +574,6 @@ int main(int argc, char* argv[]) } } } - else - { - LOG_PRINT_L0("Bad vin type in txid " << txid); - return 1; - } } } const size_t ancestry_size = get_ancestry(state.ancestry, txid).size(); From 07c3f45e5585d461f8f7438fee2898ffbe659416 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 10 Jul 2018 19:56:05 +0100 Subject: [PATCH 04/24] blockchain_depth: get the average min depth of a set of txes /monero#4147 --- src/blockchain_utilities/CMakeLists.txt | 32 ++ src/blockchain_utilities/blockchain_depth.cpp | 347 ++++++++++++++++++ 2 files changed, 379 insertions(+) create mode 100644 src/blockchain_utilities/blockchain_depth.cpp diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index c4f8cacb6b..bcaa9a0dbe 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -102,6 +102,17 @@ monero_private_headers(blockchain_ancestry +set(blockchain_depth_sources + blockchain_depth.cpp + ) + +set(blockchain_depth_private_headers) + +monero_private_headers(blockchain_depth + ${blockchain_depth_private_headers}) + + + monero_add_executable(blockchain_import ${blockchain_import_sources} ${blockchain_import_private_headers} @@ -215,3 +226,24 @@ set_property(TARGET blockchain_ancestry OUTPUT_NAME "monero-blockchain-ancestry") install(TARGETS blockchain_ancestry DESTINATION bin) +monero_add_executable(blockchain_depth + ${blockchain_depth_sources} + ${blockchain_depth_private_headers}) + +target_link_libraries(blockchain_depth + PRIVATE + cryptonote_core + blockchain_db + version + epee + ${Boost_FILESYSTEM_LIBRARY} + ${Boost_SYSTEM_LIBRARY} + ${Boost_THREAD_LIBRARY} + ${CMAKE_THREAD_LIBS_INIT} + ${EXTRA_LIBRARIES}) + +set_property(TARGET blockchain_depth + PROPERTY + OUTPUT_NAME "monero-blockchain-depth") +install(TARGETS blockchain_depth DESTINATION bin) + diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp new file mode 100644 index 0000000000..89fe6d03f1 --- /dev/null +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -0,0 +1,347 @@ +// Copyright (c) 2014-2018, The Monero Project +// +// All rights reserved. +// +// Redistribution and use in source and binary forms, with or without modification, are +// permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, this list of +// conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright notice, this list +// of conditions and the following disclaimer in the documentation and/or other +// materials provided with the distribution. +// +// 3. Neither the name of the copyright holder nor the names of its contributors may be +// used to endorse or promote products derived from this software without specific +// prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY +// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL +// THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, +// STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF +// THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +#include +#include +#include "common/command_line.h" +#include "common/varint.h" +#include "cryptonote_core/tx_pool.h" +#include "cryptonote_core/cryptonote_core.h" +#include "cryptonote_core/blockchain.h" +#include "blockchain_db/blockchain_db.h" +#include "blockchain_db/db_types.h" +#include "version.h" + +#undef MONERO_DEFAULT_LOG_CATEGORY +#define MONERO_DEFAULT_LOG_CATEGORY "bcutil" + +namespace po = boost::program_options; +using namespace epee; +using namespace cryptonote; + +int main(int argc, char* argv[]) +{ + TRY_ENTRY(); + + epee::string_tools::set_module_name_and_folder(argv[0]); + + std::string default_db_type = "lmdb"; + + std::string available_dbs = cryptonote::blockchain_db_types(", "); + available_dbs = "available: " + available_dbs; + + uint32_t log_level = 0; + + tools::on_startup(); + + boost::filesystem::path output_file_path; + + po::options_description desc_cmd_only("Command line options"); + po::options_description desc_cmd_sett("Command line options and settings options"); + const command_line::arg_descriptor arg_log_level = {"log-level", "0-4 or categories", ""}; + const command_line::arg_descriptor arg_database = { + "database", available_dbs.c_str(), default_db_type + }; + const command_line::arg_descriptor arg_txid = {"txid", "Get min depth for this txid", ""}; + const command_line::arg_descriptor arg_height = {"height", "Get min depth for all txes at this height", 0}; + const command_line::arg_descriptor arg_include_coinbase = {"include-coinbase", "Include coinbase in the average", false}; + + command_line::add_arg(desc_cmd_sett, cryptonote::arg_data_dir); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_testnet_on); + command_line::add_arg(desc_cmd_sett, cryptonote::arg_stagenet_on); + command_line::add_arg(desc_cmd_sett, arg_log_level); + command_line::add_arg(desc_cmd_sett, arg_database); + command_line::add_arg(desc_cmd_sett, arg_txid); + command_line::add_arg(desc_cmd_sett, arg_height); + command_line::add_arg(desc_cmd_sett, arg_include_coinbase); + command_line::add_arg(desc_cmd_only, command_line::arg_help); + + po::options_description desc_options("Allowed options"); + desc_options.add(desc_cmd_only).add(desc_cmd_sett); + + po::variables_map vm; + bool r = command_line::handle_error_helper(desc_options, [&]() + { + auto parser = po::command_line_parser(argc, argv).options(desc_options); + po::store(parser.run(), vm); + po::notify(vm); + return true; + }); + if (! r) + return 1; + + if (command_line::get_arg(vm, command_line::arg_help)) + { + std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << desc_options << std::endl; + return 1; + } + + mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + if (!command_line::is_arg_defaulted(vm, arg_log_level)) + mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); + else + mlog_set_log(std::string(std::to_string(log_level) + ",bcutil:INFO").c_str()); + + LOG_PRINT_L0("Starting..."); + + std::string opt_data_dir = command_line::get_arg(vm, cryptonote::arg_data_dir); + bool opt_testnet = command_line::get_arg(vm, cryptonote::arg_testnet_on); + bool opt_stagenet = command_line::get_arg(vm, cryptonote::arg_stagenet_on); + network_type net_type = opt_testnet ? TESTNET : opt_stagenet ? STAGENET : MAINNET; + std::string opt_txid_string = command_line::get_arg(vm, arg_txid); + uint64_t opt_height = command_line::get_arg(vm, arg_height); + bool opt_include_coinbase = command_line::get_arg(vm, arg_include_coinbase); + + if (!opt_txid_string.empty() && opt_height) + { + std::cerr << "txid and height cannot be given at the same time" << std::endl; + return 1; + } + crypto::hash opt_txid = crypto::null_hash; + if (!opt_txid_string.empty()) + { + if (!epee::string_tools::hex_to_pod(opt_txid_string, opt_txid)) + { + std::cerr << "Invalid txid" << std::endl; + return 1; + } + } + + std::string db_type = command_line::get_arg(vm, arg_database); + if (!cryptonote::blockchain_valid_db_type(db_type)) + { + std::cerr << "Invalid database type: " << db_type << std::endl; + return 1; + } + + // If we wanted to use the memory pool, we would set up a fake_core. + + // Use Blockchain instead of lower-level BlockchainDB for two reasons: + // 1. Blockchain has the init() method for easy setup + // 2. exporter needs to use get_current_blockchain_height(), get_block_id_by_height(), get_block_by_hash() + // + // cannot match blockchain_storage setup above with just one line, + // e.g. + // Blockchain* core_storage = new Blockchain(NULL); + // because unlike blockchain_storage constructor, which takes a pointer to + // tx_memory_pool, Blockchain's constructor takes tx_memory_pool object. + LOG_PRINT_L0("Initializing source blockchain (BlockchainDB)"); + std::unique_ptr core_storage; + tx_memory_pool m_mempool(*core_storage); + core_storage.reset(new Blockchain(m_mempool)); + BlockchainDB *db = new_db(db_type); + if (db == NULL) + { + LOG_ERROR("Attempted to use non-existent database type: " << db_type); + throw std::runtime_error("Attempting to use non-existent database type"); + } + LOG_PRINT_L0("database: " << db_type); + + const std::string filename = (boost::filesystem::path(opt_data_dir) / db->get_db_name()).string(); + LOG_PRINT_L0("Loading blockchain from folder " << filename << " ..."); + + try + { + db->open(filename, DBF_RDONLY); + } + catch (const std::exception& e) + { + LOG_PRINT_L0("Error opening database: " << e.what()); + return 1; + } + r = core_storage->init(db, net_type); + + CHECK_AND_ASSERT_MES(r, 1, "Failed to initialize source blockchain storage"); + LOG_PRINT_L0("Source blockchain storage initialized OK"); + + std::vector start_txids; + if (!opt_txid_string.empty()) + { + start_txids.push_back(opt_txid); + } + else + { + const crypto::hash block_hash = db->get_block_hash_from_height(opt_height); + const cryptonote::blobdata bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + for (const crypto::hash &txid: b.tx_hashes) + start_txids.push_back(txid); + if (opt_include_coinbase) + start_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + } + + if (start_txids.empty()) + { + LOG_PRINT_L0("No transaction(s) to check"); + return 1; + } + + double cumulative_depth = 0.0; + for (const crypto::hash &start_txid: start_txids) + { + uint64_t depth = 0; + bool coinbase = false; + + LOG_PRINT_L0("Checking depth for txid " << start_txid); + std::vector txids(1, start_txid); + while (!coinbase) + { + LOG_PRINT_L0("Considering "<< txids.size() << " transaction(s) at depth " << depth); + std::vector new_txids; + for (const crypto::hash &txid: txids) + { + cryptonote::blobdata bd; + if (!db->get_pruned_tx_blob(txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << txid << " from db"); + return 1; + } + cryptonote::transaction tx; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx)) + { + LOG_PRINT_L0("Bad tx: " << txid); + return 1; + } + for (size_t ring = 0; ring < tx.vin.size(); ++ring) + { + if (tx.vin[ring].type() == typeid(cryptonote::txin_gen)) + { + MDEBUG(txid << " is a coinbase transaction"); + coinbase = true; + goto done; + } + if (tx.vin[ring].type() == typeid(cryptonote::txin_to_key)) + { + const cryptonote::txin_to_key &txin = boost::get(tx.vin[ring]); + const uint64_t amount = txin.amount; + auto absolute_offsets = cryptonote::relative_output_offsets_to_absolute(txin.key_offsets); + for (uint64_t offset: absolute_offsets) + { + const output_data_t od = db->get_output_key(amount, offset); + const crypto::hash block_hash = db->get_block_hash_from_height(od.height); + bd = db->get_block_blob(block_hash); + cryptonote::block b; + if (!cryptonote::parse_and_validate_block_from_blob(bd, b)) + { + LOG_PRINT_L0("Bad block from db"); + return 1; + } + // find the tx which created this output + bool found = false; + for (size_t out = 0; out < b.miner_tx.vout.size(); ++out) + { + if (b.miner_tx.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(b.miner_tx.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(cryptonote::get_transaction_hash(b.miner_tx)); + MDEBUG("adding txid: " << cryptonote::get_transaction_hash(b.miner_tx)); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << cryptonote::get_transaction_hash(b.miner_tx)); + return 1; + } + } + for (const crypto::hash &block_txid: b.tx_hashes) + { + if (found) + break; + if (!db->get_pruned_tx_blob(block_txid, bd)) + { + LOG_PRINT_L0("Failed to get txid " << block_txid << " from db"); + return 1; + } + cryptonote::transaction tx2; + if (!cryptonote::parse_and_validate_tx_base_from_blob(bd, tx2)) + { + LOG_PRINT_L0("Bad tx: " << block_txid); + return 1; + } + for (size_t out = 0; out < tx2.vout.size(); ++out) + { + if (tx2.vout[out].target.type() == typeid(cryptonote::txout_to_key)) + { + const auto &txout = boost::get(tx2.vout[out].target); + if (txout.key == od.pubkey) + { + found = true; + new_txids.push_back(block_txid); + MDEBUG("adding txid: " << block_txid); + break; + } + } + else + { + LOG_PRINT_L0("Bad vout type in txid " << block_txid); + return 1; + } + } + } + if (!found) + { + LOG_PRINT_L0("Output originating transaction not found"); + return 1; + } + } + } + else + { + LOG_PRINT_L0("Bad vin type in txid " << txid); + return 1; + } + } + } + if (!coinbase) + { + std::swap(txids, new_txids); + ++depth; + } + } +done: + LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); + cumulative_depth += depth; + } + + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/start_txids.size()); + + core_storage->deinit(); + return 0; + + CATCH_ENTRY("Depth query error", 1); +} From d2d3b0fc92524b86483bb3e858e28bf70d8f9968 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 7 Aug 2018 20:03:48 +0000 Subject: [PATCH 05/24] blockchain_depth: add average min depth /monero#4147 --- src/blockchain_utilities/blockchain_depth.cpp | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index 89fe6d03f1..dd2387e5b9 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -207,7 +207,7 @@ int main(int argc, char* argv[]) return 1; } - double cumulative_depth = 0.0; + std::vector depths; for (const crypto::hash &start_txid: start_txids) { uint64_t depth = 0; @@ -335,10 +335,14 @@ int main(int argc, char* argv[]) } done: LOG_PRINT_L0("Min depth for txid " << start_txid << ": " << depth); - cumulative_depth += depth; + depths.push_back(depth); } - LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/start_txids.size()); + uint64_t cumulative_depth = 0; + for (uint64_t depth: depths) + cumulative_depth += depth; + LOG_PRINT_L0("Average min depth for " << start_txids.size() << " transaction(s): " << cumulative_depth/(float)depths.size()); + LOG_PRINT_L0("Median min depth for " << start_txids.size() << " transaction(s): " << epee::misc_utils::median(depths)); core_storage->deinit(); return 0; From dd9a4ccf4e93de134907773b7590403a8906a7aa Mon Sep 17 00:00:00 2001 From: stoffu Date: Wed, 27 May 2020 18:32:02 +0900 Subject: [PATCH 06/24] replace Monero string with Aeon --- src/blockchain_utilities/CMakeLists.txt | 4 ++-- src/blockchain_utilities/blockchain_ancestry.cpp | 4 ++-- src/blockchain_utilities/blockchain_depth.cpp | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/blockchain_utilities/CMakeLists.txt b/src/blockchain_utilities/CMakeLists.txt index bcaa9a0dbe..8b22a6b962 100644 --- a/src/blockchain_utilities/CMakeLists.txt +++ b/src/blockchain_utilities/CMakeLists.txt @@ -223,7 +223,7 @@ target_link_libraries(blockchain_ancestry set_property(TARGET blockchain_ancestry PROPERTY - OUTPUT_NAME "monero-blockchain-ancestry") + OUTPUT_NAME "aeon-blockchain-ancestry") install(TARGETS blockchain_ancestry DESTINATION bin) monero_add_executable(blockchain_depth @@ -244,6 +244,6 @@ target_link_libraries(blockchain_depth set_property(TARGET blockchain_depth PROPERTY - OUTPUT_NAME "monero-blockchain-depth") + OUTPUT_NAME "aeon-blockchain-depth") install(TARGETS blockchain_depth DESTINATION bin) diff --git a/src/blockchain_utilities/blockchain_ancestry.cpp b/src/blockchain_utilities/blockchain_ancestry.cpp index 2f0bbffd64..ce92c734e0 100644 --- a/src/blockchain_utilities/blockchain_ancestry.cpp +++ b/src/blockchain_utilities/blockchain_ancestry.cpp @@ -282,12 +282,12 @@ int main(int argc, char* argv[]) if (command_line::get_arg(vm, command_line::arg_help)) { - std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << "Aeon '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; std::cout << desc_options << std::endl; return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-ancestry.log"), true); + mlog_configure(mlog_get_default_log_path("aeon-blockchain-ancestry.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else diff --git a/src/blockchain_utilities/blockchain_depth.cpp b/src/blockchain_utilities/blockchain_depth.cpp index dd2387e5b9..b139463a12 100644 --- a/src/blockchain_utilities/blockchain_depth.cpp +++ b/src/blockchain_utilities/blockchain_depth.cpp @@ -97,12 +97,12 @@ int main(int argc, char* argv[]) if (command_line::get_arg(vm, command_line::arg_help)) { - std::cout << "Monero '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; + std::cout << "Aeon '" << MONERO_RELEASE_NAME << "' (v" << MONERO_VERSION_FULL << ")" << ENDL << ENDL; std::cout << desc_options << std::endl; return 1; } - mlog_configure(mlog_get_default_log_path("monero-blockchain-depth.log"), true); + mlog_configure(mlog_get_default_log_path("aeon-blockchain-depth.log"), true); if (!command_line::is_arg_defaulted(vm, arg_log_level)) mlog_set_log(command_line::get_arg(vm, arg_log_level).c_str()); else From 92876f0b919169f44e3a9b510350f7f13f4e5179 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 19 Jul 2018 11:31:23 +0100 Subject: [PATCH 07/24] cn_deserialize: support pruned transactions /monero#4154 --- src/debug_utilities/cn_deserialize.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 7926eb1051..97cbc7dad9 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -168,9 +168,12 @@ int main(int argc, char* argv[]) std::cout << "Parsed block:" << std::endl; std::cout << cryptonote::obj_to_json_str(block) << std::endl; } - else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx)) + else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { - std::cout << "Parsed transaction:" << std::endl; + if (tx.pruned) + std::cout << "Parsed pruned transaction:" << std::endl; + else + std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; bool parsed = cryptonote::parse_tx_extra(tx.extra, fields); From 6a2c6d11ac1990c2695aaa4837b0990991a9e015 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Thu, 19 Jul 2018 11:31:40 +0100 Subject: [PATCH 08/24] cn_deserialize: extract payment ids from extra nonce /monero#4154 --- src/debug_utilities/cn_deserialize.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index 97cbc7dad9..dfbe361adc 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -43,6 +43,15 @@ using namespace epee; using namespace cryptonote; +static std::string extra_nonce_to_string(const cryptonote::tx_extra_nonce &extra_nonce) +{ + if (extra_nonce.nonce.size() == 9 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_ENCRYPTED_PAYMENT_ID) + return "encrypted payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + if (extra_nonce.nonce.size() == 33 && extra_nonce.nonce[0] == TX_EXTRA_NONCE_PAYMENT_ID) + return "plaintext payment ID: " + epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce.substr(1)); + return epee::string_tools::buff_to_hex_nodelimer(extra_nonce.nonce); +} + static void print_extra_fields(const std::vector &fields) { std::cout << "tx_extra has " << fields.size() << " field(s)" << std::endl; @@ -51,7 +60,7 @@ static void print_extra_fields(const std::vector &fi std::cout << "field " << n << ": "; if (typeid(cryptonote::tx_extra_padding) == fields[n].type()) std::cout << "extra padding: " << boost::get(fields[n]).size << " bytes"; else if (typeid(cryptonote::tx_extra_pub_key) == fields[n].type()) std::cout << "extra pub key: " << boost::get(fields[n]).pub_key; - else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << epee::string_tools::buff_to_hex_nodelimer(boost::get(fields[n]).nonce); + else if (typeid(cryptonote::tx_extra_nonce) == fields[n].type()) std::cout << "extra nonce: " << extra_nonce_to_string(boost::get(fields[n])); else if (typeid(cryptonote::tx_extra_merge_mining_tag) == fields[n].type()) std::cout << "extra merge mining tag: depth " << boost::get(fields[n]).depth << ", merkle root " << boost::get(fields[n]).merkle_root; else if (typeid(cryptonote::tx_extra_additional_pub_keys) == fields[n].type()) std::cout << "additional tx pubkeys: " << boost::join(boost::get(fields[n]).data | boost::adaptors::transformed([](const crypto::public_key &key){ return epee::string_tools::pod_to_hex(key); }), ", " ); else if (typeid(cryptonote::tx_extra_mysterious_minergate) == fields[n].type()) std::cout << "extra minergate custom: " << epee::string_tools::buff_to_hex_nodelimer(boost::get(fields[n]).data); From dca6fea10cc1d0b364fc6918c725858e961de6ba Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Wed, 1 Aug 2018 16:01:36 +0000 Subject: [PATCH 09/24] simplewallet: handle transfers using an aeon: URI /monero#4208 --- src/simplewallet/simplewallet.cpp | 108 +++++++++++++++++++++--------- 1 file changed, 78 insertions(+), 30 deletions(-) diff --git a/src/simplewallet/simplewallet.cpp b/src/simplewallet/simplewallet.cpp index 95bf9a8f10..06a770b721 100644 --- a/src/simplewallet/simplewallet.cpp +++ b/src/simplewallet/simplewallet.cpp @@ -2335,12 +2335,12 @@ simple_wallet::simple_wallet() boost::bind(&simple_wallet::show_blockchain_height, this, _1), tr("Show the blockchain height.")); m_cmd_binder.set_handler("transfer", boost::bind(&simple_wallet::transfer, this, _1), - tr("transfer [index=[,,...]] [] []
[]"), - tr("Transfer to
. If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. is the number of inputs to include for untraceability. Multiple payments can be made at once by adding etcetera (before the payment ID, if it's included)")); + tr("transfer [index=[,,...]] [] [] ( |
) []"), + tr("Transfer to
. If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_transfer", boost::bind(&simple_wallet::locked_transfer, this, _1), - tr("locked_transfer [index=[,,...]] [] [] []"), - tr("Transfer to
and lock it for (max. 1000000). If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. is the number of inputs to include for untraceability. Multiple payments can be made at once by adding etcetera (before the payment ID, if it's included)")); + tr("locked_transfer [index=[,,...]] [] [] ( | ) []"), + tr("Transfer to
and lock it for (max. 1000000). If the parameter \"index=[,,...]\" is specified, the wallet uses outputs received by addresses of those indices. is the priority of the transaction. The higher the priority, the higher the transaction fee. Valid values in priority order (from lowest to highest) are: unimportant, normal, elevated, priority. If omitted, the default value (see the command \"set priority\") is used. is the number of inputs to include for untraceability. Multiple payments can be made at once by adding URI_2 or etcetera (before the payment ID, if it's included)")); m_cmd_binder.set_handler("locked_sweep_all", boost::bind(&simple_wallet::locked_sweep_all, this, _1), tr("locked_sweep_all [index=[,,...]] [] []
[]"), @@ -4800,7 +4800,7 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vectoradjust_priority(priority); - const size_t min_args = (transfer_type == TransferLocked) ? 3 : 2; + const size_t min_args = (transfer_type == TransferLocked) ? 2 : 1; if(local_args.size() < min_args) { fail_msg_writer() << tr("wrong number of arguments"); @@ -4809,39 +4809,38 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector extra; bool payment_id_seen = false; - bool expect_even = (transfer_type == TransferLocked); - if ((expect_even ? 0 : 1) == local_args.size() % 2) + if (!local_args.empty()) { std::string payment_id_str = local_args.back(); - local_args.pop_back(); - crypto::hash payment_id; - bool r = tools::wallet2::parse_long_payment_id(payment_id_str, payment_id); - if(r) + bool r = true; + if (tools::wallet2::parse_long_payment_id(payment_id_str, payment_id)) { std::string extra_nonce; set_payment_id_to_tx_extra_nonce(extra_nonce, payment_id); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; + message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } else { crypto::hash8 payment_id8; - r = tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8); - if(r) + if (tools::wallet2::parse_short_payment_id(payment_id_str, payment_id8)) { std::string extra_nonce; set_encrypted_payment_id_to_tx_extra_nonce(extra_nonce, payment_id8); r = add_extra_nonce_to_tx_extra(extra, extra_nonce); + local_args.pop_back(); + payment_id_seen = true; } } if(!r) { - fail_msg_writer() << tr("payment id has invalid format, expected 16 or 64 character hex string: ") << payment_id_str; + fail_msg_writer() << tr("payment id failed to encode"); return true; } - payment_id_seen = true; - message_writer() << tr("Unencrypted payment IDs are bad for privacy: ask the recipient to use subaddresses instead"); } uint64_t locked_blocks = 0; @@ -4866,11 +4865,54 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector dsts; size_t num_subaddresses = 0; - for (size_t i = 0; i < local_args.size(); i += 2) + for (size_t i = 0; i < local_args.size(); ) { - cryptonote::address_parse_info info; cryptonote::tx_destination_entry de; - if (!cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter)) + cryptonote::address_parse_info info; + bool r = true; + + // check for a URI + std::string address_uri, payment_id_uri, tx_description, recipient_name, error; + std::vector unknown_parameters; + uint64_t amount = 0; + bool has_uri = m_wallet->parse_uri(local_args[i], address_uri, payment_id_uri, amount, tx_description, recipient_name, unknown_parameters, error); + if (has_uri) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), address_uri, oa_prompter); + if (payment_id_uri.size() == 16) + { + if (!tools::wallet2::parse_short_payment_id(payment_id_uri, info.payment_id)) + { + fail_msg_writer() << tr("failed to parse short payment ID from URI"); + return true; + } + info.has_payment_id = true; + } + de.amount = amount; + ++i; + } + else if (i + 1 < local_args.size()) + { + r = cryptonote::get_account_address_from_str_or_url(info, m_wallet->nettype(), local_args[i], oa_prompter); + bool ok = cryptonote::parse_amount(de.amount, local_args[i + 1]); + if(!ok || 0 == de.amount) + { + fail_msg_writer() << tr("amount is wrong: ") << local_args[i] << ' ' << local_args[i + 1] << + ", " << tr("expected number from 0 to ") << print_money(std::numeric_limits::max()); + return true; + } + i += 2; + } + else + { + if (boost::starts_with(local_args[i], "aeon:")) + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back() << ": " << error; + else + fail_msg_writer() << tr("Invalid last argument: ") << local_args.back(); + return true; + } + + if (!r) { fail_msg_writer() << tr("failed to parse address"); return true; @@ -4879,16 +4921,30 @@ bool simple_wallet::transfer_main(int transfer_type, const std::vector::max()); - return true; - } - dsts.push_back(de); } From 9b8ed9b8ee4e23ee14ec27c5ea99016a9a0fcb2a Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 4 Aug 2018 12:37:39 +0000 Subject: [PATCH 10/24] wallet2: fix checking the wrong vector when adding hashes /monero#4221 The two vectors should be the same size anyway, so add an assert to catch any case where they aren't --- src/wallet/wallet2.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 15766f00b1..07cae26a97 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2127,9 +2127,11 @@ void wallet2::pull_and_parse_next_blocks(uint64_t start_height, uint64_t &blocks { drop_from_short_history(short_chain_history, 3); + THROW_WALLET_EXCEPTION_IF(prev_blocks.size() != prev_parsed_blocks.size(), error::wallet_internal_error, "size mismatch"); + // prepend the last 3 blocks, should be enough to guard against a block or two's reorg std::vector::const_reverse_iterator i = prev_parsed_blocks.rbegin(); - for (size_t n = 0; n < std::min((size_t)3, prev_blocks.size()); ++n) + for (size_t n = 0; n < std::min((size_t)3, prev_parsed_blocks.size()); ++n) { short_chain_history.push_front(i->hash); ++i; From 4d8ce3676c2fa1d6399b329d7b4cad3fa7449dfe Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 4 Aug 2018 12:38:55 +0000 Subject: [PATCH 11/24] wallet2: trim hash chain after fast refresh of hashes /monero#4221 This ensures it can't end up filled with the actual placeholders --- src/wallet/wallet2.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 07cae26a97..0ce1fe688d 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2460,6 +2460,7 @@ void wallet2::fast_refresh(uint64_t stop_height, uint64_t &blocks_start_height, while (missing_blocks-- > 0) m_blockchain.push_back(crypto::null_hash); // maybe a bit suboptimal, but deque won't do huge reallocs like vector m_blockchain.push_back(m_checkpoints.get_points().at(checkpoint_height)); + m_blockchain.trim(checkpoint_height); short_chain_history.clear(); get_short_chain_history(short_chain_history); } From bbce88dba09049f0d97471c873a9b31caa82025c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 4 Aug 2018 12:41:15 +0000 Subject: [PATCH 12/24] wallet2: fix refresh retry when a block/tx fails to parse /monero#4221 It would switch to a new set of blocks and fail, getting out of sync with the hash chain in the process --- src/wallet/wallet2.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 0ce1fe688d..03a4b34994 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -2680,10 +2680,6 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo break; } - // switch to the new blocks from the daemon - blocks_start_height = next_blocks_start_height; - blocks = std::move(next_blocks); - parsed_blocks = std::move(next_parsed_blocks); first = false; // handle error from async fetching thread @@ -2691,6 +2687,11 @@ void wallet2::refresh(bool trusted_daemon, uint64_t start_height, uint64_t & blo { throw std::runtime_error("proxy exception in refresh thread"); } + + // switch to the new blocks from the daemon + blocks_start_height = next_blocks_start_height; + blocks = std::move(next_blocks); + parsed_blocks = std::move(next_parsed_blocks); } catch (const tools::error::password_needed&) { From 66e16f78b18f971afd047cfec4000e09631e865d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 7 Aug 2018 13:05:52 +0000 Subject: [PATCH 13/24] CMakeLists.txt: add -mmitigate-rop to security flags if found /monero#4232 This is GCC/x86/x86_64 only --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 79fe57d80c..1bf90dce78 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -655,6 +655,9 @@ else() add_cxx_flag_if_supported(-mmitigate-rop CXX_SECURITY_FLAGS) endif() + add_c_flag_if_supported(-mmitigate-rop C_SECURITY_FLAGS) + add_cxx_flag_if_supported(-mmitigate-rop CXX_SECURITY_FLAGS) + # linker if (NOT (WIN32 AND (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_LESS 9.1))) # Windows binaries die on startup with PIE when compiled with GCC <9.x From 829a030710353263675afeebe223a640776ad7d1 Mon Sep 17 00:00:00 2001 From: cslashm Date: Mon, 30 Jul 2018 10:16:13 +0200 Subject: [PATCH 14/24] Add bulletproof support /monero#4251 Aeon note: this is of course irrelevant in practice, since Ledger doesn't support Aeon and also we don't use RingCT. --- src/device/device_ledger.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/device/device_ledger.cpp b/src/device/device_ledger.cpp index b2c2c3e77a..94ccac95b4 100644 --- a/src/device/device_ledger.cpp +++ b/src/device/device_ledger.cpp @@ -1366,7 +1366,7 @@ namespace hw { this->exchange(); //pseudoOuts - if ((type == rct::RCTTypeSimple) || (type == rct::RCTTypeBulletproof)) { + if (type == rct::RCTTypeSimple) { for ( i = 0; i < inputs_size; i++) { offset = set_command_header(INS_VALIDATE, 0x01, i+2); //options From 130a6f45594803dbadcfe6beadf4c447fbbe179b Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 13 Aug 2018 22:13:11 +0000 Subject: [PATCH 15/24] fuzz_tests: use __AFL_INIT when available /monero#4254 alleged to speed things up --- tests/fuzz/fuzzer.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/fuzz/fuzzer.cpp b/tests/fuzz/fuzzer.cpp index 7c140236ec..44c0a7a558 100644 --- a/tests/fuzz/fuzzer.cpp +++ b/tests/fuzz/fuzzer.cpp @@ -52,6 +52,10 @@ int run_fuzzer(int argc, const char **argv, Fuzzer &fuzzer) return 1; } +#ifdef __AFL_HAVE_MANUAL_CONTROL + __AFL_INIT(); +#endif + int ret = fuzzer.init(); if (ret) return ret; From dd65fc61c048dcd6c2efa569b2cc56d42e3806b4 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 27 Aug 2018 16:29:29 +0000 Subject: [PATCH 16/24] epee: use the socket::bind variant which does not throw /monero#4307 When this throws in a loop, stack trace generation can take a significant amount of CPU --- .../epee/include/net/abstract_tcp_server2.inl | 20 +++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 7198c4943b..2892935137 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1116,7 +1116,15 @@ POP_WARNINGS if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); + boost::system::error_code ec; + sock_.bind(local_endpoint, ec); + if (ec) + { + MERROR("Error binding to " << adr << ": " << ec.message()); + if (sock_.is_open()) + sock_.close(); + return false; + } } /* @@ -1222,7 +1230,15 @@ POP_WARNINGS if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); - sock_.bind(local_endpoint); + boost::system::error_code ec; + sock_.bind(local_endpoint, ec); + if (ec) + { + MERROR("Error binding to " << adr << ": " << ec.message()); + if (sock_.is_open()) + sock_.close(); + return false; + } } boost::shared_ptr sh_deadline(new boost::asio::deadline_timer(io_service_)); From a0468e3a03f7677ebb388e4bc8bddea0fdb120b9 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sun, 9 Sep 2018 10:46:58 +0000 Subject: [PATCH 17/24] abstract_tcp_server2: fix binding to the wrong IP /monero#4307 --- contrib/epee/include/net/abstract_tcp_server2.inl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/contrib/epee/include/net/abstract_tcp_server2.inl b/contrib/epee/include/net/abstract_tcp_server2.inl index 2892935137..277df1977b 100644 --- a/contrib/epee/include/net/abstract_tcp_server2.inl +++ b/contrib/epee/include/net/abstract_tcp_server2.inl @@ -1115,12 +1115,12 @@ POP_WARNINGS sock_.open(remote_endpoint.protocol()); if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0); boost::system::error_code ec; sock_.bind(local_endpoint, ec); if (ec) { - MERROR("Error binding to " << adr << ": " << ec.message()); + MERROR("Error binding to " << bind_ip << ": " << ec.message()); if (sock_.is_open()) sock_.close(); return false; @@ -1229,12 +1229,12 @@ POP_WARNINGS sock_.open(remote_endpoint.protocol()); if(bind_ip != "0.0.0.0" && bind_ip != "0" && bind_ip != "" ) { - boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(adr.c_str()), 0); + boost::asio::ip::tcp::endpoint local_endpoint(boost::asio::ip::address::from_string(bind_ip.c_str()), 0); boost::system::error_code ec; sock_.bind(local_endpoint, ec); if (ec) { - MERROR("Error binding to " << adr << ": " << ec.message()); + MERROR("Error binding to " << bind_ip << ": " << ec.message()); if (sock_.is_open()) sock_.close(); return false; From 244dd09cf543ae499a525dfa8c1cd3886de74e75 Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 14 Sep 2018 10:55:50 +0000 Subject: [PATCH 18/24] cn_deserialize: comment out pruned tx case /monero#4380 it's not merged yet --- src/debug_utilities/cn_deserialize.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/debug_utilities/cn_deserialize.cpp b/src/debug_utilities/cn_deserialize.cpp index dfbe361adc..6de6db4a8a 100644 --- a/src/debug_utilities/cn_deserialize.cpp +++ b/src/debug_utilities/cn_deserialize.cpp @@ -179,9 +179,11 @@ int main(int argc, char* argv[]) } else if (cryptonote::parse_and_validate_tx_from_blob(blob, tx) || cryptonote::parse_and_validate_tx_base_from_blob(blob, tx)) { +/* if (tx.pruned) std::cout << "Parsed pruned transaction:" << std::endl; else +*/ std::cout << "Parsed transaction:" << std::endl; std::cout << cryptonote::obj_to_json_str(tx) << std::endl; From c1d26ce1f9d5b3b829eede1cac1bc25dbedc1f61 Mon Sep 17 00:00:00 2001 From: el00ruobuob Date: Sun, 2 Sep 2018 23:50:31 +0200 Subject: [PATCH 19/24] remove unused fields from relay_tx RPC /monero#4329 --- src/wallet/wallet_rpc_server_commands_defs.h | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/wallet/wallet_rpc_server_commands_defs.h b/src/wallet/wallet_rpc_server_commands_defs.h index 116638b112..7b0a445b90 100644 --- a/src/wallet/wallet_rpc_server_commands_defs.h +++ b/src/wallet/wallet_rpc_server_commands_defs.h @@ -770,15 +770,9 @@ namespace wallet_rpc struct response { std::string tx_hash; - std::string tx_key; - uint64_t fee; - std::string tx_blob; BEGIN_KV_SERIALIZE_MAP() KV_SERIALIZE(tx_hash) - KV_SERIALIZE(tx_key) - KV_SERIALIZE(fee) - KV_SERIALIZE(tx_blob) END_KV_SERIALIZE_MAP() }; }; From 6ae6f22bdf210be2876cd7e195e81111ea98fc3c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Fri, 24 Aug 2018 09:50:34 +0000 Subject: [PATCH 20/24] cryptonote_format_utils: do not early out on invalid tx pubkeys /monero#4330 Another such pubkey might be valid --- src/cryptonote_basic/cryptonote_format_utils.cpp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/cryptonote_basic/cryptonote_format_utils.cpp b/src/cryptonote_basic/cryptonote_format_utils.cpp index e62f679def..bfd34f0472 100644 --- a/src/cryptonote_basic/cryptonote_format_utils.cpp +++ b/src/cryptonote_basic/cryptonote_format_utils.cpp @@ -219,15 +219,25 @@ namespace cryptonote { crypto::key_derivation recv_derivation = AUTO_VAL_INIT(recv_derivation); bool r = hwdev.generate_key_derivation(tx_public_key, ack.m_view_secret_key, recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << tx_public_key << ", " << ack.m_view_secret_key << ")"); + memcpy(&recv_derivation, rct::identity().bytes, sizeof(recv_derivation)); + } std::vector additional_recv_derivations; for (size_t i = 0; i < additional_tx_public_keys.size(); ++i) { crypto::key_derivation additional_recv_derivation = AUTO_VAL_INIT(additional_recv_derivation); r = hwdev.generate_key_derivation(additional_tx_public_keys[i], ack.m_view_secret_key, additional_recv_derivation); - CHECK_AND_ASSERT_MES(r, false, "key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); - additional_recv_derivations.push_back(additional_recv_derivation); + if (!r) + { + MWARNING("key image helper: failed to generate_key_derivation(" << additional_tx_public_keys[i] << ", " << ack.m_view_secret_key << ")"); + } + else + { + additional_recv_derivations.push_back(additional_recv_derivation); + } } boost::optional subaddr_recv_info = is_out_to_acc_precomp(subaddresses, out_key, recv_derivation, additional_recv_derivations, real_output_index,hwdev); From 4a3acf98656de2e0ce8cfa3912da98f9905ebc7c Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Mon, 3 Sep 2018 11:22:38 +0000 Subject: [PATCH 21/24] wallet2: fill in v2 height for stagenet /monero#4331 Aeon note: the heights were completely identical to Monero's and thus wrong. They are corrected now. --- src/wallet/wallet2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 03a4b34994..f230886661 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -4856,7 +4856,7 @@ bool wallet2::is_tx_spendtime_unlocked(uint64_t unlock_time, uint64_t block_heig uint64_t current_time = static_cast(time(NULL)); // XXX: this needs to be fast, so we'd need to get the starting heights // from the daemon to be correct once voting kicks in - uint64_t v2height = m_nettype == TESTNET ? 624634 : m_nettype == STAGENET ? (uint64_t)-1/*TODO*/ : 1009827; + uint64_t v2height = m_nettype == TESTNET ? 44000 : m_nettype == STAGENET ? 90000 : 592000; uint64_t leeway = block_height < v2height ? CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V1 : CRYPTONOTE_LOCKED_TX_ALLOWED_DELTA_SECONDS_V2; if(current_time + leeway >= unlock_time) return true; From 95065e19111742cf3df153db0ab32d3ac132786f Mon Sep 17 00:00:00 2001 From: stoffu Date: Tue, 4 Sep 2018 13:00:38 +0900 Subject: [PATCH 22/24] wallet2.get_reserve_proof: throw when specified amount is zero /monero#4336 --- src/wallet/wallet2.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index f230886661..78c6a90065 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -9597,6 +9597,7 @@ std::string wallet2::get_reserve_proof(const boost::optionalsecond == 0, error::wallet_internal_error, "Proved amount must be greater than 0"); // minimize the number of outputs included in the proof, by only picking the N largest outputs that can cover the requested min reserve amount std::sort(selected_transfers.begin(), selected_transfers.end(), [&](const size_t a, const size_t b) { return m_transfers[a].amount() > m_transfers[b].amount(); }); From 779a43c9bc5817ef129b1cd7429612d109daa4fc Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Tue, 4 Sep 2018 09:18:46 +0000 Subject: [PATCH 23/24] wallet_rpc_server: error out if wallet-file and wallet-dir are both used /monero#4337 --- src/wallet/wallet_rpc_server.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/wallet/wallet_rpc_server.cpp b/src/wallet/wallet_rpc_server.cpp index 4cedcdf66e..a8afd08d4d 100644 --- a/src/wallet/wallet_rpc_server.cpp +++ b/src/wallet/wallet_rpc_server.cpp @@ -160,8 +160,13 @@ namespace tools std::string bind_port = command_line::get_arg(*m_vm, arg_rpc_bind_port); const bool disable_auth = command_line::get_arg(*m_vm, arg_disable_rpc_login); m_restricted = command_line::get_arg(*m_vm, arg_restricted); - if (command_line::has_arg(*m_vm, arg_wallet_dir)) + if (!command_line::is_arg_defaulted(*m_vm, arg_wallet_dir)) { + if (!command_line::is_arg_defaulted(*m_vm, wallet_args::arg_wallet_file())) + { + MERROR(arg_wallet_dir.name << " and " << wallet_args::arg_wallet_file().name << " are incompatible, use only one of them"); + return false; + } m_wallet_dir = command_line::get_arg(*m_vm, arg_wallet_dir); #ifdef _WIN32 #define MKDIR(path, mode) mkdir(path) From 3bac43ac3adc7880ebe072b06253503e5810c35d Mon Sep 17 00:00:00 2001 From: moneromooo-monero Date: Sat, 8 Sep 2018 23:11:16 +0000 Subject: [PATCH 24/24] wallet2: fix secondary partially signed multisig txes /monero#4347 Aeon note: This patch is noop since neither RingCT nor multisig are currently in use. --- src/wallet/wallet2.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/wallet/wallet2.cpp b/src/wallet/wallet2.cpp index 78c6a90065..0c00ce27da 100644 --- a/src/wallet/wallet2.cpp +++ b/src/wallet/wallet2.cpp @@ -7353,7 +7353,7 @@ void wallet2::transfer_selected_rct(std::vector