From 04edec7ed5b71f2aadd803ee5091d099f1604534 Mon Sep 17 00:00:00 2001 From: lupin012 <58134934+lupin012@users.noreply.github.com> Date: Thu, 7 Nov 2024 18:23:34 +0100 Subject: [PATCH] rpcdaemon: debug_getModifiedAccountsBy[Number|Hash] for E3 (#2468) --- .github/workflows/rpc-integration-tests.yml | 2 +- .github/workflows/run_integration_tests.sh | 2 - .../db/kv/api/endpoint/temporal_range.hpp | 14 +++++ silkworm/infra/grpc/client/call.hpp | 2 +- silkworm/rpc/commands/debug_api.cpp | 43 ++++++++----- silkworm/rpc/commands/debug_api_test.cpp | 63 +++++++++++++++---- 6 files changed, 95 insertions(+), 31 deletions(-) diff --git a/.github/workflows/rpc-integration-tests.yml b/.github/workflows/rpc-integration-tests.yml index d854be4532..06af902afa 100644 --- a/.github/workflows/rpc-integration-tests.yml +++ b/.github/workflows/rpc-integration-tests.yml @@ -33,7 +33,7 @@ jobs: - name: Checkout RPC Tests Repository & Install Requirements run: | rm -rf ${{runner.workspace}}/rpc-tests - git -c advice.detachedHead=false clone --depth 1 --branch v1.10.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests + git -c advice.detachedHead=false clone --depth 1 --branch v1.11.0 https://github.com/erigontech/rpc-tests ${{runner.workspace}}/rpc-tests cd ${{runner.workspace}}/rpc-tests pip3 install -r requirements.txt diff --git a/.github/workflows/run_integration_tests.sh b/.github/workflows/run_integration_tests.sh index 660923b6b8..6e84c75546 100755 --- a/.github/workflows/run_integration_tests.sh +++ b/.github/workflows/run_integration_tests.sh @@ -12,7 +12,6 @@ rm -rf ./mainnet/results/ # eth_getBlockReceipts/test_07.json new blobFields # debug_accountRange: new algo using TKV -# debug_getModifiedAccounts: new algo using TKV # debug_storageRangeAt: new algo using TKV # debug_traceCall/test_02.json: requested is_latest fix to support ethbackend # erigon_getBalanceChangesInBlock: new algo using TKV @@ -28,7 +27,6 @@ rm -rf ./mainnet/results/ python3 ./run_tests.py --continue --blockchain mainnet --jwt "$2" --display-only-fail --json-diff --port 51515 --transport_type http,websocket -x \ debug_accountRange,\ -debug_getModifiedAccounts,\ debug_storageRangeAt,\ debug_traceBlockByHash/test_05,\ debug_traceBlockByHash/test_08,\ diff --git a/silkworm/db/kv/api/endpoint/temporal_range.hpp b/silkworm/db/kv/api/endpoint/temporal_range.hpp index 0dd2c17739..d8ae159a6b 100644 --- a/silkworm/db/kv/api/endpoint/temporal_range.hpp +++ b/silkworm/db/kv/api/endpoint/temporal_range.hpp @@ -61,8 +61,22 @@ struct HistoryRangeQuery { int64_t limit{kUnlimited}; uint32_t page_size{0}; std::string page_token; + + // TODO(canepat) we need clang >= 17 to use spaceship operator instead of hand-made operator== below + // auto operator<=>(const HistoryRangeQuery&) const = default; }; +inline bool operator==(const HistoryRangeQuery& lhs, const HistoryRangeQuery& rhs) { + return (lhs.tx_id == rhs.tx_id) && + (lhs.table == rhs.table) && + (lhs.from_timestamp == rhs.from_timestamp) && + (lhs.to_timestamp == rhs.to_timestamp) && + (lhs.ascending_order == rhs.ascending_order) && + (lhs.limit == rhs.limit) && + (lhs.page_size == rhs.page_size) && + (lhs.page_token == rhs.page_token); +} + using HistoryRangeResult = RangeResult; struct DomainRangeQuery { diff --git a/silkworm/infra/grpc/client/call.hpp b/silkworm/infra/grpc/client/call.hpp index f1f8477d19..2933b826d5 100644 --- a/silkworm/infra/grpc/client/call.hpp +++ b/silkworm/infra/grpc/client/call.hpp @@ -57,7 +57,7 @@ Task unary_rpc( agrpc::GrpcContext& grpc_context, boost::asio::cancellation_slot* cancellation_slot = nullptr) { grpc::ClientContext client_context; - client_context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(10)); + client_context.set_deadline(std::chrono::system_clock::now() + std::chrono::seconds(60)); if (cancellation_slot) { cancellation_slot->assign([&client_context](boost::asio::cancellation_type /*type*/) { client_context.TryCancel(); diff --git a/silkworm/rpc/commands/debug_api.cpp b/silkworm/rpc/commands/debug_api.cpp index a247663b61..c25f22a17c 100644 --- a/silkworm/rpc/commands/debug_api.cpp +++ b/silkworm/rpc/commands/debug_api.cpp @@ -127,7 +127,13 @@ Task DebugRpcApi::handle_debug_get_modified_accounts_by_number(const nlohm const auto start_block_number = co_await core::get_block_number(start_block_id, *tx); const auto end_block_number = co_await core::get_block_number(end_block_id, *tx); - const auto addresses = co_await get_modified_accounts(*tx, start_block_number, end_block_number); + if (end_block_number < start_block_number) { + std::stringstream msg; + msg << "start block (" << start_block_number << ") must be less or equal to end block (" << end_block_number << ")"; + throw std::invalid_argument(msg.str()); + } + + const auto addresses = co_await get_modified_accounts(*tx, start_block_number, end_block_number + 1); reply = make_json_content(request, addresses); } catch (const std::invalid_argument& e) { SILK_ERROR << "exception: " << e.what() << " processing request: " << request.dump(); @@ -173,7 +179,7 @@ Task DebugRpcApi::handle_debug_get_modified_accounts_by_hash(const nlohman if (!end_block_number) { throw std::invalid_argument("end block " + silkworm::to_hex(end_hash) + " not found"); } - const auto addresses = co_await get_modified_accounts(*tx, *start_block_number, *end_block_number); + const auto addresses = co_await get_modified_accounts(*tx, *start_block_number, *end_block_number + 1); reply = make_json_content(request, addresses); } catch (const std::invalid_argument& e) { @@ -634,28 +640,33 @@ Task> get_modified_accounts(db::kv::api::Transaction& tx SILK_DEBUG << "latest: " << latest_block_number << " start: " << start_block_number << " end: " << end_block_number; - std::set addresses; if (start_block_number > latest_block_number) { std::stringstream msg; msg << "start block (" << start_block_number << ") is later than the latest block (" << latest_block_number << ")"; throw std::invalid_argument(msg.str()); } - if (start_block_number <= end_block_number) { - auto walker = [&](const silkworm::Bytes& key, const silkworm::Bytes& value) { - auto block_number = static_cast(std::stol(silkworm::to_hex(key), nullptr, 16)); - if (block_number <= end_block_number) { - auto address = bytes_to_address(value.substr(0, kAddressLength)); - SILK_TRACE << "Walker: processing block " << block_number << " address " << address; - addresses.insert(address); - } - return block_number <= end_block_number; - }; + if (end_block_number > latest_block_number) { + std::stringstream msg; + msg << "end block (" << end_block_number << ") is later than the latest block (" << latest_block_number << ")"; + throw std::invalid_argument(msg.str()); + } + + const auto start_txn_number = co_await tx.first_txn_num_in_block(start_block_number); + const auto end_txn_number = co_await tx.first_txn_num_in_block(end_block_number == start_block_number ? end_block_number + 1 : end_block_number) - 1; - const auto key = silkworm::db::block_key(start_block_number); - SILK_TRACE << "Ready to walk starting from key: " << silkworm::to_hex(key); + db::kv::api::HistoryRangeQuery query{ + .table = db::table::kAccountsHistory, + .from_timestamp = static_cast(start_txn_number), + .to_timestamp = static_cast(end_txn_number), + .ascending_order = true}; - co_await walk(tx, db::table::kAccountChangeSetName, key, 0, walker); + auto paginated_result = co_await tx.history_range(std::move(query)); + auto it = co_await paginated_result.begin(); + + std::set addresses; + while (const auto value = co_await it.next()) { + addresses.insert(bytes_to_address(value->first)); } co_return addresses; diff --git a/silkworm/rpc/commands/debug_api_test.cpp b/silkworm/rpc/commands/debug_api_test.cpp index 271bd5aeed..dcdd144d99 100644 --- a/silkworm/rpc/commands/debug_api_test.cpp +++ b/silkworm/rpc/commands/debug_api_test.cpp @@ -29,7 +29,10 @@ #include #include #include +#include +#include #include +#include #include #include #include @@ -46,6 +49,17 @@ using db::chain::ChainStorage; using db::kv::api::Cursor; using db::kv::api::CursorDupSort; using db::kv::api::KeyValue; +using testing::_; +using testing::Invoke; +using testing::InvokeWithoutArgs; +using testing::Unused; +using namespace evmc::literals; + +using PaginatedKV = db::kv::api::PaginatedSequencePair; +using PaginatorKV = PaginatedKV::Paginator; +using PageK = PaginatedKV::KPage; +using PageV = PaginatedKV::VPage; +using PageResultKV = PaginatedKV::PageResult; static const nlohmann::json kEmpty; static const std::string kZeros = "00000000000000000000000000000000000000000000000000000000000000000000000000000000"; @@ -366,27 +380,53 @@ TEST_CASE("get_modified_accounts") { {"000000000052a057", "f3a3956d084e3f2a24add02c35c8afd09e3e9bf5030105080c9eea7771667e25"}, // NOLINT {"000000000052a058", "053eafe07f12033715d31e1599bbf27dd1c05fb2030105080ddd58b6af8be86e"} // NOLINT }; - // std::cout << "json: " << json << "\n" << std::flush; - - auto database = DummyDatabase{json}; - auto begin_result = boost::asio::co_spawn(pool, database.begin(), boost::asio::use_future); - auto tx = begin_result.get(); + db::test_util::MockTransaction transaction; + auto& tx = transaction; SECTION("end == start") { - auto result = boost::asio::co_spawn(pool, get_modified_accounts(*tx, 0x52a010, 0x52a010), boost::asio::use_future); - auto accounts = result.get(); - - CHECK(accounts.size() == 1); - + EXPECT_CALL(transaction, get(db::table::kLastForkchoiceName, _)).WillOnce(InvokeWithoutArgs([]() -> Task { + co_return KeyValue{silkworm::Bytes{}, silkworm::Bytes{}}; + })); + EXPECT_CALL(transaction, get(db::table::kSyncStageProgressName, ByteView{stages::kExecution})) + .WillOnce(InvokeWithoutArgs([]() -> Task { + co_return KeyValue{silkworm::Bytes{}, *silkworm::from_hex("000000000052a010")}; + })); + EXPECT_CALL(transaction, first_txn_num_in_block(0x52a010)) + .WillOnce(InvokeWithoutArgs([]() -> Task { + co_return 0; + })); + EXPECT_CALL(transaction, first_txn_num_in_block(0x52a011)) + .WillOnce(InvokeWithoutArgs([]() -> Task { + co_return 20; + })); + db::kv::api::HistoryRangeQuery query{ + .table = db::table::kAccountsHistory, + .from_timestamp = static_cast(0), + .to_timestamp = static_cast(19), + .ascending_order = true}; + EXPECT_CALL(transaction, history_range(std::move(query))).WillOnce(Invoke([=](Unused) -> Task { + PaginatorKV paginator = [](auto next_page_token) -> Task { + co_return PageResultKV{ + .keys = {*from_hex("07aaec0b237ccf56b03a7c43c1c7a783da5606420501010101")}, + .values = {Bytes{}}, // encoded account value doesn't care + .next_page_token = std::move(next_page_token)}; + }; + db::kv::api::PaginatedKeysValues result{paginator}; + co_return result; + })); + + auto result = boost::asio::co_spawn(pool, get_modified_accounts(tx, 0x52a010, 0x52a010), boost::asio::use_future); + const auto accounts = result.get(); nlohmann::json j = accounts; CHECK(j == R"([ "0x07aaec0b237ccf56b03a7c43c1c7a783da560642" ])"_json); } - +#ifdef notdef SECTION("end == start + 1") { auto result = boost::asio::co_spawn(pool, get_modified_accounts(*tx, 0x52a010, 0x52a011), boost::asio::use_future); auto accounts = result.get(); + std::cout << "size2: " << accounts.size() << "\n"; CHECK(accounts.size() == 2); @@ -489,6 +529,7 @@ TEST_CASE("get_modified_accounts") { auto result = boost::asio::co_spawn(pool, get_modified_accounts(*tx, 0x52a061, 0x52a061), boost::asio::use_future); CHECK_THROWS_AS(result.get(), std::invalid_argument); } +#endif } #endif // !defined(__clang__)