Skip to content

Commit

Permalink
access_layer DataModel refactoring to use multi-bundle queries
Browse files Browse the repository at this point in the history
  • Loading branch information
@battlmonstr
battlmonstr committed Dec 31, 2024
1 parent 2e62831 commit 9515f59
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 65 deletions.
49 changes: 12 additions & 37 deletions silkworm/db/access_layer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1266,25 +1266,11 @@ bool DataModel::read_block_from_snapshot(BlockNum block_num, Block& block) const
}

std::optional<BlockHeader> DataModel::read_header_from_snapshot(BlockNum block_num) const {
std::optional<BlockHeader> block_header;
// We know the header snapshot in advance: find it based on target block number
const auto [segment_and_index, _] = repository_.find_segment(blocks::kHeaderSegmentAndIdxNames, block_num);
if (segment_and_index) {
block_header = HeaderFindByBlockNumSegmentQuery{*segment_and_index}.exec(block_num);
}
return block_header;
return HeaderFindByBlockNumQuery{repository_}.exec(block_num);
}

std::optional<BlockHeader> DataModel::read_header_from_snapshot(const Hash& hash) const {
std::optional<BlockHeader> block_header;
// We don't know the header snapshot in advance: search for block hash in each header snapshot in reverse order
for (const auto& bundle_ptr : repository_.view_bundles_reverse()) {
const auto& bundle = *bundle_ptr;
auto segment_and_index = bundle.segment_and_accessor_index(blocks::kHeaderSegmentAndIdxNames);
block_header = HeaderFindByHashSegmentQuery{segment_and_index}.exec(hash);
if (block_header) break;
}
return block_header;
return HeaderFindByHashQuery{repository_}.exec(hash);
}

std::optional<BlockBodyForStorage> DataModel::read_body_for_storage_from_snapshot(BlockNum block_num) const {
Expand All @@ -1310,46 +1296,35 @@ bool DataModel::read_body_from_snapshot(BlockNum block_num, BlockBody& body) con
}

bool DataModel::is_body_in_snapshot(BlockNum block_num) const {
// We know the body snapshot in advance: find it based on target block number
const auto [segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentAndIdxNames, block_num);
if (segment_and_index) {
const auto stored_body = BodyFindByBlockNumSegmentQuery{*segment_and_index}.exec(block_num);
return stored_body.has_value();
}

return false;
return BodyFindByBlockNumQuery{repository_}.exec(block_num).has_value();
}

bool DataModel::read_transactions_from_snapshot(BlockNum block_num, uint64_t base_txn_id, uint64_t txn_count, std::vector<Transaction>& txs) const {
if (txn_count == 0) {
return true;
}

const auto [segment_and_index, _] = repository_.find_segment(blocks::kTxnSegmentAndIdxNames, block_num);
if (!segment_and_index) return false;

txs = TransactionRangeFromIdSegmentQuery{*segment_and_index}.exec_into_vector(base_txn_id, txn_count);
auto txs_opt = TransactionRangeFromIdQuery{repository_}.exec(block_num, base_txn_id, txn_count);
if (!txs_opt) return false;

txs = std::move(*txs_opt);
return true;
}

bool DataModel::read_rlp_transactions_from_snapshot(BlockNum block_num, std::vector<Bytes>& rlp_txs) const {
const auto [body_segment_and_index, _] = repository_.find_segment(blocks::kBodySegmentAndIdxNames, block_num);
if (body_segment_and_index) {
auto stored_body = BodyFindByBlockNumSegmentQuery{*body_segment_and_index}.exec(block_num);
if (!stored_body) return false;
auto stored_body = BodyFindByBlockNumQuery{repository_}.exec(block_num);
if (!stored_body) return false;

{
// Skip first and last *system transactions* in block body
const auto base_txn_id{stored_body->base_txn_id + 1};
const auto txn_count{stored_body->txn_count >= 2 ? stored_body->txn_count - 2 : stored_body->txn_count};

if (txn_count == 0) return true;

const auto [tx_segment_and_index, _2] = repository_.find_segment(blocks::kTxnSegmentAndIdxNames, block_num);
if (!tx_segment_and_index) return false;

rlp_txs = TransactionPayloadRlpRangeFromIdSegmentQuery{*tx_segment_and_index}.exec_into_vector(base_txn_id, txn_count);
auto txs_opt = TransactionPayloadRlpRangeFromIdQuery{repository_}.exec(block_num, base_txn_id, txn_count);
if (!txs_opt) return false;

rlp_txs = std::move(*txs_opt);
return true;
}

Expand Down
14 changes: 12 additions & 2 deletions silkworm/db/blocks/headers/header_queries.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,21 @@

#include <silkworm/db/datastore/snapshots/basic_queries.hpp>

#include "../schema_config.hpp"
#include "header_segment.hpp"

namespace silkworm::snapshots {

using HeaderFindByBlockNumSegmentQuery = FindByIdSegmentQuery<HeaderSegmentReader>;
using HeaderFindByHashSegmentQuery = FindByHashSegmentQuery<HeaderSegmentReader>;
using HeaderFindByBlockNumSegmentQuery = FindByIdSegmentQuery<HeaderSegmentReader, &db::blocks::kHeaderSegmentAndIdxNames>;

struct HeaderFindByBlockNumQuery : public FindByTimestampMapQuery<HeaderFindByBlockNumSegmentQuery> {
using FindByTimestampMapQuery::FindByTimestampMapQuery;
std::optional<BlockHeader> exec(BlockNum block_num) {
return FindByTimestampMapQuery::exec(block_num, block_num);
}
};

using HeaderFindByHashSegmentQuery = FindByHashSegmentQuery<HeaderSegmentReader, &db::blocks::kHeaderSegmentAndIdxNames>;
using HeaderFindByHashQuery = FindMapQuery<HeaderFindByHashSegmentQuery>;

} // namespace silkworm::snapshots
12 changes: 8 additions & 4 deletions silkworm/db/blocks/transactions/txn_queries.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,14 @@

namespace silkworm::snapshots {

using TransactionFindByIdSegmentQuery = FindByIdSegmentQuery<TransactionSegmentReader>;
using TransactionFindByHashSegmentQuery = FindByHashSegmentQuery<TransactionSegmentReader>;
using TransactionRangeFromIdSegmentQuery = RangeFromIdSegmentQuery<TransactionSegmentReader>;
using TransactionPayloadRlpRangeFromIdSegmentQuery = RangeFromIdSegmentQuery<TransactionSegmentPayloadRlpReader<Bytes>>;
using TransactionFindByIdSegmentQuery = FindByIdSegmentQuery<TransactionSegmentReader, &db::blocks::kTxnSegmentAndIdxNames>;
using TransactionFindByHashSegmentQuery = FindByHashSegmentQuery<TransactionSegmentReader, &db::blocks::kTxnSegmentAndIdxNames>;

using TransactionRangeFromIdSegmentQuery = RangeFromIdSegmentQuery<TransactionSegmentReader, &db::blocks::kTxnSegmentAndIdxNames>;
using TransactionRangeFromIdQuery = FindByTimestampMapQuery<TransactionRangeFromIdSegmentQuery>;

using TransactionPayloadRlpRangeFromIdSegmentQuery = RangeFromIdSegmentQuery<TransactionSegmentPayloadRlpReader<Bytes>, &db::blocks::kTxnSegmentAndIdxNames>;
using TransactionPayloadRlpRangeFromIdQuery = FindByTimestampMapQuery<TransactionPayloadRlpRangeFromIdSegmentQuery>;

class TransactionBlockNumByTxnHashSegmentQuery {
public:
Expand Down
12 changes: 6 additions & 6 deletions silkworm/db/datastore/snapshots/basic_queries.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ namespace silkworm::snapshots {

template <
segment::SegmentReaderConcept TSegmentReader,
const SegmentAndAccessorIndexNames* segment_names = nullptr>
const SegmentAndAccessorIndexNames* segment_names>
class BasicSegmentQuery {
public:
explicit BasicSegmentQuery(
Expand All @@ -49,7 +49,7 @@ class BasicSegmentQuery {

template <
segment::SegmentReaderConcept TSegmentReader,
const SegmentAndAccessorIndexNames* segment_names = nullptr>
const SegmentAndAccessorIndexNames* segment_names>
struct FindByIdSegmentQuery : public BasicSegmentQuery<TSegmentReader, segment_names> {
using BasicSegmentQuery<TSegmentReader, segment_names>::BasicSegmentQuery;

Expand All @@ -65,7 +65,7 @@ struct FindByIdSegmentQuery : public BasicSegmentQuery<TSegmentReader, segment_n

template <
segment::SegmentReaderConcept TSegmentReader,
const SegmentAndAccessorIndexNames* segment_names = nullptr>
const SegmentAndAccessorIndexNames* segment_names>
struct FindByHashSegmentQuery : public BasicSegmentQuery<TSegmentReader, segment_names> {
using BasicSegmentQuery<TSegmentReader, segment_names>::BasicSegmentQuery;

Expand All @@ -88,14 +88,14 @@ struct FindByHashSegmentQuery : public BasicSegmentQuery<TSegmentReader, segment

template <
segment::SegmentReaderConcept TSegmentReader,
const SegmentAndAccessorIndexNames* segment_names = nullptr>
const SegmentAndAccessorIndexNames* segment_names>
struct RangeFromIdSegmentQuery : public BasicSegmentQuery<TSegmentReader, segment_names> {
using BasicSegmentQuery<TSegmentReader, segment_names>::BasicSegmentQuery;

std::vector<typename TSegmentReader::Iterator::value_type> exec_into_vector(uint64_t first_id, uint64_t count) {
std::optional<std::vector<typename TSegmentReader::Iterator::value_type>> exec(uint64_t first_id, uint64_t count) {
auto offset = this->index_.lookup_by_data_id(first_id);
if (!offset) {
return {};
return std::nullopt;
}

return this->reader_.read_into_vector(*offset, count);
Expand Down
32 changes: 16 additions & 16 deletions silkworm/db/snapshot_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,18 +211,18 @@ TEST_CASE("TransactionSnapshot::txn_range OK", "[silkworm][node][snapshot][index
TransactionRangeFromIdSegmentQuery query{{txn_segment, idx_txn_hash}};

// block 1'500'012: base_txn_id is 7'341'263, txn_count is 7
CHECK(query.exec_into_vector(7'341'263, 0).empty());
CHECK(query.exec_into_vector(7'341'263, 7).size() == 7);
CHECK(query.exec(7'341'263, 0)->empty());
CHECK(query.exec(7'341'263, 7)->size() == 7);

// block 1'500'013: base_txn_id is 7'341'272, txn_count is 1
CHECK(query.exec_into_vector(7'341'272, 0).empty());
CHECK(query.exec_into_vector(7'341'272, 1).size() == 1);
CHECK(query.exec(7'341'272, 0)->empty());
CHECK(query.exec(7'341'272, 1)->size() == 1);

// invalid base_txn_id returns empty
CHECK(query.exec_into_vector(0, 1).empty());
CHECK(query.exec_into_vector(10'000'000, 1).empty());
CHECK(query.exec_into_vector(7'341'261, 1).empty()); // before the first system tx
CHECK(query.exec_into_vector(7'341'274, 1).empty()); // after the last system tx
CHECK_FALSE(query.exec(0, 1));
CHECK_FALSE(query.exec(10'000'000, 1));
CHECK_FALSE(query.exec(7'341'261, 1)); // before the first system tx
CHECK_FALSE(query.exec(7'341'274, 1)); // after the last system tx
}

TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][index]") {
Expand All @@ -240,18 +240,18 @@ TEST_CASE("TransactionSnapshot::txn_rlp_range OK", "[silkworm][node][snapshot][i
TransactionPayloadRlpRangeFromIdSegmentQuery query{{txn_segment, idx_txn_hash}};

// block 1'500'012: base_txn_id is 7'341'263, txn_count is 7
CHECK(query.exec_into_vector(7'341'263, 0).empty());
CHECK(query.exec_into_vector(7'341'263, 7).size() == 7);
CHECK(query.exec(7'341'263, 0)->empty());
CHECK(query.exec(7'341'263, 7)->size() == 7);

// block 1'500'013: base_txn_id is 7'341'272, txn_count is 1
CHECK(query.exec_into_vector(7'341'272, 0).empty());
CHECK(query.exec_into_vector(7'341'272, 1).size() == 1);
CHECK(query.exec(7'341'272, 0)->empty());
CHECK(query.exec(7'341'272, 1)->size() == 1);

// invalid base_txn_id returns empty
CHECK(query.exec_into_vector(0, 1).empty());
CHECK(query.exec_into_vector(10'000'000, 1).empty());
CHECK(query.exec_into_vector(7'341'261, 1).empty()); // before the first system tx
CHECK(query.exec_into_vector(7'341'274, 1).empty()); // after the last system tx
CHECK_FALSE(query.exec(0, 1));
CHECK_FALSE(query.exec(10'000'000, 1));
CHECK_FALSE(query.exec(7'341'261, 1)); // before the first system tx
CHECK_FALSE(query.exec(7'341'274, 1)); // after the last system tx
}

TEST_CASE("slice_tx_payload", "[silkworm][node][snapshot]") {
Expand Down

0 comments on commit 9515f59

Please sign in to comment.