diff --git a/src/blockchain_db/lmdb/db_lmdb.cpp b/src/blockchain_db/lmdb/db_lmdb.cpp index 50ae84406ec..21a413a9b96 100644 --- a/src/blockchain_db/lmdb/db_lmdb.cpp +++ b/src/blockchain_db/lmdb/db_lmdb.cpp @@ -1555,32 +1555,119 @@ fcmp_pp::curve_trees::CurveTreesV1::TreeReduction BlockchainLMDB::get_tree_reduc // TODO: cleaner name std::pair BlockchainLMDB::get_last_hashes(const uint64_t block_idx) const { - fcmp_pp::curve_trees::PathBytes last_hashes; - // Calculate what the last hashes should be at every layer for the provided block const auto tree_reduction = this->get_tree_reduction(block_idx + 1)/*new_n_blocks*/; - // NOW: get ALL the children from every layer's last chunk as of the provided block - // TODO: cleaner function for this, de-dup from code above. The only diff is get_trim_instructions - const uint64_t new_n_leaf_tuples = block_idx == 0 ? 0 : this->get_block_n_leaf_tuples(block_idx); - const uint64_t old_n_leaf_tuples = this->get_num_leaf_tuples(); + // Get ALL the children from every layer's last chunk as of the provided block + const uint64_t new_n_leaf_tuples = tree_reduction.new_total_leaf_tuples; + if (new_n_leaf_tuples == 0) + return { 0, {} }; - if (new_n_leaf_tuples > old_n_leaf_tuples) - throw1(DB_ERROR("Unexpected: more leaf tuples are in prev block, tree is expected to only grow")); - const uint64_t trim_n_leaf_tuples = old_n_leaf_tuples - new_n_leaf_tuples; + const auto last_path_indexes = m_curve_trees->get_path_indexes(new_n_leaf_tuples, new_n_leaf_tuples - 1); + auto path = this->get_path(last_path_indexes); - CHECK_AND_ASSERT_THROW_MES(m_curve_trees != nullptr, "curve trees must be set"); - const auto trim_instructions = m_curve_trees->get_trim_instructions(old_n_leaf_tuples, - trim_n_leaf_tuples, - true/*always_regrow_with_remaining*/); + bool use_c2 = true; + std::size_t c1_idx = 0; + std::size_t c2_idx = 0; + for (auto &layer : path.layer_chunks) + { + if (use_c2) + { + const auto &layer_reduction = tree_reduction.c2_layer_reductions[c2_idx]; + if (layer_reduction.update_existing_last_hash) + layer.back() = m_curve_trees->m_c2->to_bytes(layer_reduction.new_last_hash); + ++c2_idx; + } + else + { + const auto &layer_reduction = tree_reduction.c1_layer_reductions[c1_idx]; + if (layer_reduction.update_existing_last_hash) + layer.back() = m_curve_trees->m_c1->to_bytes(layer_reduction.new_last_hash); + ++c1_idx; + } + + use_c2 = !use_c2; + } - // FIXME: I don't like this because it will sometimes exclude complete paths, which seriously complicates the code for very little benefit (only a 1/18 chance the leaf layer will be full and doesn't need to be sent to client) - const auto last_chunk_children = this->get_last_chunk_children(trim_instructions); + return { tree_reduction.new_total_leaf_tuples, path }; +} + +fcmp_pp::curve_trees::PathBytes BlockchainLMDB::get_path(const fcmp_pp::curve_trees::PathIndexes &path_indexes) const +{ + LOG_PRINT_L3("BlockchainLMDB::" << __func__); + check_open(); + + TXN_PREFIX_RDONLY(); + RCURSOR(layers) + + fcmp_pp::curve_trees::PathBytes path_bytes; + + auto &leaves_out = path_bytes.leaves; + auto &layer_chunks_out = path_bytes.layer_chunks; + + // Get the leaves + // TODO: separate function for leaves + { + if (path_indexes.leaf_range.second > path_indexes.leaf_range.first) + { + leaves_out.reserve(path_indexes.leaf_range.second - path_indexes.leaf_range.first); + + uint64_t idx = path_indexes.leaf_range.first; + + MDB_val k = zerokval; + MDB_val_copy v(idx); - // TODO: m_cuve_trees->get_last_path_indexes(new_n_leaf_tuples) - // TODO: Then use those indexes to get all the path elems from the db + MDB_cursor_op leaf_op = MDB_GET_BOTH; + do + { + int result = mdb_cursor_get(m_cur_leaves, &k, &v, leaf_op); + leaf_op = MDB_NEXT; + if (result == MDB_NOTFOUND) + throw0(DB_ERROR("leaf not found")); // TODO: specific error type instead of DB_ERROR + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to get leaf: ", result).c_str())); + + auto *db_leaf = (mdb_leaf *)v.mv_data; + leaves_out.emplace_back(std::move(db_leaf->output_context)); + + ++idx; + } + while (idx < path_indexes.leaf_range.second); + } + } + + // Traverse the tree layer-by-layer starting at the layer closest to leaf layer, getting children to trim + // TODO: separate function for layers + std::size_t layer_idx = 0; + for (const auto &layer_idx_range : path_indexes.layers) + { + std::vector> chunk; + + MDB_val_set(k, layer_idx); + MDB_val_set(v, layer_idx_range.first); + MDB_cursor_op op = MDB_GET_BOTH; + for (std::size_t i = layer_idx_range.first; i < layer_idx_range.second; ++i) + { + MDEBUG("Getting child to trim at layer_idx: " << layer_idx << " , idx: " << i); + + int result = mdb_cursor_get(m_cur_layers, &k, &v, op); + op = MDB_NEXT_DUP; + if (result == MDB_NOTFOUND) + throw0(DB_ERROR("layer elem not found")); // TODO: specific error type instead of DB_ERROR + if (result != MDB_SUCCESS) + throw0(DB_ERROR(lmdb_error("Failed to get layer elem: ", result).c_str())); + + auto *lv = (layer_val *)v.mv_data; + chunk.emplace_back(std::move(lv->child_chunk_hash)); + } + + layer_chunks_out.emplace_back(std::move(chunk)); + ++layer_idx; + } + + TXN_POSTFIX_RDONLY(); - return { tree_reduction.new_total_leaf_tuples, last_hashes }; + return path_bytes; } void BlockchainLMDB::trim_block() diff --git a/src/blockchain_db/lmdb/db_lmdb.h b/src/blockchain_db/lmdb/db_lmdb.h index ec9cd5111e2..7ae5d67efe9 100644 --- a/src/blockchain_db/lmdb/db_lmdb.h +++ b/src/blockchain_db/lmdb/db_lmdb.h @@ -381,6 +381,8 @@ class BlockchainLMDB : public BlockchainDB virtual bool audit_tree(const uint64_t expected_n_leaf_tuples) const; + fcmp_pp::curve_trees::PathBytes get_path(const fcmp_pp::curve_trees::PathIndexes &path_indexes) const; + virtual std::pair get_last_hashes(const uint64_t block_idx) const; private: diff --git a/src/fcmp_pp/curve_trees.cpp b/src/fcmp_pp/curve_trees.cpp index 09d3b18b619..705310cbb39 100644 --- a/src/fcmp_pp/curve_trees.cpp +++ b/src/fcmp_pp/curve_trees.cpp @@ -1138,10 +1138,9 @@ std::size_t CurveTrees::n_layers(const uint64_t n_leaf_tuples) const template std::size_t CurveTrees::n_layers(const uint64_t n_leaf_tuples) const; //---------------------------------------------------------------------------------------------------------------------- template -typename CurveTrees::PathIndexes CurveTrees::get_path_indexes(const uint64_t n_leaf_tuples, - const uint64_t leaf_tuple_idx) const +PathIndexes CurveTrees::get_path_indexes(const uint64_t n_leaf_tuples, const uint64_t leaf_tuple_idx) const { - CurveTrees::PathIndexes path_indexes_out; + PathIndexes path_indexes_out; if (n_leaf_tuples <= leaf_tuple_idx) return path_indexes_out; @@ -1173,10 +1172,8 @@ typename CurveTrees::PathIndexes CurveTrees::get_path_indexes(co std::pair range = { start_range, end_range }; if (leaf_layer) path_indexes_out.leaf_range = std::move(range); - else if (parent_is_c2) - path_indexes_out.c1_layers.emplace_back(std::move(range)); else - path_indexes_out.c2_layers.emplace_back(std::move(range)); + path_indexes_out.layers.emplace_back(std::move(range)); child_idx = parent_idx; n_children = n_parents; @@ -1190,8 +1187,7 @@ typename CurveTrees::PathIndexes CurveTrees::get_path_indexes(co } // Explicit instantiation -template CurveTrees::PathIndexes CurveTrees::get_path_indexes( - const uint64_t n_leaf_tuples, +template PathIndexes CurveTrees::get_path_indexes(const uint64_t n_leaf_tuples, const uint64_t leaf_tuple_idx) const; //---------------------------------------------------------------------------------------------------------------------- template<> diff --git a/src/fcmp_pp/curve_trees.h b/src/fcmp_pp/curve_trees.h index 3254047a4c8..5512e9643db 100644 --- a/src/fcmp_pp/curve_trees.h +++ b/src/fcmp_pp/curve_trees.h @@ -221,6 +221,26 @@ struct PathBytes final END_KV_SERIALIZE_MAP() }; + +// The indexes in the tree of a leaf's path elems containing whole chunks at each layer +// - leaf_range refers to a complete chunk of leaves +// - c2_layers[0] refers to the chunk of elems in the tree in the layer after leaves. The index of the hash of the chunk +// of leaves is 1 member of the c2_layers[0] chunk. The rest of c2_layers[0] is the chunk of elems that hash is in. +// - layers alternate between C1 and C2 +// - c1_layers[0] refers to the chunk of elems in the tree in the layer after c2_layers[0]. The indeox of the hash of +// the chunk of c2_layers[0] is 1 member of the c1_layers[0] chunk. The rest of c1_layers[0] is the chunk of elems +// that has is in. +// - c2_layers[1] refers to the chunk of elems in the tree in the layer after c1_layers[0] etc. +struct PathIndexes final +{ + using StartIdx = uint64_t; + using EndIdxExclusive = uint64_t; + using Range = std::pair; + + Range leaf_range; + std::vector layers; +}; + //---------------------------------------------------------------------------------------------------------------------- // Hash a chunk of new children template @@ -337,26 +357,6 @@ class CurveTrees bool empty() { return leaves.empty() && c1_layers.empty() && c2_layers.empty(); } }; - // The indexes in the tree of a leaf's path elems containing whole chunks at each layer - // - leaf_range refers to a complete chunk of leaves - // - c2_layers[0] refers to the chunk of elems in the tree in the layer after leaves. The index of the hash of the chunk - // of leaves is 1 member of the c2_layers[0] chunk. The rest of c2_layers[0] is the chunk of elems that hash is in. - // - layers alternate between C1 and C2 - // - c1_layers[0] refers to the chunk of elems in the tree in the layer after c2_layers[0]. The indeox of the hash of - // the chunk of c2_layers[0] is 1 member of the c1_layers[0] chunk. The rest of c1_layers[0] is the chunk of elems - // that has is in. - // - c2_layers[1] refers to the chunk of elems in the tree in the layer after c1_layers[0] etc. - struct PathIndexes final - { - using StartIdx = uint64_t; - using EndIdxExclusive = uint64_t; - using Range = std::pair; - - Range leaf_range; - std::vector c1_layers; - std::vector c2_layers; - }; - //member functions public: // Convert output pairs into leaf tuples, from {output pubkey,commitment} -> {O,C} -> {O.x,I.x,C.x} diff --git a/src/fcmp_pp/tree_sync_memory.cpp b/src/fcmp_pp/tree_sync_memory.cpp index 7a2f579c152..80a3aa2c6f4 100644 --- a/src/fcmp_pp/tree_sync_memory.cpp +++ b/src/fcmp_pp/tree_sync_memory.cpp @@ -1176,6 +1176,7 @@ bool TreeSyncMemory::pop_block() continue; // Get the output's cached path indexes in the tree + // TODO: Why did I get this old_path_idxs here? const LeafIdx leaf_idx = registered_o.second.leaf_idx; const auto old_path_idxs = TreeSync::m_curve_trees->get_path_indexes(old_n_leaf_tuples, leaf_idx);