fcmp++: scan from arbitrary height working

- migration code done

src/blockchain_db/lmdb/db_lmdb.cpp

@@ -867,22 +867,25 @@ void BlockchainLMDB::add_block(const block& blk, size_t block_weight, uint64_t l
     throw0(DB_ERROR(lmdb_error("Failed to add block height by hash to db transaction: ", result).c_str()));
 
   CURSOR(locked_outputs)
+  CURSOR(timelocked_outputs)
 
   // Add the locked outputs from this block to the locked outputs and custom timelocked tables
   for (const auto &unlock_block : outs_by_unlock_block)
   {
-    MDB_val_set(k_block_id, unlock_block.first);
+    const uint64_t last_locked_block_idx = unlock_block.first;
     for (const auto &locked_output : unlock_block.second)
     {
+      MDB_val_set(k_block_id, last_locked_block_idx);
       MDB_val_set(v_output, locked_output);
       result = mdb_cursor_put(m_cur_locked_outputs, &k_block_id, &v_output, MDB_APPENDDUP);
       if (result != MDB_SUCCESS)
         throw0(DB_ERROR(lmdb_error("Failed to add locked output: ", result).c_str()));
 
       if (timelocked_outputs.find(locked_output.output_id) != timelocked_outputs.end())
       {
+        MDB_val_set(k_timelocked_block_id, last_locked_block_idx);
         MDB_val_set(v_timelocked_output, locked_output);
-        result = mdb_cursor_put(m_cur_timelocked_outputs, &k_block_id, &v_timelocked_output, MDB_APPENDDUP);
+        result = mdb_cursor_put(m_cur_timelocked_outputs, &k_timelocked_block_id, &v_timelocked_output, MDB_APPENDDUP);
         if (result != MDB_SUCCESS)
           throw0(DB_ERROR(lmdb_error("Failed to add timelocked output: ", result).c_str()));
       }
@@ -1275,8 +1278,27 @@ void BlockchainLMDB::remove_output(const uint64_t amount, const uint64_t& out_in
       throw0(DB_ERROR(lmdb_error(std::string("Error deleting locked output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
   }
 
-  // TODO: remove output from custom timelocked outputs table if custom timelocked
-  // Just search for it and see if it's in the table, remove if so
+  // Remove output from custom timelocked outputs table if present
+  CURSOR(timelocked_outputs);
+
+  MDB_val_set(k_timelocked_block_id, unlock_block);
+  MDB_val_set(v_timelocked_output, ok->output_id);
+
+  result = mdb_cursor_get(m_cur_timelocked_outputs, &k_timelocked_block_id, &v_timelocked_output, MDB_GET_BOTH);
+  if (result == MDB_NOTFOUND)
+  {
+    // Output is either not timelocked or is invalid
+  }
+  else if (result)
+  {
+    throw1(DB_ERROR(lmdb_error("Error adding removal of timelocked output to db transaction", result).c_str()));
+  }
+  else
+  {
+    result = mdb_cursor_del(m_cur_timelocked_outputs, 0);
+    if (result)
+      throw0(DB_ERROR(lmdb_error(std::string("Error deleting timelocked output index ").append(boost::lexical_cast<std::string>(out_index).append(": ")).c_str(), result).c_str()));
+  }
 
   result = mdb_cursor_del(m_cur_output_txs, 0);
   if (result)
@@ -1559,7 +1581,7 @@ fcmp_pp::curve_trees::CurveTreesV1::TreeReduction BlockchainLMDB::get_tree_reduc
 std::pair<uint64_t, fcmp_pp::curve_trees::PathBytes> BlockchainLMDB::get_last_hashes(const uint64_t block_idx) const
 {
   // Calculate what the last hashes should be at every layer for the provided block
-  const auto tree_reduction = this->get_tree_reduction(block_idx + 1)/*new_n_blocks*/;
+  const auto tree_reduction = this->get_tree_reduction(block_idx + 1/*new_n_blocks*/);
 
   // Get ALL the children from every layer's last chunk as of the provided block
   const uint64_t new_n_leaf_tuples = tree_reduction.new_total_leaf_tuples;
@@ -2591,6 +2613,9 @@ fcmp_pp::curve_trees::OutputsByUnlockBlock BlockchainLMDB::get_custom_timelocked
   uint64_t blk_idx = start_block_idx;
   MDB_cursor_op op = MDB_SET;
   bool reading_first_op_next_multiple = false;
+  // TODO: iterate backwards over timelocked outputs table until encountering last locked block < start_block_idx
+  // instead of checking every unlock block
+  // WARNING: slow if there are a lot of timelocked outputs
   while (blk_idx < n_blocks)
   {
     MDB_val_set(k_block_id, blk_idx);
@@ -2660,14 +2685,14 @@ fcmp_pp::curve_trees::OutputsByUnlockBlock BlockchainLMDB::get_custom_timelocked
   for (const auto &output_id : output_ids)
   {
     const auto output_tx = this->get_output_tx_and_index_from_global(output_id.first);
-    const uint64_t tx_block_idx = this->get_tx_block_height(output_tx.first);
+    const uint64_t created_block_idx = this->get_tx_block_height(output_tx.first);
 
-    // As soon as we encounter an output created before start_block_idx, we're done
-    if (tx_block_idx < start_block_idx)
+    // As soon as we encounter an output created before start_block_idx, we're done, we've removed all we needed to
+    if (created_block_idx < start_block_idx)
       break;
 
     // Find the output in the outs container
-    auto blk_it = outs.find(output_id.second);
+    auto blk_it = outs.find(output_id.second/*unlock_block*/);
 
     // The last one in the vec should be the output id since it should be sorted
     if (blk_it == outs.end())
@@ -2697,9 +2722,14 @@ fcmp_pp::curve_trees::OutputsByUnlockBlock BlockchainLMDB::get_recent_locked_out
   RCURSOR(timelocked_outputs)
 
   fcmp_pp::curve_trees::OutputsByUnlockBlock outs;
+
   if (end_block_idx == 0)
     return outs;
 
+  const uint64_t height = this->height();
+  if (height == 0)
+    return outs;
+
   const uint64_t coinbase_start_idx = CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW > end_block_idx
     ? 0
     : end_block_idx - CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW;
@@ -2708,7 +2738,7 @@ fcmp_pp::curve_trees::OutputsByUnlockBlock BlockchainLMDB::get_recent_locked_out
     ? 0
     : end_block_idx - CRYPTONOTE_DEFAULT_TX_SPENDABLE_AGE;
 
-  const uint64_t end_blk_idx = std::min(this->height(), end_block_idx);
+  const uint64_t end_blk_idx = std::min(height - 1, end_block_idx);
 
   const auto get_out_unlock_blocks = [this, &outs, normal_start_idx](uint64_t b_idx, const crypto::hash, const block &b) -> bool
   {
@@ -2739,7 +2769,7 @@ fcmp_pp::curve_trees::OutputsByUnlockBlock BlockchainLMDB::get_recent_locked_out
     // Add coinbase outputs
     add_outs_by_unlock_block(cryptonote::get_transaction_hash(b.miner_tx), true);
 
-    if (b_idx < normal_start_idx)
+    if (normal_start_idx > b_idx)
       return true;
 
     // Add normal outputs
@@ -7374,7 +7404,7 @@ void BlockchainLMDB::migrate_5_6()
       struct tmp_output_cache { uint64_t n_outputs_read; uint64_t amount; outkey ok; };
       tmp_output_cache last_output;
 
-      MDB_cursor *c_output_amounts, *c_locked_outputs, *c_tmp_last_output;
+      MDB_cursor *c_output_amounts, *c_locked_outputs, *c_tmp_last_output, *c_timelocked_outputs;
       MDB_val k, v;
 
       i = 0;
@@ -7420,6 +7450,9 @@ void BlockchainLMDB::migrate_5_6()
           result = mdb_cursor_open(txn, m_tmp_last_output, &c_tmp_last_output);
           if (result)
             throw0(DB_ERROR(lmdb_error("Failed to open a cursor for temp last output: ", result).c_str()));
+          result = mdb_cursor_open(txn, m_timelocked_outputs, &c_timelocked_outputs);
+          if (result)
+            throw0(DB_ERROR(lmdb_error("Failed to open a cursor for timelocked outputs: ", result).c_str()));
 
           // Get the cached last output from the db
           bool found_cached_output = false;
@@ -7527,7 +7560,7 @@ void BlockchainLMDB::migrate_5_6()
         // Get the block in which the output will unlock
         const uint64_t unlock_block = cryptonote::get_unlock_block_index(output_data.unlock_time, output_data.height);
 
-        // Now add the output to the locked outputs table
+        // Add the output to the locked outputs table
         MDB_val_set(k_block_id, unlock_block);
         MDB_val_set(v_output, output_context);
 
@@ -7536,6 +7569,30 @@ void BlockchainLMDB::migrate_5_6()
         result = mdb_cursor_put(c_locked_outputs, &k_block_id, &v_output, MDB_NODUPDATA);
         if (result != MDB_SUCCESS)
           throw0(DB_ERROR(lmdb_error("Failed to add locked output: ", result).c_str()));
+
+        // Check if the output is a coinbase output
+        bool is_coinbase = false;
+        const bool has_coinbase_unlock_block = unlock_block == (output_data.height + CRYPTONOTE_MINED_MONEY_UNLOCK_WINDOW - 1);
+        if (has_coinbase_unlock_block)
+        {
+          // Only coinbase outputs could potentially have the coinbase unlock block (see prevalidate_miner_transaction)
+          auto toi = this->get_output_tx_and_index_from_global(output_id);
+          auto tx = this->get_pruned_tx(toi.first);
+          is_coinbase = cryptonote::is_coinbase(tx);
+        }
+
+        // Add custom timelocked outputs to the timelocked outputs table
+        if (!cryptonote::is_custom_timelocked(is_coinbase, unlock_block, output_data.height))
+          continue;
+
+        MDB_val_set(k_timelocked_block_id, unlock_block);
+        MDB_val_set(v_timelocked_output, output_context);
+
+        // MDB_NODUPDATA because all output id's should be unique
+        // Can't use MDB_APPENDDUP because outputs aren't inserted in order sorted by output_id
+        result = mdb_cursor_put(c_timelocked_outputs, &k_timelocked_block_id, &v_timelocked_output, MDB_NODUPDATA);
+        if (result != MDB_SUCCESS)
+          throw0(DB_ERROR(lmdb_error("Failed to add timelocked output: ", result).c_str()));
       }
     }
 
@@ -7678,8 +7735,6 @@ void BlockchainLMDB::migrate_5_6()
 
       txn.commit();
     }
-
-    // TODO: Step 4, iterate over all txs in order and add the custom timelocked outputs to the timelocked_outputs table
   } while(0);
 
   // Update db version

src/fcmp_pp/tree_sync_memory.cpp

@@ -1407,6 +1407,14 @@ void TreeSyncMemory<C1, C2>::init(const uint64_t start_block_idx,
     // - Assume the created block idx is the genesis block so the outputs won't get pruned.
     const CreatedBlockIdx created_block_idx{0};
     add_to_locked_outputs_cache(timelocked_outputs, created_block_idx, m_locked_outputs, m_locked_output_refs);
+
+    // Set the output count to the max output id + 1
+    // WARNING: this is a little hacky because if there are no timelocked outputs provided (which should never be the
+    // case), then the output count would be 0 even if initializing at a block index > 0
+    for (const auto &bl: timelocked_outputs)
+        for (const auto &o : bl.second)
+            if (o.output_id >= m_output_count)
+                m_output_count = o.output_id + 1;
 }
 
 // Explicit instantiation

src/rpc/core_rpc_server.cpp

@@ -616,17 +616,12 @@ namespace cryptonote
     init_tree_sync_data.init_block_idx = init_block_idx;
     init_tree_sync_data.init_block_hash = init_hash;
 
-    if (init_block_idx == 0)
-    {
-      res.init_tree_sync_data = std::move(init_tree_sync_data);
-      return true;
-    }
-
-    // 1. Custom timelocked outputs created before init_block_idx with last locked block >= init_block_idx
-    auto custom_outs_by_unlock_block = m_core.get_blockchain_storage().get_db().get_custom_timelocked_outputs(init_block_idx);
+    // 1. Custom timelocked outputs created before sync_start_idx with last locked block >= sync_start_idx
+    const uint64_t sync_start_idx = init_block_idx + 1;
+    auto custom_outs_by_unlock_block = m_core.get_blockchain_storage().get_db().get_custom_timelocked_outputs(sync_start_idx);
 
-    // 2a. Coinbase output contexts created between blocks [init_block_idx - 60, init_block_idx]
-    // 2b. Normal output contexts created between blocks [init_block_idx - 10, init_block_idx]
+    // 2a. Coinbase output contexts created between blocks [init_block_idx - 60, init_block_idx] inclusive
+    // 2b. Normal output contexts created between blocks [init_block_idx - 10, init_block_idx] inclusive
     auto outs_by_unlock_block = m_core.get_blockchain_storage().get_db().get_recent_locked_outputs(init_block_idx);
 
     // 3. Combine all locked outputs into vec
@@ -673,13 +668,14 @@ namespace cryptonote
       locked_outputs.push_back({ unlock_block, std::move(o.second) });
     }
 
-    // 4. N leaf tuples and last chunk at each layer of the tree at requested block height - 1
-    auto last_hashes = m_core.get_blockchain_storage().get_db().get_last_hashes(init_block_idx - 1);
+    // 4. N leaf tuples and last chunk at each layer of the tree at init_block_idx
+    auto last_hashes = m_core.get_blockchain_storage().get_db().get_last_hashes(init_block_idx);
     init_tree_sync_data.n_leaf_tuples = last_hashes.first;
     init_tree_sync_data.last_hashes = std::move(last_hashes.second);
 
     MDEBUG("Set init tree sync data, blk " << init_tree_sync_data.init_block_idx << " , hash " << init_tree_sync_data.init_block_hash);
     res.init_tree_sync_data = std::move(init_tree_sync_data);
+    res.included_init_tree_sync_data = true;
     return true;
   }
   //------------------------------------------------------------------------------------------------------------------------------

src/rpc/core_rpc_server_commands_defs.h

@@ -288,6 +288,7 @@ inline const std::string get_rpc_status(const bool trusted_daemon, const std::st
       std::vector<pool_tx_info> added_pool_txs;
       std::vector<crypto::hash> remaining_added_pool_txids;
       std::vector<crypto::hash> removed_pool_txids;
+      bool included_init_tree_sync_data;
       init_tree_sync_data_t init_tree_sync_data;
 
       BEGIN_KV_SERIALIZE_MAP()
@@ -308,7 +309,11 @@ inline const std::string get_rpc_status(const bool trusted_daemon, const std::st
         {
           KV_SERIALIZE_CONTAINER_POD_AS_BLOB(removed_pool_txids)
         }
-        KV_SERIALIZE(init_tree_sync_data) // FIXME: make this optional (it could also intro a major breaking change to the daemon)
+        KV_SERIALIZE_OPT(included_init_tree_sync_data, false)
+        if (included_init_tree_sync_data)
+        {
+          KV_SERIALIZE(init_tree_sync_data)
+        }
       END_KV_SERIALIZE_MAP()
     };
     typedef epee::misc_utils::struct_init<response_t> response;

src/wallet/wallet2.cpp

@@ -3137,7 +3137,7 @@ void wallet2::process_pool_info_extent(const cryptonote::COMMAND_RPC_GET_BLOCKS_
   update_pool_state_from_pool_data(res.pool_info_extent == COMMAND_RPC_GET_BLOCKS_FAST::INCREMENTAL, res.removed_pool_txids, added_pool_txs, process_txs, refreshed);
 }
 //----------------------------------------------------------------------------------------------------
-void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>>& process_pool_txs, cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::init_tree_sync_data_t &init_tree_sync_data)
+void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_height, uint64_t &blocks_start_height, const std::list<crypto::hash> &short_chain_history, std::vector<cryptonote::block_complete_entry> &blocks, std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> &o_indices, uint64_t &current_height, std::vector<std::tuple<cryptonote::transaction, crypto::hash, bool>>& process_pool_txs, boost::optional<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::init_tree_sync_data_t> &init_tree_sync_data)
 {
   cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::request req = AUTO_VAL_INIT(req);
   cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::response res = AUTO_VAL_INIT(res);
@@ -3167,9 +3167,10 @@ void wallet2::pull_blocks(bool first, bool try_incremental, uint64_t start_heigh
   blocks = std::move(res.blocks);
   o_indices = std::move(res.output_indices);
   current_height = res.current_height;
-  init_tree_sync_data = std::move(res.init_tree_sync_data);
   if (res.pool_info_extent != COMMAND_RPC_GET_BLOCKS_FAST::NONE)
     m_pool_info_query_time = res.daemon_time;
+  if (res.included_init_tree_sync_data)
+    init_tree_sync_data = boost::optional<COMMAND_RPC_GET_BLOCKS_FAST::init_tree_sync_data_t>(std::move(res.init_tree_sync_data));
 
   MDEBUG("Pulled blocks: blocks_start_height " << blocks_start_height << ", count " << blocks.size()
       << ", height " << blocks_start_height + blocks.size() << ", node height " << res.current_height
@@ -3347,6 +3348,7 @@ static void tree_sync_blocks_async(const TreeSyncStartParams &tree_sync_start_pa
   const uint64_t start_parsed_block_i = tree_sync_start_params.start_parsed_block_i;
   const crypto::hash &prev_block_hash = tree_sync_start_params.prev_block_hash;
 
+  THROW_WALLET_EXCEPTION_IF(start_parsed_block_i > sync_start_block_idx, error::wallet_internal_error, "high start_parsed_block_i");
   THROW_WALLET_EXCEPTION_IF(start_parsed_block_i >= parsed_blocks.size(), error::wallet_internal_error, "high start_parsed_block_i");
   const uint64_t n_new_blocks = parsed_blocks.size() - start_parsed_block_i;
 
@@ -3689,7 +3691,7 @@ void wallet2::pull_and_parse_next_blocks(bool first, bool try_incremental, uint6
     // pull the new blocks
     std::vector<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::block_output_indices> o_indices;
     uint64_t current_height;
-    cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::init_tree_sync_data_t init_tree_sync_data;
+    boost::optional<cryptonote::COMMAND_RPC_GET_BLOCKS_FAST::init_tree_sync_data_t> init_tree_sync_data;
     pull_blocks(first, try_incremental, start_height, blocks_start_height, short_chain_history, blocks, o_indices, current_height, process_pool_txs, init_tree_sync_data);
     THROW_WALLET_EXCEPTION_IF(blocks.size() != o_indices.size(), error::wallet_internal_error, "Mismatched sizes of blocks and o_indices");
 
@@ -3747,19 +3749,18 @@ void wallet2::pull_and_parse_next_blocks(bool first, bool try_incremental, uint6
     THROW_WALLET_EXCEPTION_IF(!waiter.wait(), error::wallet_internal_error, "Exception in thread pool");
     last = !blocks.empty() && cryptonote::get_block_height(parsed_blocks.back().block) + 1 == current_height;
 
-    if (m_tree_sync.empty())
+    if (init_tree_sync_data)
     {
-      // TODO: need to handle the case when pointing to an outdated daemon
       // Initialize the tree
-      const uint64_t init_block_idx = init_tree_sync_data.init_block_idx;
-      const crypto::hash &init_block_hash = init_tree_sync_data.init_block_hash;
+      const uint64_t init_block_idx = init_tree_sync_data->init_block_idx;
+      const crypto::hash &init_block_hash = init_tree_sync_data->init_block_hash;
       MINFO("Initializing wallet tree at block " << init_block_idx << " with block hash " << init_block_hash);
 
       fcmp_pp::curve_trees::OutputsByUnlockBlock locked_outputs;
-      for (auto &lo : init_tree_sync_data.locked_outputs)
+      for (auto &lo : init_tree_sync_data->locked_outputs)
         locked_outputs[lo.unlock_block] = std::move(lo.outputs);
 
-      m_tree_sync.init(init_block_idx, init_block_hash, init_tree_sync_data.n_leaf_tuples, init_tree_sync_data.last_hashes, locked_outputs);
+      m_tree_sync.init(init_block_idx, init_block_hash, init_tree_sync_data->n_leaf_tuples, init_tree_sync_data->last_hashes, locked_outputs);
     }
   }
   catch(...)