Enable snapshot and finalizer commits integration

src/esperanza/checks.cpp

@@ -6,7 +6,6 @@
 #include <esperanza/adminparams.h>
 #include <esperanza/checks.h>
 #include <esperanza/finalizationstate.h>
-#include <finalization/vote_recorder.h>
 #include <script/interpreter.h>
 #include <script/standard.h>
 #include <txmempool.h>
@@ -204,9 +203,9 @@ bool ContextualCheckLogoutTx(const CTransaction &tx, CValidationState &err_state
   }
 
   // We keep the check for the prev at the end because is the most expensive
-  // check (potentially goes to disk) and there is a good chance that if the
-  // vote is not valid (i.e. outdated) then the function will return before
-  // reaching this point.
+  // check and there is a good chance that if the vote is not valid (i.e. outdated)
+  // then the function will return before reaching this point.
+
   TxType prev_tx_type = TxType::REGULAR;
   CScript prev_out_script;
 
@@ -345,18 +344,13 @@ bool ContextualCheckVoteTx(const CTransaction &tx, CValidationState &err_state,
     return false;
   }
 
-  if (!finalization::RecordVote(tx, err_state)) {
-    return false;
-  }
-
   if (fin_state.ValidateVote(vote) != +Result::SUCCESS) {
     return err_state.DoS(10, false, REJECT_INVALID, "bad-vote-invalid-state");
   }
 
   // We keep the check for the prev at the end because is the most expensive
-  // check (potentially goes to disk) and there is a good chance that if the
-  // vote is not valid (i.e. outdated) then the function will return before
-  // reaching this point.
+  // check and there is a good chance that if the vote is not valid (i.e. outdated)
+  // then the function will return before reaching this point.
 
   TxType prev_tx_type = TxType::REGULAR;
   CScript prev_out_script;

src/finalization/vote_recorder.cpp

@@ -133,24 +133,22 @@ std::shared_ptr<VoteRecorder> VoteRecorder::GetVoteRecorder() {
 
 CScript VoteRecord::GetScript() const { return CScript::EncodeVote(vote, sig); }
 
-bool RecordVote(const CTransaction &tx, CValidationState &err_state) {
+bool RecordVote(const CTransaction &tx,
+                CValidationState &err_state,
+                const FinalizationState &fin_state) {
   assert(tx.IsVote());
 
-  LOCK(GetComponent<StateRepository>()->GetLock());
-  const FinalizationState *fin_state = GetComponent<StateRepository>()->GetTipState();
-  assert(fin_state != nullptr);
-
   esperanza::Vote vote;
   std::vector<unsigned char> voteSig;
 
   if (!CScript::ExtractVoteFromVoteSignature(tx.vin[0].scriptSig, vote, voteSig)) {
     return err_state.DoS(10, false, REJECT_INVALID, "bad-vote-data-format");
   }
-  const esperanza::Result res = fin_state->ValidateVote(vote);
+  const esperanza::Result res = fin_state.ValidateVote(vote);
 
   if (res != +esperanza::Result::ADMIN_BLACKLISTED &&
       res != +esperanza::Result::VOTE_NOT_BY_VALIDATOR) {
-    finalization::VoteRecorder::GetVoteRecorder()->RecordVote(vote, voteSig, *fin_state);
+    finalization::VoteRecorder::GetVoteRecorder()->RecordVote(vote, voteSig, fin_state);
   }
 
   return true;

src/finalization/vote_recorder.h

@@ -63,7 +63,14 @@ class VoteRecorder : private boost::noncopyable {
   static std::shared_ptr<VoteRecorder> GetVoteRecorder();
 };
 
-bool RecordVote(const CTransaction &tx, CValidationState &err_state);
+//! \brief Records the vote.
+//!
+//! tx must be a vote transaction
+//! fin_state is a FinalizationState VoteRecorder must rely on when it checks transaction validity
+//!           and slashable condition. It must be best known finalization state on a moment.
+bool RecordVote(const CTransaction &tx,
+                CValidationState &err_state,
+                const FinalizationState &fin_state);
 
 }  // namespace finalization
 

src/net_processing.cpp

@@ -3265,7 +3265,26 @@ bool PeerLogicValidation::SendMessages(CNode* pto, size_t node_index, size_t tot
                 pto->vAddrToSend.shrink_to_fit();
         }
 
-        snapshot::StartInitialSnapshotDownload(*pto, node_index, total_nodes, msgMaker);
+        //! UNIT-E: When snapshot becomes a component, we can hide this code there and
+        //! evaluate it only if needed.
+        const CBlockIndex *last_finalized_checkpoint =
+            GetComponent<p2p::FinalizerCommitsHandler>()->GetLastFinalizedCheckpoint();
+        {
+            LOCK(GetComponent<finalization::StateRepository>()->GetLock());
+            const auto *fin_state = GetComponent<finalization::StateRepository>()->GetTipState();
+            assert(fin_state != nullptr);
+            const uint32_t epoch = fin_state->GetLastFinalizedEpoch();
+            if (last_finalized_checkpoint == nullptr ||
+                epoch > fin_state->GetEpoch(*last_finalized_checkpoint)) {
+
+                const blockchain::Height h = fin_state->GetEpochCheckpointHeight(epoch);
+                last_finalized_checkpoint = chainActive[h];
+            }
+        }
+
+        assert(last_finalized_checkpoint != nullptr);
+
+        snapshot::StartInitialSnapshotDownload(*pto, node_index, total_nodes, msgMaker, *last_finalized_checkpoint);
 
         // Start block sync
         if (pindexBestHeader == nullptr)
@@ -3291,13 +3310,8 @@ bool PeerLogicValidation::SendMessages(CNode* pto, size_t node_index, size_t tot
                    got back an empty response.  */
                 if (pindexStart->pprev)
                     pindexStart = pindexStart->pprev;
-                if (snapshot::IsISDEnabled()) {
-                    LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight);
-                    connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256()));
-                } else {
-                    LogPrint(BCLog::NET, "initial getcommits (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight);
-                    connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETCOMMITS, GetComponent<p2p::FinalizerCommitsHandler>()->GetFinalizerCommitsLocator(*pindexStart, nullptr)));
-                }
+                LogPrint(BCLog::NET, "initial getcommits (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), pto->nStartingHeight);
+                connman->PushMessage(pto, msgMaker.Make(NetMsgType::GETCOMMITS, GetComponent<p2p::FinalizerCommitsHandler>()->GetFinalizerCommitsLocator(*pindexStart, nullptr)));
             }
         }
 

src/p2p/finalizer_commits_handler.h

@@ -54,11 +54,17 @@ class FinalizerCommitsHandler {
   virtual void OnDisconnect(NodeId nodeid) = 0;
 
   //! \brief Find whether we need to download blocks to satisfy commits full sync
-  //
-  // Returns true when blocks_out modified.
+  //!
+  //! Returns true when blocks_out modified.
   virtual bool FindNextBlocksToDownload(
       NodeId nodeid, size_t count, std::vector<const CBlockIndex *> &blocks_out) = 0;
 
+  //! \brief Returns the last finalized checkpoint.
+  //!
+  //! This value is actual during commits-exchange stage only. After node leaves full-sync or fast-sync,
+  //! and processes blocks in a normal sync mode, it becomes outdated.
+  virtual const CBlockIndex *GetLastFinalizedCheckpoint() const = 0;
+
   virtual ~FinalizerCommitsHandler() = default;
 
   static std::unique_ptr<FinalizerCommitsHandler> New(

src/p2p/finalizer_commits_handler_impl.cpp

@@ -13,7 +13,6 @@
 #include <finalization/state_repository.h>
 #include <finalization/vote_recorder.h>
 #include <net_processing.h>
-#include <snapshot/p2p_processing.h>
 #include <snapshot/state.h>
 #include <staking/active_chain.h>
 #include <validation.h>
@@ -323,9 +322,12 @@ bool FinalizerCommitsHandlerImpl::OnCommits(
 
   std::list<const CBlockIndex *> to_append;
 
+  const bool fast_sync = snapshot::IsISDEnabled() && snapshot::IsInitialSnapshotDownload();
+
   const CBlockIndex *last_index = nullptr;
   {
     LOCK(m_active_chain->GetLock());
+    LOCK(m_repo->GetLock());
 
     for (const HeaderAndFinalizerCommits &d : msg.data) {
 
@@ -341,15 +343,25 @@ bool FinalizerCommitsHandlerImpl::OnCommits(
       }
 
       // UNIT-E TODO: Store finalizer transactions somewhere.
-      // We cannot perform ContextualCheck now as it relies on GetTransaction which effectively
-      // loads prev transaction from the disk. During commits exchange we do not have such data
-      // on the disk.
+      // We cannot perform ContextualCheck now as it relies on UTXO lookup. During commits
+      // exchange we do not have such data.
       // So, now just record the votes. ContextualCheck would be performed later after block
       // arrives.
 
+      // In case of fast-sync record votes relying on the previously processed finalization state.
+      // Otherwise use the tip's state.
+
+      const finalization::FinalizationState *fin_state = nullptr;
+      if (fast_sync && new_index->pprev != nullptr) {
+        fin_state = m_repo->Find(*new_index->pprev);
+      } else {
+        fin_state = m_repo->GetTipState();
+      }
+      assert(fin_state != nullptr);
+
       for (const auto &c : d.commits) {
         if (c->IsVote()) {
-          if (!finalization::RecordVote(*c, err_state)) {
+          if (!finalization::RecordVote(*c, err_state, *fin_state)) {
             return false;
           }
         }
@@ -387,7 +399,16 @@ bool FinalizerCommitsHandlerImpl::OnCommits(
   {
     LOCK(m_repo->GetLock());
 
-    const finalization::FinalizationState *tip_state = m_repo->GetTipState();
+    const finalization::FinalizationState *tip_state = nullptr;
+
+    if (m_last_finalization_point != nullptr) {
+      tip_state = m_repo->Find(*m_last_finalization_point);
+    }
+
+    if (tip_state == nullptr) {
+      tip_state = m_repo->GetTipState();
+    }
+
     const finalization::FinalizationState *index_state = m_repo->Find(*last_index);
     assert(tip_state != nullptr);
     assert(index_state != nullptr);
@@ -396,9 +417,14 @@ bool FinalizerCommitsHandlerImpl::OnCommits(
     const uint32_t tip_epoch = tip_state->GetLastFinalizedEpoch();
 
     if (index_epoch > tip_epoch) {
-      download_until = index_state->GetEpochCheckpointHeight(index_epoch + 1);
-      LogPrint(BCLog::NET, "Commits sync reached finalization at epoch=%d, mark blocks up to height %d to download\n",
-               index_epoch, download_until);
+      m_last_finalization_point = last_index;
+      const blockchain::Height h = index_state->GetEpochCheckpointHeight(index_epoch);
+      m_last_finalized_checkpoint = last_index->GetAncestor(h);
+      if (!fast_sync) {
+        download_until = index_state->GetEpochCheckpointHeight(index_epoch + 1);
+        LogPrint(BCLog::NET, "Commits sync reached finalization at epoch=%d, mark blocks up to height %d to download\n",
+                 index_epoch, download_until);
+      }
     }
   }
 
@@ -412,13 +438,20 @@ bool FinalizerCommitsHandlerImpl::OnCommits(
       LogPrint(BCLog::NET, "Commits sync finished after processing header=%s, height=%d\n",
                last_index->GetBlockHash().GetHex(), last_index->nHeight);
       download_until = last_index->nHeight;
+      if (fast_sync) {
+        snapshot::HeadersDownloaded();
+      }
       break;
 
     case FinalizerCommitsResponse::Status::LengthExceeded:
       // Just wait the next message to come
       break;
   }
 
+  if (fast_sync) {
+    return true;
+  }
+
   LOCK(cs);
 
   auto &wait_list = m_wait_list[node.GetId()];
@@ -493,6 +526,10 @@ bool FinalizerCommitsHandlerImpl::FindNextBlocksToDownload(
     return true;
   }
   return false;
-};
+}
+
+const CBlockIndex *FinalizerCommitsHandlerImpl::GetLastFinalizedCheckpoint() const {
+  return m_last_finalized_checkpoint;
+}
 
 }  // namespace p2p

src/p2p/finalizer_commits_handler_impl.h

@@ -46,6 +46,8 @@ class FinalizerCommitsHandlerImpl : public FinalizerCommitsHandler {
   bool FindNextBlocksToDownload(
       NodeId nodeid, size_t count, std::vector<const CBlockIndex *> &blocks_out) override;
 
+  const CBlockIndex *GetLastFinalizedCheckpoint() const override;
+
  protected:
   const CBlockIndex *FindMostRecentStart(const FinalizerCommitsLocator &locator) const;
 
@@ -93,6 +95,16 @@ class FinalizerCommitsHandlerImpl : public FinalizerCommitsHandler {
   mutable CCriticalSection cs;
   std::map<NodeId, std::multiset<const CBlockIndex *, HeightComparator>> m_wait_list;
   std::map<NodeId, std::list<const CBlockIndex *>> m_blocks_to_download;
+  //! The last finalized checkpoint.
+  //!  F  J votes
+  //! e1 e2 e3
+  //! It's a checkpoint of epoch e1.
+  const CBlockIndex *m_last_finalized_checkpoint = nullptr;
+  //! The point in the chain where finalization happened.
+  //!  F  J votes
+  //! e1 e2 e3
+  //! It's one of the index from epoch e3.
+  const CBlockIndex *m_last_finalization_point = nullptr;
 };
 
 }  // namespace p2p

src/rpc/rawtransaction.cpp

@@ -1050,19 +1050,59 @@ UniValue sendrawtransaction(const JSONRPCRequest& request)
     return hashTx.GetHex();
 }
 
+UniValue extractvotefromsignature(const JSONRPCRequest &request) {
+    if (request.fHelp || request.params.size() != 1) {
+        throw std::runtime_error(
+            "extractvotefromsignature\n"
+            "\nReturns JSON representation of decoded vote\n"
+            "\nArguments:\n"
+            "1. \"signature\"               (string).\n"
+            "Result:\n"
+            "{\n"
+            "  \"validator_address\": xxxx   (string) the validator address\n"
+            "  \"target_hash\": xxxx\n      (string) the target hash"
+            "  \"source_epoch\": xxxx       (numeric) the source epoch\n"
+            "  \"target_epoch\": xxxx       (numeric) the target epoch\n"
+            "}\n"
+            "\n"
+            + HelpExampleCli("extractvotefromsignature", "\"hexstring\"")
+            + HelpExampleRpc("extractvotefromsignature", "\"hexstring\""));
+    }
+    esperanza::Vote vote;
+    std::vector<unsigned char> vote_sig_out;
+
+    UniValue r(UniValue::VOBJ);
+    CScript script;
+    if (request.params[0].get_str().size() > 0) {
+        std::vector<unsigned char> data(ParseHexV(request.params[0].get_str(), "signature"));
+        script = CScript(data.begin(), data.end());
+    }
+
+    if (!CScript::ExtractVoteFromVoteSignature(script, vote, vote_sig_out)) {
+        throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "script decode failed");
+    }
+
+    r.push_back(Pair("validator_address", vote.m_validator_address.GetHex()));
+    r.push_back(Pair("target_hash", vote.m_target_hash.GetHex()));
+    r.push_back(Pair("source_epoch", (uint64_t)vote.m_source_epoch));
+    r.push_back(Pair("target_epoch", (uint64_t)vote.m_target_epoch));
+    return r;
+}
+
 static const CRPCCommand commands[] =
-{ //  category              name                      actor (function)         argNames
-  //  --------------------- ------------------------  -----------------------  ----------
-    { "rawtransactions",    "getrawtransaction",      &getrawtransaction,      {"txid","verbose","blockhash"} },
-    { "rawtransactions",    "createrawtransaction",   &createrawtransaction,   {"inputs","outputs","locktime","replaceable"} },
-    { "rawtransactions",    "decoderawtransaction",   &decoderawtransaction,   {"hexstring","iswitness"} },
-    { "rawtransactions",    "decodescript",           &decodescript,           {"hexstring"} },
-    { "rawtransactions",    "sendrawtransaction",     &sendrawtransaction,     {"hexstring","allowhighfees"} },
-    { "rawtransactions",    "combinerawtransaction",  &combinerawtransaction,  {"txs"} },
-    { "rawtransactions",    "signrawtransaction",     &signrawtransaction,     {"hexstring","prevtxs","privkeys","sighashtype"} }, /* uses wallet if enabled */
-
-    { "blockchain",         "gettxoutproof",          &gettxoutproof,          {"txids", "blockhash"} },
-    { "blockchain",         "verifytxoutproof",       &verifytxoutproof,       {"proof"} },
+{ //  category              name                        actor (function)           argNames
+  //  --------------------- --------------------------  -------------------------  ----------
+    { "rawtransactions",    "getrawtransaction",        &getrawtransaction,        {"txid","verbose","blockhash"} },
+    { "rawtransactions",    "createrawtransaction",     &createrawtransaction,     {"inputs","outputs","locktime","replaceable"} },
+    { "rawtransactions",    "decoderawtransaction",     &decoderawtransaction,     {"hexstring","iswitness"} },
+    { "rawtransactions",    "decodescript",             &decodescript,             {"hexstring"} },
+    { "rawtransactions",    "sendrawtransaction",       &sendrawtransaction,       {"hexstring","allowhighfees"} },
+    { "rawtransactions",    "combinerawtransaction",    &combinerawtransaction,    {"txs"} },
+    { "rawtransactions",    "signrawtransaction",       &signrawtransaction,       {"hexstring","prevtxs","privkeys","sighashtype"} }, /* uses wallet if enabled */
+    { "rawtransactions",    "extractvotefromsignature", &extractvotefromsignature, {"hexstring"}},
+
+    { "blockchain",         "gettxoutproof",            &gettxoutproof,          {"txids", "blockhash"} },
+    { "blockchain",         "verifytxoutproof",         &verifytxoutproof,       {"proof"} },
 };
 
 void RegisterRawTransactionRPCCommands(CRPCTable &t)

src/rpc/server.cpp

@@ -116,6 +116,23 @@ CAmount AmountFromValue(const UniValue& value)
     return amount;
 }
 
+uint160 ParseHash160V(const UniValue& v, std::string strName)
+{
+    std::string strHex;
+    if (v.isStr())
+        strHex = v.get_str();
+    if (!IsHex(strHex)) // Note: IsHex("") is false
+        throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')");
+    if (40 != strHex.length())
+        throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d)", strName, 40, strHex.length()));
+    uint160 result;
+    result.SetHex(strHex);
+    return result;
+}
+uint160 ParseHash160O(const UniValue& o, std::string strKey)
+{
+    return ParseHash160V(find_value(o, strKey), strKey);
+}
 uint256 ParseHashV(const UniValue& v, std::string strName)
 {
     std::string strHex;