Merge #6579: chore: drop legacy CDeterministicMNState formats, evodb upgrade logic

39f76c8 chore: drop legacy `CDeterministicMNState` formats, evodb upgrade logic (Kittywhiskers Van Gogh)

Pull request description:

  ## Additional Information

  * `CDeterministicMNState_`{`Oldformat`, `mntype_format`} was introduced in [dash#5039](#5039) and [dash#5403](#5403), included in Dash Core v19 and v20 respectively.

  The above change has been in the codebase for more than two major releases and does not concern databases with a long shelf life (like wallets), this pull request removes them entirely.

  ## Breaking Changes

  Users upgrading from these versions are now expected to `-reindex`.

  ## Checklist:

  - [x] I have performed a self-review of my own code
  - [x] I have commented my code, particularly in hard-to-understand areas **(note: N/A)**
  - [x] I have added or updated relevant unit/integration/functional/e2e tests **(note: N/A)**
  - [x] I have made corresponding changes to the documentation **(note: N/A)**
  - [x] I have assigned this pull request to a milestone _(for repository code-owners and collaborators only)_

ACKs for top commit:
  UdjinM6:
    utACK 39f76c8
  knst:
    utACK 39f76c8

Tree-SHA512: 3fabe0aeb7f3d01a858b0b19d49a22523f462157597a1ade93181288e4ba7b133a5ee9272a8abe41e40cff177c0ddf94123274f0427693db6fdb90c4b8e613fa

Author: PastaPastaPasta

doc/release-notes-6579.md

@@ -0,0 +1,5 @@
+Notable Changes
+---------------
+
+* Dash Core will no longer migrate EvoDb databases generated in v19 and v20, users upgrading
+  from these versions are recommended to run `-reindex` to rebuild all databases and indexes.

src/evo/deterministicmns.cpp

@@ -1199,238 +1199,6 @@ void CDeterministicMNManager::CleanupCache(int nHeight)
     return erased;
 }
 
-bool CDeterministicMNManager::MigrateDBIfNeeded()
-{
-    static const std::string DB_OLD_LIST_SNAPSHOT = "dmn_S";
-    static const std::string DB_OLD_LIST_DIFF = "dmn_D";
-    static const std::string DB_OLD_BEST_BLOCK = "b_b2";
-    static const std::string DB_OLD_BEST_BLOCK2 = "b_b3";
-    const auto& consensusParams = Params().GetConsensus();
-
-    LOCK(cs_main);
-
-    LogPrintf("CDeterministicMNManager::%s -- upgrading DB to migrate MN type\n", __func__);
-
-    if (m_chainstate.m_chain.Tip() == nullptr) {
-        // should have no records
-        LogPrintf("CDeterministicMNManager::%s -- Chain empty. evoDB:%d.\n", __func__, m_evoDb.IsEmpty());
-        return m_evoDb.IsEmpty();
-    }
-
-    if (m_evoDb.GetRawDB().Exists(EVODB_BEST_BLOCK) || m_evoDb.GetRawDB().Exists(DB_OLD_BEST_BLOCK2)) {
-        if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) {
-            // we messed up, make sure this time we actually drop old data
-            LogPrintf("CDeterministicMNManager::%s -- migration already done. cleaned old data.\n", __func__);
-            m_evoDb.GetRawDB().CompactFull();
-            LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__);
-            // flush it to disk
-            if (!m_evoDb.CommitRootTransaction()) {
-                LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-                return false;
-            }
-        } else {
-            LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__);
-        }
-        return true;
-    }
-
-    // Removing the old EVODB_BEST_BLOCK value early results in older version to crash immediately, even if the upgrade
-    // process is cancelled in-between. But if the new version sees that the old EVODB_BEST_BLOCK is already removed,
-    // then we must assume that the upgrade process was already running before but was interrupted.
-    if (m_chainstate.m_chain.Height() > 1 && !m_evoDb.GetRawDB().Exists(DB_OLD_BEST_BLOCK)) {
-        LogPrintf("CDeterministicMNManager::%s -- previous migration attempt failed.\n", __func__);
-        return false;
-    }
-    m_evoDb.GetRawDB().Erase(DB_OLD_BEST_BLOCK);
-
-    if (!DeploymentActiveAt(*m_chainstate.m_chain.Tip(), consensusParams, Consensus::DEPLOYMENT_DIP0003)) {
-        // not reached DIP3 height yet, so no upgrade needed
-        LogPrintf("CDeterministicMNManager::%s -- migration not needed. dip3 not reached\n", __func__);
-        auto dbTx = m_evoDb.BeginTransaction();
-        m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
-        dbTx->Commit();
-        if (!m_evoDb.CommitRootTransaction()) {
-            LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-            return false;
-        }
-        return true;
-    }
-
-    if (DeploymentActiveAt(*m_chainstate.m_chain.Tip(), consensusParams, Consensus::DEPLOYMENT_V19)) {
-        // too late
-        LogPrintf("CDeterministicMNManager::%s -- migration is not possible\n", __func__);
-        return false;
-    }
-
-
-    CDBBatch batch(m_evoDb.GetRawDB());
-
-    for (const auto nHeight : irange::range(Params().GetConsensus().DIP0003Height, m_chainstate.m_chain.Height() + 1)) {
-        auto pindex = m_chainstate.m_chain[nHeight];
-        // Unserialise CDeterministicMNListDiff using MN_OLD_FORMAT and set it's type to the default value TYPE_REGULAR_MASTERNODE
-        // It will be later written with format MN_CURRENT_FORMAT which includes the type field and MN state bls version.
-        CDataStream diff_data(SER_DISK, CLIENT_VERSION);
-        if (!m_evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_OLD_LIST_DIFF, pindex->GetBlockHash()), diff_data)) {
-            LogPrintf("CDeterministicMNManager::%s -- missing CDeterministicMNListDiff at height %d\n", __func__, nHeight);
-            return false;
-        }
-        CDeterministicMNListDiff mndiff;
-        mndiff.Unserialize(diff_data, CDeterministicMN::MN_OLD_FORMAT);
-        batch.Write(std::make_pair(DB_LIST_DIFF, pindex->GetBlockHash()), mndiff);
-        CDataStream snapshot_data(SER_DISK, CLIENT_VERSION);
-        if (!m_evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_OLD_LIST_SNAPSHOT, pindex->GetBlockHash()), snapshot_data)) {
-            // it's ok, we write snapshots every DISK_SNAPSHOT_PERIOD blocks only
-            continue;
-        }
-        CDeterministicMNList mnList;
-        mnList.Unserialize(snapshot_data, CDeterministicMN::MN_OLD_FORMAT);
-        batch.Write(std::make_pair(DB_LIST_SNAPSHOT, pindex->GetBlockHash()), mnList);
-        m_evoDb.GetRawDB().WriteBatch(batch);
-        batch.Clear();
-        LogPrintf("CDeterministicMNManager::%s -- wrote snapshot at height %d\n", __func__, nHeight);
-    }
-
-    m_evoDb.GetRawDB().WriteBatch(batch);
-
-    // Writing EVODB_BEST_BLOCK (which is b_b4 now) marks the DB as upgraded
-    auto dbTx = m_evoDb.BeginTransaction();
-    m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
-    dbTx->Commit();
-
-    LogPrintf("CDeterministicMNManager::%s -- done migrating\n", __func__);
-
-    if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) {
-        LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__);
-    }
-
-    m_evoDb.GetRawDB().CompactFull();
-
-    LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__);
-
-    // flush it to disk
-    if (!m_evoDb.CommitRootTransaction()) {
-        LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-        return false;
-    }
-
-    return true;
-}
-
-bool CDeterministicMNManager::MigrateDBIfNeeded2()
-{
-    static const std::string DB_OLD_LIST_SNAPSHOT = "dmn_S2";
-    static const std::string DB_OLD_LIST_DIFF = "dmn_D2";
-    static const std::string DB_OLD_BEST_BLOCK = "b_b3";
-    const auto& consensusParams = Params().GetConsensus();
-
-    LOCK(cs_main);
-
-    LogPrintf("CDeterministicMNManager::%s -- upgrading DB to migrate MN state bls version\n", __func__);
-
-    if (m_chainstate.m_chain.Tip() == nullptr) {
-        // should have no records
-        LogPrintf("CDeterministicMNManager::%s -- Chain empty. evoDB:%d.\n", __func__, m_evoDb.IsEmpty());
-        return m_evoDb.IsEmpty();
-    }
-
-    if (m_evoDb.GetRawDB().Exists(EVODB_BEST_BLOCK)) {
-        if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) {
-            // we messed up, make sure this time we actually drop old data
-            LogPrintf("CDeterministicMNManager::%s -- migration already done. cleaned old data.\n", __func__);
-            m_evoDb.GetRawDB().CompactFull();
-            LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__);
-            // flush it to disk
-            if (!m_evoDb.CommitRootTransaction()) {
-                LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-                return false;
-            }
-        } else {
-            LogPrintf("CDeterministicMNManager::%s -- migration already done. skipping.\n", __func__);
-        }
-        return true;
-    }
-
-    // Removing the old EVODB_BEST_BLOCK value early results in older version to crash immediately, even if the upgrade
-    // process is cancelled in-between. But if the new version sees that the old EVODB_BEST_BLOCK is already removed,
-    // then we must assume that the upgrade process was already running before but was interrupted.
-    if (m_chainstate.m_chain.Height() > 1 && !m_evoDb.GetRawDB().Exists(DB_OLD_BEST_BLOCK)) {
-        LogPrintf("CDeterministicMNManager::%s -- previous migration attempt failed.\n", __func__);
-        return false;
-    }
-    m_evoDb.GetRawDB().Erase(DB_OLD_BEST_BLOCK);
-
-    if (!DeploymentActiveAt(*m_chainstate.m_chain.Tip(), consensusParams, Consensus::DEPLOYMENT_DIP0003)) {
-        // not reached DIP3 height yet, so no upgrade needed
-        LogPrintf("CDeterministicMNManager::%s -- migration not needed. dip3 not reached\n", __func__);
-        auto dbTx = m_evoDb.BeginTransaction();
-        m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
-        dbTx->Commit();
-        if (!m_evoDb.CommitRootTransaction()) {
-            LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-            return false;
-        }
-        return true;
-    }
-
-    if (DeploymentActiveAt(*m_chainstate.m_chain.Tip(), consensusParams, Consensus::DEPLOYMENT_V19)) {
-        // too late
-        LogPrintf("CDeterministicMNManager::%s -- migration is not possible\n", __func__);
-        return false;
-    }
-
-    CDBBatch batch(m_evoDb.GetRawDB());
-
-    for (const auto nHeight : irange::range(Params().GetConsensus().DIP0003Height, m_chainstate.m_chain.Height() + 1)) {
-        auto pindex = m_chainstate.m_chain[nHeight];
-        // Unserialise CDeterministicMNListDiff using MN_TYPE_FORMAT and set MN state bls version to LEGACY_BLS_VERSION.
-        // It will be later written with format MN_CURRENT_FORMAT which includes the type field.
-        CDataStream diff_data(SER_DISK, CLIENT_VERSION);
-        if (!m_evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_OLD_LIST_DIFF, pindex->GetBlockHash()), diff_data)) {
-            LogPrintf("CDeterministicMNManager::%s -- missing CDeterministicMNListDiff at height %d\n", __func__, nHeight);
-            return false;
-        }
-        CDeterministicMNListDiff mndiff;
-        mndiff.Unserialize(diff_data, CDeterministicMN::MN_TYPE_FORMAT);
-        batch.Write(std::make_pair(DB_LIST_DIFF, pindex->GetBlockHash()), mndiff);
-        CDataStream snapshot_data(SER_DISK, CLIENT_VERSION);
-        if (!m_evoDb.GetRawDB().ReadDataStream(std::make_pair(DB_OLD_LIST_SNAPSHOT, pindex->GetBlockHash()), snapshot_data)) {
-            // it's ok, we write snapshots every DISK_SNAPSHOT_PERIOD blocks only
-            continue;
-        }
-        CDeterministicMNList mnList;
-        mnList.Unserialize(snapshot_data, CDeterministicMN::MN_TYPE_FORMAT);
-        batch.Write(std::make_pair(DB_LIST_SNAPSHOT, pindex->GetBlockHash()), mnList);
-        m_evoDb.GetRawDB().WriteBatch(batch);
-        batch.Clear();
-        LogPrintf("CDeterministicMNManager::%s -- wrote snapshot at height %d\n", __func__, nHeight);
-    }
-
-    m_evoDb.GetRawDB().WriteBatch(batch);
-
-    // Writing EVODB_BEST_BLOCK (which is b_b4 now) marks the DB as upgraded
-    auto dbTx = m_evoDb.BeginTransaction();
-    m_evoDb.WriteBestBlock(m_chainstate.m_chain.Tip()->GetBlockHash());
-    dbTx->Commit();
-
-    LogPrintf("CDeterministicMNManager::%s -- done migrating\n", __func__);
-
-    if (EraseOldDBData(m_evoDb.GetRawDB(), {DB_OLD_LIST_DIFF, DB_OLD_LIST_SNAPSHOT})) {
-        LogPrintf("CDeterministicMNManager::%s -- done cleaning old data\n", __func__);
-    }
-
-    m_evoDb.GetRawDB().CompactFull();
-
-    LogPrintf("CDeterministicMNManager::%s -- done compacting database\n", __func__);
-
-    // flush it to disk
-    if (!m_evoDb.CommitRootTransaction()) {
-        LogPrintf("CDeterministicMNManager::%s -- failed to commit to evoDB\n", __func__);
-        return false;
-    }
-
-    return true;
-}
-
 template <typename ProTx>
 static bool CheckService(const ProTx& proTx, TxValidationState& state)
 {

src/evo/deterministicmns.h

@@ -46,8 +46,6 @@ class CDeterministicMN
     uint64_t internalId{std::numeric_limits<uint64_t>::max()};
 
 public:
-    static constexpr uint16_t MN_OLD_FORMAT = 0;
-    static constexpr uint16_t MN_TYPE_FORMAT = 1;
     static constexpr uint16_t MN_VERSION_FORMAT = 2;
     static constexpr uint16_t MN_CURRENT_FORMAT = MN_VERSION_FORMAT;
 
@@ -75,29 +73,19 @@ class CDeterministicMN
     template <typename Stream, typename Operation>
     inline void SerializationOp(Stream& s, Operation ser_action, const uint8_t format_version)
     {
+        // We no longer support EvoDB formats below MN_VERSION_FORMAT
+        if (format_version < MN_VERSION_FORMAT) {
+            throw std::ios_base::failure("EvoDb too old, run Dash Core with -reindex to rebuild");
+        }
         READWRITE(proTxHash);
         READWRITE(VARINT(internalId));
         READWRITE(collateralOutpoint);
         READWRITE(nOperatorReward);
-        // We need to read CDeterministicMNState using the old format only when called with MN_OLD_FORMAT or MN_TYPE_FORMAT on Unserialize()
-        // Serialisation (writing) will be done always using new format
-        if (ser_action.ForRead() && format_version == MN_OLD_FORMAT) {
-            CDeterministicMNState_Oldformat old_state;
-            READWRITE(old_state);
-            pdmnState = std::make_shared<const CDeterministicMNState>(old_state);
-        } else if (ser_action.ForRead() && format_version == MN_TYPE_FORMAT) {
-            CDeterministicMNState_mntype_format old_state;
-            READWRITE(old_state);
-            pdmnState = std::make_shared<const CDeterministicMNState>(old_state);
-        } else {
-            READWRITE(pdmnState);
-        }
-        // We need to read/write nType if:
-        // format_version is set to MN_TYPE_FORMAT (For writing (serialisation) it is always the case) Needed for the MNLISTDIFF Migration in evoDB
+        READWRITE(pdmnState);
         // We can't know if we are serialising for the Disk or for the Network here (s.GetType() is not accessible)
         // Therefore if s.GetVersion() == CLIENT_VERSION -> Then we know we are serialising for the Disk
         // Otherwise, we can safely check with protocol versioning logic so we won't break old clients
-        if (format_version >= MN_TYPE_FORMAT && (s.GetVersion() == CLIENT_VERSION || s.GetVersion() >= DMN_TYPE_PROTO_VERSION)) {
+        if (s.GetVersion() == CLIENT_VERSION || s.GetVersion() >= DMN_TYPE_PROTO_VERSION) {
             READWRITE(nType);
         } else {
             nType = MnType::Regular;
@@ -221,15 +209,9 @@ class CDeterministicMNList
 
         SerializationOpBase(s, CSerActionUnserialize());
 
-        bool evodb_migration = (format_version == CDeterministicMN::MN_OLD_FORMAT || format_version == CDeterministicMN::MN_TYPE_FORMAT);
         size_t cnt = ReadCompactSize(s);
         for (size_t i = 0; i < cnt; i++) {
-            if (evodb_migration) {
-                const auto dmn = std::make_shared<CDeterministicMN>(deserialize, s, format_version);
-                mnMap = mnMap.set(dmn->proTxHash, dmn);
-            } else {
-                AddMN(std::make_shared<CDeterministicMN>(deserialize, s, format_version), false);
-            }
+            AddMN(std::make_shared<CDeterministicMN>(deserialize, s, format_version), false);
         }
     }
 
@@ -620,9 +602,6 @@ class CDeterministicMNManager
     // Test if given TX is a ProRegTx which also contains the collateral at index n
     static bool IsProTxWithCollateral(const CTransactionRef& tx, uint32_t n);
 
-    bool MigrateDBIfNeeded();
-    bool MigrateDBIfNeeded2();
-
     void DoMaintenance() EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
 private: