merge bitcoin#24236: Remove utxo db upgrade code

Author: kwvg

ci/test/00_setup_env_native_qt5.sh

@@ -14,5 +14,5 @@ export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude fe
 export RUN_UNIT_TESTS_SEQUENTIAL="true"
 export RUN_UNIT_TESTS="false"
 export GOAL="install"
-export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.0.0 v0.16.1.1 v0.17.0.3 v18.2.2 v19.3.0 v20.0.1"
+export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.12.1.5 v0.15.0.0 v0.16.1.1 v0.17.0.3 v18.2.2 v19.3.0 v20.0.1"
 export BITCOIN_CONFIG="--enable-zmq --with-libs=no --enable-reduce-exports --disable-fuzz-binary LDFLAGS=-static-libstdc++ --with-boost-process"

src/init.cpp

@@ -2037,8 +2037,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
                 strLoadError = _("Error initializing block database");
                 break;
             case ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED:
-                strLoadError = _("Error upgrading chainstate database");
-                break;
+                return InitError(_("Unsupported chainstate database format found. "
+                                   "Please restart with -reindex-chainstate. This will "
+                                   "rebuild the chainstate database."));
             case ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED:
                 strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.");
                 break;

src/node/chainstate.cpp

@@ -156,9 +156,9 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset,
             chainstate->CoinsErrorCatcher().AddReadErrCallback(coins_error_cb);
         }
 
-        // If necessary, upgrade from older database format.
+        // Refuse to load unsupported database format.
         // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate
-        if (!chainstate->CoinsDB().Upgrade()) {
+        if (chainstate->CoinsDB().NeedsUpgrade()) {
             return ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED;
         }
 

src/txdb.cpp

@@ -6,7 +6,6 @@
 #include <txdb.h>
 
 #include <chain.h>
-#include <node/interface_ui.h>
 #include <pow.h>
 #include <random.h>
 #include <shutdown.h>
@@ -18,7 +17,6 @@
 #include <stdint.h>
 
 static constexpr uint8_t DB_COIN{'C'};
-static constexpr uint8_t DB_COINS{'c'};
 static constexpr uint8_t DB_BLOCK_FILES{'f'};
 static constexpr uint8_t DB_ADDRESSINDEX{'a'};
 static constexpr uint8_t DB_ADDRESSUNSPENTINDEX{'u'};
@@ -33,6 +31,7 @@ static constexpr uint8_t DB_REINDEX_FLAG{'R'};
 static constexpr uint8_t DB_LAST_BLOCK{'l'};
 
 // Keys used in previous version that might still be found in the DB:
+static constexpr uint8_t DB_COINS{'c'};
 static constexpr uint8_t DB_TXINDEX_BLOCK{'T'};
 //               uint8_t DB_TXINDEX{'t'}
 
@@ -54,6 +53,15 @@ std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db)
     return std::nullopt;
 }
 
+bool CCoinsViewDB::NeedsUpgrade()
+{
+    std::unique_ptr<CDBIterator> cursor{m_db->NewIterator()};
+    // DB_COINS was deprecated in v0.15.0, commit
+    // 1088b02f0ccd7358d2b7076bb9e122d59d502d02
+    cursor->Seek(std::make_pair(DB_COINS, uint256{}));
+    return cursor->Valid();
+}
+
 namespace {
 
 struct CoinEntry {
@@ -64,7 +72,7 @@ struct CoinEntry {
     SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); }
 };
 
-}
+} // namespace
 
 CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, bool fWipe) :
     m_db(std::make_unique<CDBWrapper>(ldb_path, nCacheSize, fMemory, fWipe, true)),
@@ -473,125 +481,3 @@ bool CBlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams,
 
     return true;
 }
-
-namespace {
-
-//! Legacy class to deserialize pre-pertxout database entries without reindex.
-class CCoins
-{
-public:
-    //! whether transaction is a coinbase
-    bool fCoinBase;
-
-    //! unspent transaction outputs; spent outputs are .IsNull(); spent outputs at the end of the array are dropped
-    std::vector<CTxOut> vout;
-
-    //! at which height this transaction was included in the active block chain
-    int nHeight;
-
-    //! empty constructor
-    CCoins() : fCoinBase(false), vout(0), nHeight(0) { }
-
-    template<typename Stream>
-    void Unserialize(Stream &s) {
-        unsigned int nCode = 0;
-        // version
-        unsigned int nVersionDummy;
-        ::Unserialize(s, VARINT(nVersionDummy));
-        // header code
-        ::Unserialize(s, VARINT(nCode));
-        fCoinBase = nCode & 1;
-        std::vector<bool> vAvail(2, false);
-        vAvail[0] = (nCode & 2) != 0;
-        vAvail[1] = (nCode & 4) != 0;
-        unsigned int nMaskCode = (nCode / 8) + ((nCode & 6) != 0 ? 0 : 1);
-        // spentness bitmask
-        while (nMaskCode > 0) {
-            unsigned char chAvail = 0;
-            ::Unserialize(s, chAvail);
-            for (unsigned int p = 0; p < 8; p++) {
-                bool f = (chAvail & (1 << p)) != 0;
-                vAvail.push_back(f);
-            }
-            if (chAvail != 0)
-                nMaskCode--;
-        }
-        // txouts themself
-        vout.assign(vAvail.size(), CTxOut());
-        for (unsigned int i = 0; i < vAvail.size(); i++) {
-            if (vAvail[i])
-                ::Unserialize(s, Using<TxOutCompression>(vout[i]));
-        }
-        // coinbase height
-        ::Unserialize(s, VARINT_MODE(nHeight, VarIntMode::NONNEGATIVE_SIGNED));
-    }
-};
-
-}
-
-/** Upgrade the database from older formats.
- *
- * Currently implemented: from the per-tx utxo model (0.8..0.14.x) to per-txout.
- */
-bool CCoinsViewDB::Upgrade() {
-    std::unique_ptr<CDBIterator> pcursor(m_db->NewIterator());
-    pcursor->Seek(std::make_pair(DB_COINS, uint256()));
-    if (!pcursor->Valid()) {
-        return true;
-    }
-
-    int64_t count = 0;
-    LogPrintf("Upgrading utxo-set database...\n");
-    LogPrintf("[0%%]..."); /* Continued */
-    uiInterface.ShowProgress(_("Upgrading UTXO database").translated, 0, true);
-    size_t batch_size = 1 << 24;
-    CDBBatch batch(*m_db);
-    int reportDone = 0;
-    std::pair<unsigned char, uint256> key;
-    std::pair<unsigned char, uint256> prev_key = {DB_COINS, uint256()};
-    while (pcursor->Valid()) {
-        if (ShutdownRequested()) {
-            break;
-        }
-        if (pcursor->GetKey(key) && key.first == DB_COINS) {
-            if (count++ % 256 == 0) {
-                uint32_t high = 0x100 * *key.second.begin() + *(key.second.begin() + 1);
-                int percentageDone = (int)(high * 100.0 / 65536.0 + 0.5);
-                uiInterface.ShowProgress(_("Upgrading UTXO database").translated, percentageDone, true);
-                if (reportDone < percentageDone/10) {
-                    // report max. every 10% step
-                    LogPrintf("[%d%%]...", percentageDone); /* Continued */
-                    reportDone = percentageDone/10;
-                }
-            }
-            CCoins old_coins;
-            if (!pcursor->GetValue(old_coins)) {
-                return error("%s: cannot parse CCoins record", __func__);
-            }
-            COutPoint outpoint(key.second, 0);
-            for (size_t i = 0; i < old_coins.vout.size(); ++i) {
-                if (!old_coins.vout[i].IsNull() && !old_coins.vout[i].scriptPubKey.IsUnspendable()) {
-                    Coin newcoin(std::move(old_coins.vout[i]), old_coins.nHeight, old_coins.fCoinBase);
-                    outpoint.n = i;
-                    CoinEntry entry(&outpoint);
-                    batch.Write(entry, newcoin);
-                }
-            }
-            batch.Erase(key);
-            if (batch.SizeEstimate() > batch_size) {
-                m_db->WriteBatch(batch);
-                batch.Clear();
-                m_db->CompactRange(prev_key, key);
-                prev_key = key;
-            }
-            pcursor->Next();
-        } else {
-            break;
-        }
-    }
-    m_db->WriteBatch(batch);
-    m_db->CompactRange({DB_COINS, uint256()}, key);
-    uiInterface.ShowProgress("", 100, false);
-    LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE");
-    return !ShutdownRequested();
-}

src/txdb.h

@@ -68,8 +68,8 @@ class CCoinsViewDB final : public CCoinsView
     bool BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock, bool erase = true) override;
     std::unique_ptr<CCoinsViewCursor> Cursor() const override;
 
-    //! Attempt to update from an older database format. Returns whether an error occurred.
-    bool Upgrade();
+    //! Whether an unsupported database format is used.
+    bool NeedsUpgrade();
     size_t EstimateSize() const override;
 
     //! Dynamically alter the underlying leveldb cache size.

test/README.md

@@ -103,7 +103,7 @@ test/functional/test_runner.py --extended
 In order to run backwards compatibility tests, download the previous node binaries:
 
 ```
-test/get_previous_releases.py -b v19.3.0 v18.2.2 v0.17.0.3 v0.16.1.1 v0.15.0.0
+test/get_previous_releases.py -b v19.3.0 v18.2.2 v0.17.0.3 v0.16.1.1 v0.15.0.0 v0.12.1.5
 ```
 
 By default, up to 4 tests will be run in parallel by test_runner. To specify

test/functional/feature_unsupported_utxo_db.py

@@ -0,0 +1,62 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test that unsupported utxo db causes an init error.
+
+Previous releases are required by this test, see test/README.md.
+"""
+
+import shutil
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+class UnsupportedUtxoDbTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_no_previous_releases()
+
+    def setup_network(self):
+        self.add_nodes(
+            self.num_nodes,
+            versions=[
+                120105,  # Last release with previous utxo db format
+                None,    # For MiniWallet, without migration code
+            ],
+        )
+
+    def run_test(self):
+        self.log.info("Create previous version (v0.12.1.5) utxo db")
+        self.start_node(0)
+        # 'generatetoaddress' was introduced in v0.12.3, use legacy 'generate' call
+        block = self.nodes[0].rpc.generate(1)[-1]
+        assert_equal(self.nodes[0].getbestblockhash(), block)
+        assert_equal(self.nodes[0].gettxoutsetinfo()["total_amount"], 500)
+        self.stop_nodes()
+
+        self.log.info("Check init error")
+        legacy_utxos_dir = self.nodes[0].chain_path / "chainstate"
+        legacy_blocks_dir = self.nodes[0].chain_path / "blocks"
+        recent_utxos_dir = self.nodes[1].chain_path / "chainstate"
+        recent_blocks_dir = self.nodes[1].chain_path / "blocks"
+        shutil.copytree(legacy_utxos_dir, recent_utxos_dir)
+        shutil.copytree(legacy_blocks_dir, recent_blocks_dir)
+        self.nodes[1].assert_start_raises_init_error(
+            expected_msg="Error: Unsupported chainstate database format found. "
+            "Please restart with -reindex-chainstate. "
+            "This will rebuild the chainstate database.",
+        )
+
+        self.log.info("Drop legacy utxo db")
+        self.start_node(1, extra_args=["-reindex-chainstate", "-txindex=0"])
+        assert_equal(self.nodes[1].getbestblockhash(), block)
+        assert_equal(self.nodes[1].gettxoutsetinfo()["total_amount"], 500)
+
+
+if __name__ == "__main__":
+    UnsupportedUtxoDbTest().main()

test/functional/test_runner.py

@@ -368,6 +368,7 @@
     'p2p_sendtxrcncl.py',
     'rpc_scantxoutset.py',
     'feature_txindex_compatibility.py',
+    'feature_unsupported_utxo_db.py',
     'feature_logging.py',
     'feature_anchors.py',
     'feature_coinstatsindex.py',

test/get_previous_releases.py

@@ -19,7 +19,6 @@
 import sys
 import hashlib
 
-
 SHA256_SUMS = {
     "d1f7121a7d7bdd4077709284076860389d6a0f4481a934ad9acb85cae3d7b83e":  "dashcore-20.0.1-aarch64-linux-gnu.tar.gz",
     "37375229e5ab18d7050b729fb016df24acdd72d60bc3fa074270d89030a27827":  "dashcore-20.0.1-arm-linux-gnueabihf.tar.gz",
@@ -92,6 +91,15 @@
     "f532bc7e0360e80908eb6b9c3aeec7e0037e70e25dee3b040dbbf7a124e05619":  "dashcore-0.15.0.0-win64-setup.exe",
     "3ba6ff98113af30319fb1499d132d993633380476f9980443d630d21a40e0efb":  "dashcore-0.15.0.0-win64.zip",
     "4cc0815ebd595f3d0134a8df9e6224cbe3d79398a5a899b60ca5f4ab8a576160":  "dashcore-0.15.0.0-x86_64-linux-gnu.tar.gz",
+    #
+    "060c86587176ffcea5615ea46aa080152ea5ebd035d2a7e0340d312345c2eee0":  "dashcore-0.12.1.5-RPi2.tar.gz",
+    "369dee3e2ae8854592ca696fd5fd7314bc8bbdd865920407bc96d59acb06f81d":  "dashcore-0.12.1.5-linux32.tar.gz",
+    "81692699b99d64b50d1bb1db427f6a0d68c84b98d0e0152f854cfbe048c08fc4":  "dashcore-0.12.1.5-linux64.tar.gz",
+    "b4514d4a705cc1adb400ec0c69630612fe394508decd7bf3edef068021fc47b5":  "dashcore-0.12.1.5-osx.dmg",
+    "35eb7bd9f5883d986b32ae84ddd074566c9e01a438bddd0d147c4a1e96b81b02":  "dashcore-0.12.1.5-win32-setup.exe",
+    "2ea20e8671dcae6ed4f980b8b7dcaa1024961e0267956f4e8bd9eea31b0030e4":  "dashcore-0.12.1.5-win32.zip",
+    "db276b7697777203e98f93e1f498bdab0016f2a96f6454a56c51846120aae0b7":  "dashcore-0.12.1.5-win64-setup.exe",
+    "15ee0d8405609aaf70c31ff320b0468c86e7c6bc1f061ae90d6295a540576c39":  "dashcore-0.12.1.5-win64.zip",
 }