Merge pull request stratisproject#34 from quantumagi/kvmigrate

Add Migrate method for KeyValueStore

src/Stratis.Bitcoin.Features.Consensus.Tests/ProvenBlockHeaders/ProvenBlockHeaderStoreTests.cs

@@ -38,7 +38,7 @@ public ProvenBlockHeaderStoreTests() : base(new StratisTest())
             ibdMock.Setup(s => s.IsInitialBlockDownload()).Returns(false);
 
             var folder = CreateTestDir(this);
-            var store = new ProvenBlockHeaderKeyValueStore(this.Network, new DataFolder(folder), this.LoggerFactory.Object, new DateTimeProvider());
+            var store = new ProvenBlockHeaderKeyValueStore(this.Network, new DataFolder(folder), this.LoggerFactory.Object, new DateTimeProvider(), repositorySerializer);
             this.provenBlockHeaderRepository = new ProvenBlockHeaderRepository(store, this.Network, folder, this.LoggerFactory.Object, repositorySerializer);
 
             this.provenBlockHeaderStore = new ProvenBlockHeaderStore(DateTimeProvider.Default, this.LoggerFactory.Object, this.provenBlockHeaderRepository, nodeStats, ibdMock.Object);

src/Stratis.Bitcoin.IntegrationTests.Common/ReadyData/ReadyBlockchain.cs

@@ -1,5 +1,4 @@
-
-namespace Stratis.Bitcoin.IntegrationTests.Common.ReadyData
+namespace Stratis.Bitcoin.IntegrationTests.Common.ReadyData
 {
     /// <summary>
     /// A list of paths to readily available blockchain data in zipped folders.
@@ -41,5 +40,10 @@ public class ReadyBlockchain
 
         public static string StratisXMainnet15K = @"ReadyData/StratisXOver15K.zip";
         public static string StratisMainnet9500 = @"ReadyData/StratisMain9500.zip";
+
+        public static string[] StratisRegTestAll = {StratisRegTest10Miner, StratisRegTest10Listener, StratisRegTest10NoWallet, StratisRegTest100Miner, StratisRegTest100Listener,
+            StratisRegTest100NoWallet, StratisRegTest150Miner, StratisRegTest150Listener, StratisRegTest150NoWallet };
+        public static string[] BitcoinRegTestAll = {BitcoinRegTest10Miner, BitcoinRegTest10Listener, BitcoinRegTest10NoWallet, BitcoinRegTest100Miner, BitcoinRegTest100Listener,
+            BitcoinRegTest100NoWallet, BitcoinRegTest150Miner, BitcoinRegTest150Listener, BitcoinRegTest150NoWallet };
     }
 }

src/Stratis.Bitcoin.IntegrationTests/ReadyBlockChainTest.cs

@@ -0,0 +1,64 @@
+using System.IO;
+using System.IO.Compression;
+using NBitcoin;
+using Stratis.Bitcoin.Configuration;
+using Stratis.Bitcoin.IntegrationTests.Common.ReadyData;
+using Stratis.Bitcoin.NodeStorage;
+using Stratis.Bitcoin.Tests.Common;
+using Stratis.Bitcoin.Utilities;
+using Xunit;
+
+namespace Stratis.Bitcoin.IntegrationTests
+{
+    internal static class NormalizeDirectorySeparatorExt
+    {
+        /// <summary>
+        /// Fixes incorrect directory separator characters in path (if any).
+        /// </summary>
+        public static string NormalizeDirectorySeparator(this string path)
+        {
+            // Replace incorrect with correct
+            return path.Replace((Path.DirectorySeparatorChar == '/') ? '\\' : '/', Path.DirectorySeparatorChar);
+        }
+    }
+
+    public class ReadyBlockChainTest
+    {
+        private void MigrateStores(Network network, string readyDataName)
+        {
+            byte[] TxIndexKey = new byte[1];
+
+            string temp = Path.GetTempPath();
+            string dir = Path.Combine(temp, readyDataName.Substring(0, readyDataName.LastIndexOf('.')));
+            string workDir = dir.Replace("ReadyData", "ReadyDataLevelDB");
+            string zipTarget = Path.Combine(temp, readyDataName.Replace("ReadyData", "ReadyDataLevelDB"));
+            string typeName = network.IsBitcoin() ? "bitcoin" : "stratis";
+
+            Directory.CreateDirectory("ReadyDataLevelDB");
+
+            ZipFile.ExtractToDirectory(Path.GetFullPath(readyDataName), dir, true);
+
+            var dataFolderDBZ = new DataFolder(Path.Combine(dir, typeName, network.Name));
+            var dataFolderLDB = new DataFolder(Path.Combine(workDir, typeName, network.Name));
+
+            (new Migrate()).MigrateKeyValueStore<KeyValueStoreDBreeze.KeyValueStoreDBreeze, KeyValueStoreLevelDB.KeyValueStoreLevelDB>(network, dataFolderDBZ, dataFolderLDB);
+
+            if (File.Exists(zipTarget))
+                File.Delete(zipTarget);
+
+            ZipFile.CreateFromDirectory(workDir, zipTarget);
+        }
+
+        [Fact(Skip="Run this manually when needed")]
+        public void MigrateFromDBreezeToLevelDb()
+        {
+            foreach (string readyDataName in ReadyBlockchain.StratisRegTestAll)
+                this.MigrateStores(KnownNetworks.StratisRegTest, readyDataName.NormalizeDirectorySeparator());
+
+            foreach (string readyDataName in ReadyBlockchain.BitcoinRegTestAll)
+                this.MigrateStores(KnownNetworks.RegTest, readyDataName.NormalizeDirectorySeparator());
+
+            this.MigrateStores(KnownNetworks.StratisMain, ReadyBlockchain.StratisMainnet9500.NormalizeDirectorySeparator());
+        }
+    }
+}

src/Stratis.Bitcoin/NodeStorage/Migrate.cs

@@ -0,0 +1,106 @@
+using System;
+using System.IO;
+using Microsoft.Extensions.Logging;
+using NBitcoin;
+using Stratis.Bitcoin.Configuration;
+using Stratis.Bitcoin.Interfaces;
+using Stratis.Bitcoin.KeyValueStore;
+using Stratis.Bitcoin.Utilities;
+
+namespace Stratis.Bitcoin.NodeStorage
+{
+    public class Migrate
+    {
+        public void MigrateKeyValueStore<TFrom, TTo>(Network network, DataFolder sourceDataFolder, DataFolder targetDataFolder) where TFrom : KeyValueStoreRepository where TTo : KeyValueStoreRepository
+        {
+            // Copy Block Store.
+            using (var blockStoreSource = new KeyValueStore<TFrom>(sourceDataFolder.BlockPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+            {
+                using (var blockStoreTarget = new KeyValueStore<TTo>(targetDataFolder.BlockPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+                {
+                    CopyTable<byte[], byte[]>(blockStoreSource, blockStoreTarget, "Block");
+                    CopyTable<byte[], byte[]>(blockStoreSource, blockStoreTarget, "Transaction");
+                    CopyTable<byte[], byte[]>(blockStoreSource, blockStoreTarget, "Common", (tableName, from, to) =>
+                    {
+                        byte[] txIndexKey = new byte[1];
+
+                        if (from.Select(tableName, txIndexKey, out bool txIndex))
+                            to.Insert(tableName, txIndexKey, txIndex);
+                    });
+                }
+            }
+
+            // Copy Chain Repository.
+            using (var chainRepoSource = new KeyValueStore<TFrom>(sourceDataFolder.ChainPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+            {
+                using (var chainRepoTarget = new KeyValueStore<TTo>(targetDataFolder.ChainPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+                {
+                    // Primitive types must be used.
+                    CopyTable<int, byte[]>(chainRepoSource, chainRepoTarget, "Chain");
+                }
+            }
+
+            // Copy CoinView.
+            using (var coinViewSource = new KeyValueStore<TFrom>(sourceDataFolder.CoinViewPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+            {
+                using (var coinViewTarget = new KeyValueStore<TTo>(targetDataFolder.CoinViewPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+                {
+                    CopyTable<byte[], byte[]>(coinViewSource, coinViewTarget, "Coins");
+                    // Primitive types must be used.
+                    CopyTable<int, byte[]>(coinViewSource, coinViewTarget, "Rewind");
+                    CopyTable<byte[], byte[]>(coinViewSource, coinViewTarget, "Stake");
+                    CopyTable<byte[], byte[]>(coinViewSource, coinViewTarget, "BlockHash");
+                }
+            }
+
+            // Copy ProvenBlockHeader.
+            using (var provenSource = new KeyValueStore<TFrom>(sourceDataFolder.ProvenBlockHeaderPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+            {
+                using (var provenTarget = new KeyValueStore<TTo>(targetDataFolder.ProvenBlockHeaderPath, new LoggerFactory(), DateTimeProvider.Default, new RepositorySerializer(network.Consensus.ConsensusFactory)))
+                {
+                    // Primitive types must be used.
+                    CopyTable<int, byte[]>(provenSource, provenTarget, "ProvenBlockHeader");
+                    CopyTable<byte[], byte[]>(provenSource, provenTarget, "BlockHashHeight");
+                }
+            }
+
+            if (Directory.Exists(Path.Combine(sourceDataFolder.RootPath, "finalizedBlock")))
+            {
+                string targetFolder = Path.Combine(targetDataFolder.RootPath, "finalizedBlock");
+                if (Directory.Exists(targetFolder))
+                    Directory.Delete(targetFolder, true);
+                Directory.CreateDirectory(targetFolder);
+                foreach (string file in Directory.EnumerateFiles(Path.Combine(sourceDataFolder.RootPath, "finalizedBlock"), "*.*", SearchOption.TopDirectoryOnly))
+                    File.Copy(file, file.Replace("ReadyData", "ReadyDataLevelDB"));
+            }
+
+            foreach (string file in Directory.EnumerateFiles(sourceDataFolder.RootPath, "*.*", SearchOption.TopDirectoryOnly))
+            {
+                string targetName = file.Replace("ReadyData", "ReadyDataLevelDB");
+                if (File.Exists(targetName))
+                    File.Delete(targetName);
+                File.Copy(file, targetName);
+            }
+        }
+
+        private static void CopyTable<K, V>(IKeyValueStore keyValueStoreFrom, IKeyValueStore keyValueStoreTo, string tableName,
+            Action<string, IKeyValueStoreTransaction, IKeyValueStoreTransaction> action = null)
+        {
+            using (IKeyValueStoreTransaction tranFrom = keyValueStoreFrom.CreateTransaction(KeyValueStoreTransactionMode.Read))
+            {
+                using (IKeyValueStoreTransaction tranTo = keyValueStoreTo.CreateTransaction(KeyValueStoreTransactionMode.ReadWrite))
+                {
+                    foreach ((K key, V value) in tranFrom.SelectForward<K, V>(tableName))
+                    {
+                        tranTo.Insert(tableName, key, value);
+                    }
+
+                    // Copy primitives explicitly using the correct types.
+                    action?.Invoke(tableName, tranFrom, tranTo);
+
+                    tranTo.Commit();
+                }
+            }
+        }
+    }
+}