diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs deleted file mode 100644 index 26e37e5e274..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerOutpointsRepositoryTests.cs +++ /dev/null @@ -1,95 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Runtime.InteropServices; -using LiteDB; -using NBitcoin; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; -using Xunit; - -namespace Stratis.Bitcoin.Features.BlockStore.Tests -{ - public class AddressIndexerOutpointsRepositoryTests - { - private readonly AddressIndexerOutpointsRepository repository; - - private readonly Random random = new Random(); - - private readonly int maxItems = 10; - - public AddressIndexerOutpointsRepositoryTests() - { - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - var db = new LiteDatabase(new ConnectionString() { Filename = this.RandomString(20) + ".litedb", Mode = fileMode }); - - this.repository = new AddressIndexerOutpointsRepository(db, new ExtendedLoggerFactory(), this.maxItems); - } - - [Fact] - public void LoadPercentageCalculatedCorrectly() - { - for (int i = 0; i < this.maxItems / 2; i++) - this.repository.AddOutPointData(new OutPointData() { Outpoint = this.RandomString(20) }); - - Assert.Equal(50, this.repository.GetLoadPercentage()); - } - - [Fact] - public void CanAddAndRemoveOutpointData() - { - var outPoint = new OutPoint(new uint256(RandomUtils.GetUInt64()), 1); - - var data = new OutPointData() { Outpoint = outPoint.ToString(), Money = 1, ScriptPubKeyBytes = RandomUtils.GetBytes(20) }; - this.repository.AddOutPointData(data); - - // Add more to trigger eviction. - for (int i = 0; i < this.maxItems * 2; i++) - this.repository.AddOutPointData(new OutPointData() { Outpoint = this.RandomString(20) }); - - Assert.True(this.repository.TryGetOutPointData(outPoint, out OutPointData dataOut)); - Assert.True(data.ScriptPubKeyBytes.SequenceEqual(dataOut.ScriptPubKeyBytes)); - } - - [Fact] - public void CanRewind() - { - var rewindDataBlockHash = new uint256(RandomUtils.GetUInt64()); - - var outPoint = new OutPoint(new uint256(RandomUtils.GetUInt64()), 1); - var data = new OutPointData() { Outpoint = outPoint.ToString(), Money = 1, ScriptPubKeyBytes = RandomUtils.GetBytes(20) }; - - var rewindData = new AddressIndexerRewindData() - { - BlockHash = rewindDataBlockHash.ToString(), - BlockHeight = 100, - SpentOutputs = new List() { data } - }; - - this.repository.RecordRewindData(rewindData); - - Assert.False(this.repository.TryGetOutPointData(outPoint, out OutPointData dataOut)); - - this.repository.RewindDataAboveHeight(rewindData.BlockHeight - 1); - - Assert.True(this.repository.TryGetOutPointData(outPoint, out dataOut)); - - // Now record and purge rewind data. - this.repository.RecordRewindData(rewindData); - - this.repository.RemoveOutPointData(outPoint); - Assert.False(this.repository.TryGetOutPointData(outPoint, out dataOut)); - - this.repository.PurgeOldRewindData(rewindData.BlockHeight + 1); - - Assert.False(this.repository.TryGetOutPointData(outPoint, out dataOut)); - } - - private string RandomString(int length) - { - const string chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; - return new string(Enumerable.Repeat(chars, length) - .Select(s => s[this.random.Next(s.Length)]).ToArray()); - } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs deleted file mode 100644 index 4611de7ea17..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/AddressIndexerTests.cs +++ /dev/null @@ -1,381 +0,0 @@ -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using LiteDB; -using Moq; -using NBitcoin; -using Stratis.Bitcoin.AsyncWork; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Consensus; -using Stratis.Bitcoin.Controllers.Models; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; -using Stratis.Bitcoin.Networks; -using Stratis.Bitcoin.Primitives; -using Stratis.Bitcoin.Tests.Common; -using Stratis.Bitcoin.Utilities; -using Xunit; -using FileMode = LiteDB.FileMode; -using Script = NBitcoin.Script; - -namespace Stratis.Bitcoin.Features.BlockStore.Tests -{ - public class AddressIndexerTests - { - private readonly IAddressIndexer addressIndexer; - - private readonly Mock consensusManagerMock; - - private readonly Mock asyncProviderMock; - - private readonly Network network; - - private readonly ChainedHeader genesisHeader; - - public AddressIndexerTests() - { - this.network = new StratisMain(); - var storeSettings = new StoreSettings(NodeSettings.Default(this.network)); - - storeSettings.AddressIndex = true; - storeSettings.TxIndex = true; - - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - var stats = new Mock(); - var indexer = new ChainIndexer(this.network); - - this.consensusManagerMock = new Mock(); - - this.asyncProviderMock = new Mock(); - - this.addressIndexer = new AddressIndexer(storeSettings, dataFolder, new ExtendedLoggerFactory(), this.network, stats.Object, - this.consensusManagerMock.Object, this.asyncProviderMock.Object, indexer, new DateTimeProvider()); - - this.genesisHeader = new ChainedHeader(this.network.GetGenesis().Header, this.network.GetGenesis().Header.GetHash(), 0); - } - - [Fact] - public void CanInitializeAndDispose() - { - this.consensusManagerMock.Setup(x => x.Tip).Returns(() => this.genesisHeader); - - this.addressIndexer.Initialize(); - this.addressIndexer.Dispose(); - } - - [Fact] - public void CanIndexAddresses() - { - List headers = ChainedHeadersHelper.CreateConsecutiveHeaders(100, null, false, null, this.network); - this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headers.Last()); - - Script p2pk1 = this.GetRandomP2PKScript(out string address1); - Script p2pk2 = this.GetRandomP2PKScript(out string address2); - - var block1 = new Block() - { - Transactions = new List() - { - new Transaction() - { - Outputs = - { - new TxOut(new Money(10_000), p2pk1), - new TxOut(new Money(20_000), p2pk1), - new TxOut(new Money(30_000), p2pk1) - } - } - } - }; - - var block5 = new Block() - { - Transactions = new List() - { - new Transaction() - { - Outputs = - { - new TxOut(new Money(10_000), p2pk1), - new TxOut(new Money(1_000), p2pk2), - new TxOut(new Money(1_000), p2pk2) - } - } - } - }; - - var tx = new Transaction(); - tx.Inputs.Add(new TxIn(new OutPoint(block5.Transactions.First().GetHash(), 0))); - var block10 = new Block() { Transactions = new List() { tx } }; - - this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny())).Returns((uint256 hash) => - { - ChainedHeader header = headers.SingleOrDefault(x => x.HashBlock == hash); - - switch (header?.Height) - { - case 1: - return new ChainedHeaderBlock(block1, header); - - case 5: - return new ChainedHeaderBlock(block5, header); - - case 10: - return new ChainedHeaderBlock(block10, header); - } - - return new ChainedHeaderBlock(new Block(), header); - }); - - this.addressIndexer.Initialize(); - - TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headers.Last()); - - Assert.Equal(60_000, this.addressIndexer.GetAddressBalances(new[] { address1 }).Balances.First().Balance.Satoshi); - Assert.Equal(2_000, this.addressIndexer.GetAddressBalances(new[] { address2 }).Balances.First().Balance.Satoshi); - - Assert.Equal(70_000, this.addressIndexer.GetAddressBalances(new[] { address1 }, 93).Balances.First().Balance.Satoshi); - - // Now trigger rewind to see if indexer can handle reorgs. - ChainedHeader forkPoint = headers.Single(x => x.Height == 8); - - List headersFork = ChainedHeadersHelper.CreateConsecutiveHeaders(100, forkPoint, false, null, this.network); - - this.consensusManagerMock.Setup(x => x.GetBlockData(It.IsAny())).Returns((uint256 hash) => - { - ChainedHeader headerFork = headersFork.SingleOrDefault(x => x.HashBlock == hash); - - return new ChainedHeaderBlock(new Block(), headerFork); - }); - - this.consensusManagerMock.Setup(x => x.Tip).Returns(() => headersFork.Last()); - TestBase.WaitLoop(() => this.addressIndexer.IndexerTip == headersFork.Last()); - - Assert.Equal(70_000, this.addressIndexer.GetAddressBalances(new[] { address1 }).Balances.First().Balance.Satoshi); - - this.addressIndexer.Dispose(); - } - - private Script GetRandomP2PKScript(out string address) - { - var bytes = RandomUtils.GetBytes(33); - bytes[0] = 0x02; - - Script script = new Script() + Op.GetPushOp(bytes) + OpcodeType.OP_CHECKSIG; - - PubKey[] destinationKeys = script.GetDestinationPublicKeys(this.network); - address = destinationKeys[0].GetAddress(this.network).ToString(); - - return script; - } - - [Fact] - public void OutPointCacheCanRetrieveExisting() - { - const string CollectionName = "DummyCollection"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); - - var outPoint = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 0); - - var data = new OutPointData() { Outpoint = outPoint.ToString(), ScriptPubKeyBytes = new byte[] { 0, 0, 0, 0 }, Money = Money.Coins(1) }; - - cache.AddOutPointData(data); - - Assert.True(cache.TryGetOutPointData(outPoint, out OutPointData retrieved)); - - Assert.NotNull(retrieved); - Assert.Equal(outPoint.ToString(), retrieved.Outpoint); - } - - [Fact] - public void OutPointCacheCannotRetrieveNonexistent() - { - const string CollectionName = "DummyCollection"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory()); - - Assert.False(cache.TryGetOutPointData(new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1), out OutPointData retrieved)); - Assert.Null(retrieved); - } - - [Fact] - public void OutPointCacheEvicts() - { - const string CollectionName = "OutputsData"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexerOutpointsRepository(database, new ExtendedLoggerFactory(), 2); - - Assert.Equal(0, cache.Count); - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - var outPoint1 = new OutPoint(uint256.Parse("0000af9ab2c8660481328d0444cf167dfd31f24ca2dbba8e5e963a2434cffa93"), 1); ; - var pair1 = new OutPointData() { Outpoint = outPoint1.ToString(), ScriptPubKeyBytes = new byte[] { 0, 0, 0, 0 }, Money = Money.Coins(1) }; - - cache.AddOutPointData(pair1); - - Assert.Equal(1, cache.Count); - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - var outPoint2 = new OutPoint(uint256.Parse("cf8ce1419bbc4870b7d4f1c084534d91126dd3283b51ec379e0a20e27bd23633"), 2); ; - var pair2 = new OutPointData() { Outpoint = outPoint2.ToString(), ScriptPubKeyBytes = new byte[] { 1, 1, 1, 1 }, Money = Money.Coins(2) }; - - cache.AddOutPointData(pair2); - - Assert.Equal(2, cache.Count); - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - var outPoint3 = new OutPoint(uint256.Parse("126dd3283b51ec379e0a20e27bd23633cf8ce1419bbc4870b7d4f1c084534d91"), 3); ; - var pair3 = new OutPointData() { Outpoint = outPoint3.ToString(), ScriptPubKeyBytes = new byte[] { 2, 2, 2, 2 }, Money = Money.Coins(3) }; - - cache.AddOutPointData(pair3); - - Assert.Equal(2, cache.Count); - - // One of the cache items should have been evicted, and will therefore be persisted on disk. - Assert.Equal(1, database.GetCollection(CollectionName).Count()); - - // The evicted item should be pair1. - Assert.Equal(pair1.ScriptPubKeyBytes, database.GetCollection(CollectionName).FindAll().First().ScriptPubKeyBytes); - - // It should still be possible to retrieve pair1 from the cache (it will pull it from disk). - Assert.True(cache.TryGetOutPointData(outPoint1, out OutPointData pair1AfterEviction)); - - Assert.NotNull(pair1AfterEviction); - Assert.Equal(pair1.ScriptPubKeyBytes, pair1AfterEviction.ScriptPubKeyBytes); - Assert.Equal(pair1.Money, pair1AfterEviction.Money); - } - - [Fact] - public void AddressCacheCanRetrieveExisting() - { - const string CollectionName = "DummyCollection"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); - - string address = "xyz"; - var balanceChanges = new List(); - - balanceChanges.Add(new AddressBalanceChange() { BalanceChangedHeight = 1, Deposited = true, Satoshi = 1 }); - - var data = new AddressIndexerData() { Address = address, BalanceChanges = balanceChanges }; - - cache.AddOrUpdate(data.Address, data, data.BalanceChanges.Count + 1); - - AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz"); - - Assert.NotNull(retrieved); - Assert.Equal("xyz", retrieved.Address); - Assert.Equal(1, retrieved.BalanceChanges.First().BalanceChangedHeight); - Assert.True(retrieved.BalanceChanges.First().Deposited); - Assert.Equal(1, retrieved.BalanceChanges.First().Satoshi); - } - - [Fact] - public void AddressCacheRetrievesBlankRecordForNonexistent() - { - const string CollectionName = "DummyCollection"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory()); - - AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz"); - - // A record will be returned with no balance changes associated, if it is new. - Assert.NotNull(retrieved); - Assert.Equal("xyz", retrieved.Address); - Assert.Empty(retrieved.BalanceChanges); - } - - [Fact] - public void AddressCacheEvicts() - { - const string CollectionName = "AddrData"; - var dataFolder = new DataFolder(TestBase.CreateTestDir(this)); - string dbPath = Path.Combine(dataFolder.RootPath, CollectionName); - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - - var database = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - var cache = new AddressIndexRepository(database, new ExtendedLoggerFactory(), 4); - - // Recall, each index entry counts as 1 and each balance change associated with it is an additional 1. - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - string address1 = "xyz"; - var balanceChanges1 = new List(); - balanceChanges1.Add(new AddressBalanceChange() { BalanceChangedHeight = 1, Deposited = true, Satoshi = 1 }); - var data1 = new AddressIndexerData() { Address = address1, BalanceChanges = balanceChanges1 }; - - cache.AddOrUpdate(data1.Address, data1, data1.BalanceChanges.Count + 1); - - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - string address2 = "abc"; - var balanceChanges2 = new List(); - balanceChanges2.Add(new AddressBalanceChange() { BalanceChangedHeight = 2, Deposited = false, Satoshi = 2 }); - - cache.AddOrUpdate(address2, new AddressIndexerData() { Address = address2, BalanceChanges = balanceChanges2 }, balanceChanges2.Count + 1); - - Assert.Equal(0, database.GetCollection(CollectionName).Count()); - - string address3 = "def"; - var balanceChanges3 = new List(); - balanceChanges3.Add(new AddressBalanceChange() { BalanceChangedHeight = 3, Deposited = true, Satoshi = 3 }); - cache.AddOrUpdate(address3, new AddressIndexerData() { Address = address3, BalanceChanges = balanceChanges3 }, balanceChanges3.Count + 1); - - // One of the cache items should have been evicted, and will therefore be persisted on disk. - Assert.Equal(1, database.GetCollection(CollectionName).Count()); - - // The evicted item should be data1. - Assert.Equal(data1.Address, database.GetCollection(CollectionName).FindAll().First().Address); - Assert.Equal(1, database.GetCollection(CollectionName).FindAll().First().BalanceChanges.First().BalanceChangedHeight); - Assert.True(database.GetCollection(CollectionName).FindAll().First().BalanceChanges.First().Deposited); - Assert.Equal(1, database.GetCollection(CollectionName).FindAll().First().BalanceChanges.First().Satoshi); - - // Check that the first address can still be retrieved, it should come from disk in this case. - AddressIndexerData retrieved = cache.GetOrCreateAddress("xyz"); - - Assert.NotNull(retrieved); - Assert.Equal("xyz", retrieved.Address); - Assert.Equal(1, retrieved.BalanceChanges.First().BalanceChangedHeight); - Assert.True(retrieved.BalanceChanges.First().Deposited); - Assert.Equal(1, retrieved.BalanceChanges.First().Satoshi); - } - - [Fact] - public void MaxReorgIsCalculatedProperly() - { - var btc = new BitcoinMain(); - - int maxReorgBtc = AddressIndexer.GetMaxReorgOrFallbackMaxReorg(btc); - - Assert.Equal(maxReorgBtc, AddressIndexer.FallBackMaxReorg); - - var stratis = new StratisMain(); - - int maxReorgStratis = AddressIndexer.GetMaxReorgOrFallbackMaxReorg(stratis); - - Assert.Equal(maxReorgStratis, (int)stratis.Consensus.MaxReorgLength); - } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs index e48be533717..2ba3e76e6b1 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore.Tests/BlockStoreControllerTests.cs @@ -7,7 +7,6 @@ using NBitcoin; using Stratis.Bitcoin.Base; using Stratis.Bitcoin.Controllers.Models; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.BlockStore.Controllers; using Stratis.Bitcoin.Features.BlockStore.Models; using Stratis.Bitcoin.Interfaces; @@ -161,7 +160,6 @@ public void GetBlockCount_ReturnsHeightFromChainState() var logger = new Mock(); var store = new Mock(); var chainState = new Mock(); - var addressIndexer = new Mock(); ChainIndexer chainIndexer = WalletTestsHelpers.GenerateChainWithHeight(3, KnownNetworks.StratisTest); @@ -170,7 +168,7 @@ public void GetBlockCount_ReturnsHeightFromChainState() chainState.Setup(c => c.ConsensusTip) .Returns(chainIndexer.GetHeader(2)); - var controller = new BlockStoreController(KnownNetworks.StratisTest, logger.Object, store.Object, chainState.Object, chainIndexer, addressIndexer.Object); + var controller = new BlockStoreController(KnownNetworks.StratisTest, logger.Object, store.Object, chainState.Object, chainIndexer); var json = (JsonResult)controller.GetBlockCount(); int result = int.Parse(json.Value.ToString()); @@ -183,7 +181,6 @@ private static (Mock store, BlockStoreController controller) GetCon var logger = new Mock(); var store = new Mock(); var chainState = new Mock(); - var addressIndexer = new Mock(); logger.Setup(l => l.CreateLogger(It.IsAny())).Returns(Mock.Of); @@ -192,7 +189,7 @@ private static (Mock store, BlockStoreController controller) GetCon chain.Setup(c => c.GetHeader(It.IsAny())).Returns(new ChainedHeader(block.Header, block.Header.GetHash(), 1)); chain.Setup(x => x.Tip).Returns(new ChainedHeader(block.Header, block.Header.GetHash(), 1)); - var controller = new BlockStoreController(KnownNetworks.StratisTest, logger.Object, store.Object, chainState.Object, chain.Object, addressIndexer.Object); + var controller = new BlockStoreController(KnownNetworks.StratisTest, logger.Object, store.Object, chainState.Object, chain.Object); return (store, controller); } diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs deleted file mode 100644 index a202e8e4e63..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexRepository.cs +++ /dev/null @@ -1,90 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using LiteDB; -using Microsoft.Extensions.Logging; -using Stratis.Bitcoin.Controllers.Models; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing -{ - /// Repository for items with cache layer built in. - public class AddressIndexRepository : MemorySizeCache - { - private const string DbAddressDataKey = "AddrData"; - - private readonly LiteCollection addressIndexerDataCollection; - - private readonly ILogger logger; - - public AddressIndexRepository(LiteDatabase db, ILoggerFactory loggerFactory, int maxBalanceChangesToKeep = 50_000) : base(maxBalanceChangesToKeep) - { - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - this.addressIndexerDataCollection = db.GetCollection(DbAddressDataKey); - this.addressIndexerDataCollection.EnsureIndex("BalanceChangedHeightIndex", "$.BalanceChanges[*].BalanceChangedHeight", false); - } - - /// Retrieves address data, either the cached version if it exists, or directly from the underlying database. - /// If it is a previously unseen address an empty record will be created and added to the cache. - /// The address to retrieve data for. - public AddressIndexerData GetOrCreateAddress(string address) - { - if (!this.TryGetValue(address, out AddressIndexerData data)) - { - this.logger.LogDebug("Not found in cache."); - data = this.addressIndexerDataCollection.FindById(address) ?? new AddressIndexerData() { Address = address, BalanceChanges = new List() }; - } - - int size = 1 + data.BalanceChanges.Count / 10; - this.AddOrUpdate(address, data, size); - - return data; - } - - public double GetLoadPercentage() - { - return Math.Round(this.totalSize / (this.MaxSize / 100.0), 2); - } - - /// - /// Checks for addresses that are affected by balance changes above a given block height. - /// This method should only be relied upon for block heights lower than the consensus tip and higher - /// than (tip - maxReorg). This is because it is only used while reorging the address indexer. - /// - /// The block height above which balance changes should be considered. - /// A list of affected addresses containing balance changes above the specified block height. - public List GetAddressesHigherThanHeight(int height) - { - this.SaveAllItems(); - - // Need to specify index name explicitly so that it gets used for the query. - IEnumerable affectedAddresses = this.addressIndexerDataCollection.Find(Query.GT("BalanceChangedHeightIndex", height)); - - // Per LiteDb documentation: - // "Returning an IEnumerable your code still connected to datafile. - // Only when you finish consume all data, datafile will be disconnected." - return affectedAddresses.Select(x => x.Address).ToList(); - } - - /// - protected override void ItemRemovedLocked(CacheItem item) - { - base.ItemRemovedLocked(item); - - if (item.Dirty) - this.addressIndexerDataCollection.Upsert(item.Value); - } - - public void SaveAllItems() - { - lock (this.LockObject) - { - CacheItem[] dirtyItems = this.Keys.Where(x => x.Dirty).ToArray(); - this.addressIndexerDataCollection.Upsert(dirtyItems.Select(x => x.Value)); - - foreach (CacheItem dirtyItem in dirtyItems) - dirtyItem.Dirty = false; - } - } - } -} \ No newline at end of file diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs deleted file mode 100644 index 1f280ab996a..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexer.cs +++ /dev/null @@ -1,659 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using System.Linq; -using System.Runtime.InteropServices; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using LiteDB; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.AsyncWork; -using Stratis.Bitcoin.Configuration; -using Stratis.Bitcoin.Configuration.Logging; -using Stratis.Bitcoin.Consensus; -using Stratis.Bitcoin.Controllers.Models; -using Stratis.Bitcoin.Interfaces; -using Stratis.Bitcoin.Primitives; -using Stratis.Bitcoin.Utilities; -using FileMode = LiteDB.FileMode; -using Script = NBitcoin.Script; - -namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing -{ - /// Component that builds an index of all addresses and deposits\withdrawals that happened to\from them. - public interface IAddressIndexer : IDisposable - { - ChainedHeader IndexerTip { get; } - - void Initialize(); - - /// Returns balance of the given address confirmed with at least confirmations. - /// The set of addresses that will be queried. - /// Only blocks below consensus tip less this parameter will be considered. - /// Balance of a given address or null if address wasn't indexed or doesn't exists. - AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 0); - - /// Returns verbose balances data. - /// The set of addresses that will be queried. - VerboseAddressBalancesResult GetAddressIndexerState(string[] addresses); - } - - public class AddressIndexer : IAddressIndexer - { - public ChainedHeader IndexerTip { get; private set; } - - private readonly StoreSettings storeSettings; - - private readonly Network network; - - private readonly INodeStats nodeStats; - - private readonly ILogger logger; - - private readonly DataFolder dataFolder; - - private readonly IConsensusManager consensusManager; - - private readonly IAsyncProvider asyncProvider; - - private readonly IScriptAddressReader scriptAddressReader; - - private readonly TimeSpan flushChangesInterval; - - private const string DbTipDataKey = "AddrTipData"; - - private const string AddressIndexerDatabaseFilename = "addressindex.litedb"; - - /// Max supported reorganization length for networks without max reorg property. - public const int FallBackMaxReorg = 200; - - /// - /// Time to wait before attempting to index the next block. - /// Waiting happens after a failure to get next block to index. - /// - private const int DelayTimeMs = 2000; - - private const int CompactingThreshold = 50; - - /// Max distance between consensus and indexer tip to consider indexer synced. - private const int ConsiderSyncedMaxDistance = 10; - - private LiteDatabase db; - - private LiteCollection tipDataStore; - - /// A mapping between addresses and their balance changes. - /// All access should be protected by . - private AddressIndexRepository addressIndexRepository; - - /// Script pub keys and amounts mapped by outpoints. - /// All access should be protected by . - private AddressIndexerOutpointsRepository outpointsRepository; - - /// Protects access to and . - private readonly object lockObject; - - private readonly CancellationTokenSource cancellation; - - private readonly ILoggerFactory loggerFactory; - - private readonly ChainIndexer chainIndexer; - - private readonly AverageCalculator averageTimePerBlock; - - private readonly IDateTimeProvider dateTimeProvider; - - private Task indexingTask; - - private DateTime lastFlushTime; - - private const int PurgeIntervalSeconds = 60; - - /// Last time rewind data was purged. - private DateTime lastPurgeTime; - - private Task prefetchingTask; - - /// Indexer height at the last save. - /// Should be protected by . - private int lastSavedHeight; - - /// Distance in blocks from consensus tip at which compaction should start. - /// It can't be lower than maxReorg since compacted data can't be converted back to uncompacted state for partial reversion. - private readonly int compactionTriggerDistance; - - /// - /// This is a window of some blocks that is needed to reduce the consequences of nodes having different view of consensus chain. - /// We assume that nodes usually don't have view that is different from other nodes by that constant of blocks. - /// - public const int SyncBuffer = 50; - - public AddressIndexer(StoreSettings storeSettings, DataFolder dataFolder, ILoggerFactory loggerFactory, Network network, INodeStats nodeStats, - IConsensusManager consensusManager, IAsyncProvider asyncProvider, ChainIndexer chainIndexer, IDateTimeProvider dateTimeProvider) - { - this.storeSettings = storeSettings; - this.network = network; - this.nodeStats = nodeStats; - this.dataFolder = dataFolder; - this.consensusManager = consensusManager; - this.asyncProvider = asyncProvider; - this.dateTimeProvider = dateTimeProvider; - this.loggerFactory = loggerFactory; - this.scriptAddressReader = new ScriptAddressReader(); - - this.lockObject = new object(); - this.flushChangesInterval = TimeSpan.FromMinutes(2); - this.lastFlushTime = this.dateTimeProvider.GetUtcNow(); - this.cancellation = new CancellationTokenSource(); - this.chainIndexer = chainIndexer; - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - - this.averageTimePerBlock = new AverageCalculator(200); - int maxReorgLength = GetMaxReorgOrFallbackMaxReorg(this.network); - - this.compactionTriggerDistance = maxReorgLength * 2 + SyncBuffer + 1000; - } - - /// Returns maxReorg of in case maxReorg is 0. - public static int GetMaxReorgOrFallbackMaxReorg(Network network) - { - int maxReorgLength = network.Consensus.MaxReorgLength == 0 ? FallBackMaxReorg : (int)network.Consensus.MaxReorgLength; - - return maxReorgLength; - } - - public void Initialize() - { - // The transaction index is needed in the event of a reorg. - if (!this.storeSettings.AddressIndex) - { - this.logger.LogTrace("(-)[DISABLED]"); - return; - } - - string dbPath = Path.Combine(this.dataFolder.RootPath, AddressIndexerDatabaseFilename); - - FileMode fileMode = RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? FileMode.Exclusive : FileMode.Shared; - this.db = new LiteDatabase(new ConnectionString() { Filename = dbPath, Mode = fileMode }); - - this.addressIndexRepository = new AddressIndexRepository(this.db, this.loggerFactory); - - this.logger.LogDebug("Address indexing is enabled."); - - this.tipDataStore = this.db.GetCollection(DbTipDataKey); - - lock (this.lockObject) - { - AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault(); - - this.logger.LogDebug("Tip data: '{0}'.", tipData == null ? "null" : tipData.ToString()); - - this.IndexerTip = tipData == null ? this.chainIndexer.Genesis : this.consensusManager.Tip.FindAncestorOrSelf(new uint256(tipData.TipHashBytes)); - - if (this.IndexerTip == null) - { - // This can happen if block hash from tip data is no longer a part of the consensus chain and node was killed in the middle of a reorg. - int rewindAmount = this.compactionTriggerDistance / 2; - - if (rewindAmount > this.consensusManager.Tip.Height) - this.IndexerTip = this.chainIndexer.Genesis; - else - this.IndexerTip = this.consensusManager.Tip.GetAncestor(this.consensusManager.Tip.Height - rewindAmount); - } - } - - this.outpointsRepository = new AddressIndexerOutpointsRepository(this.db, this.loggerFactory); - - this.RewindAndSave(this.IndexerTip); - - this.logger.LogDebug("Indexer initialized at '{0}'.", this.IndexerTip); - - this.indexingTask = Task.Run(async () => await this.IndexAddressesContinuouslyAsync().ConfigureAwait(false)); - - this.asyncProvider.RegisterTask($"{nameof(AddressIndexer)}.{nameof(this.indexingTask)}", this.indexingTask); - - this.nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, this.GetType().Name, 400); - } - - private async Task IndexAddressesContinuouslyAsync() - { - var watch = Stopwatch.StartNew(); - - while (!this.cancellation.IsCancellationRequested) - { - if (this.dateTimeProvider.GetUtcNow() - this.lastFlushTime > this.flushChangesInterval) - { - this.logger.LogDebug("Flushing changes."); - - this.SaveAll(); - - this.lastFlushTime = this.dateTimeProvider.GetUtcNow(); - - this.logger.LogDebug("Flush completed."); - } - - if (this.cancellation.IsCancellationRequested) - break; - - ChainedHeader nextHeader = this.consensusManager.Tip.GetAncestor(this.IndexerTip.Height + 1); - - if (nextHeader == null) - { - this.logger.LogDebug("Next header wasn't found. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - if (nextHeader.Previous.HashBlock != this.IndexerTip.HashBlock) - { - ChainedHeader lastCommonHeader = nextHeader.FindFork(this.IndexerTip); - - this.logger.LogDebug("Reorganization detected. Rewinding till '{0}'.", lastCommonHeader); - - this.RewindAndSave(lastCommonHeader); - - continue; - } - - // First try to see if it's prefetched. - ChainedHeaderBlock prefetchedBlock = this.prefetchingTask == null ? null : await this.prefetchingTask.ConfigureAwait(false); - - Block blockToProcess; - - if (prefetchedBlock != null && prefetchedBlock.ChainedHeader == nextHeader) - blockToProcess = prefetchedBlock.Block; - else - blockToProcess = this.consensusManager.GetBlockData(nextHeader.HashBlock).Block; - - if (blockToProcess == null) - { - this.logger.LogDebug("Next block wasn't found. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - // Schedule prefetching of the next block; - ChainedHeader headerToPrefetch = this.consensusManager.Tip.GetAncestor(nextHeader.Height + 1); - - if (headerToPrefetch != null) - this.prefetchingTask = Task.Run(() => this.consensusManager.GetBlockData(headerToPrefetch.HashBlock)); - - watch.Restart(); - - bool success = this.ProcessBlock(blockToProcess, nextHeader); - - watch.Stop(); - this.averageTimePerBlock.AddSample(watch.Elapsed.TotalMilliseconds); - - if (!success) - { - this.logger.LogDebug("Failed to process next block. Waiting."); - - try - { - await Task.Delay(DelayTimeMs, this.cancellation.Token).ConfigureAwait(false); - } - catch (OperationCanceledException) - { - } - - continue; - } - - this.IndexerTip = nextHeader; - } - - this.SaveAll(); - } - - private void RewindAndSave(ChainedHeader rewindToHeader) - { - lock (this.lockObject) - { - // The cache doesn't really lend itself to handling a reorg very well. - // Therefore, we leverage LiteDb's indexing capabilities to tell us - // which records are for the affected blocks. - - List affectedAddresses = this.addressIndexRepository.GetAddressesHigherThanHeight(rewindToHeader.Height); - - foreach (string address in affectedAddresses) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - indexData.BalanceChanges.RemoveAll(x => x.BalanceChangedHeight > rewindToHeader.Height); - } - - this.logger.LogDebug("Rewinding changes for {0} addresses.", affectedAddresses.Count); - - // Rewind all the way back to the fork point. - this.outpointsRepository.RewindDataAboveHeight(rewindToHeader.Height); - - this.IndexerTip = rewindToHeader; - - this.SaveAll(); - } - } - - private void SaveAll() - { - this.logger.LogDebug("Saving address indexer."); - - lock (this.lockObject) - { - this.addressIndexRepository.SaveAllItems(); - this.outpointsRepository.SaveAllItems(); - - AddressIndexerTipData tipData = this.tipDataStore.FindAll().FirstOrDefault(); - - if (tipData == null) - tipData = new AddressIndexerTipData(); - - tipData.Height = this.IndexerTip.Height; - tipData.TipHashBytes = this.IndexerTip.HashBlock.ToBytes(); - - this.tipDataStore.Upsert(tipData); - this.lastSavedHeight = this.IndexerTip.Height; - } - - this.logger.LogDebug("Address indexer saved."); - } - - private void AddInlineStats(StringBuilder benchLog) - { - benchLog.AppendLine("AddressIndexer.Height: ".PadRight(LoggingConfiguration.ColumnLength + 1) + this.IndexerTip.Height.ToString().PadRight(9) + - "AddressCache%: " + this.addressIndexRepository.GetLoadPercentage().ToString().PadRight(8) + - "OutPointCache%: " + this.outpointsRepository.GetLoadPercentage().ToString().PadRight(8) + - $"Ms/block: {Math.Round(this.averageTimePerBlock.Average, 2)}"); - } - - /// Processes a block that was added or removed from the consensus chain. - /// The block to process. - /// The chained header associated to the block being processed. - /// true if block was sucessfully processed. - private bool ProcessBlock(Block block, ChainedHeader header) - { - lock (this.lockObject) - { - // Record outpoints. - foreach (Transaction tx in block.Transactions) - { - for (int i = 0; i < tx.Outputs.Count; i++) - { - // OP_RETURN outputs and empty outputs cannot be spent and therefore do not need to be put into the cache. - if (tx.Outputs[i].IsEmpty || tx.Outputs[i].ScriptPubKey.IsUnspendable) - continue; - - var outPoint = new OutPoint(tx, i); - - var outPointData = new OutPointData() - { - Outpoint = outPoint.ToString(), - ScriptPubKeyBytes = tx.Outputs[i].ScriptPubKey.ToBytes(), - Money = tx.Outputs[i].Value - }; - - // TODO: When the outpoint cache is full, adding outpoints singly causes overhead writing evicted entries out to the repository - this.outpointsRepository.AddOutPointData(outPointData); - } - } - } - - // Process inputs. - var inputs = new List(); - - // Collect all inputs excluding coinbases. - foreach (TxInList inputsCollection in block.Transactions.Where(x => !x.IsCoinBase).Select(x => x.Inputs)) - inputs.AddRange(inputsCollection); - - lock (this.lockObject) - { - var rewindData = new AddressIndexerRewindData() { BlockHash = header.HashBlock.ToString(), BlockHeight = header.Height, SpentOutputs = new List() }; - - foreach (TxIn input in inputs) - { - OutPoint consumedOutput = input.PrevOut; - - if (!this.outpointsRepository.TryGetOutPointData(consumedOutput, out OutPointData consumedOutputData)) - { - this.logger.LogError("Missing outpoint data for {0}.", consumedOutput); - this.logger.LogTrace("(-)[MISSING_OUTPOINTS_DATA]"); - throw new Exception($"Missing outpoint data for {consumedOutput}"); - } - - Money amountSpent = consumedOutputData.Money; - - rewindData.SpentOutputs.Add(consumedOutputData); - - // Transactions that don't actually change the balance just bloat the database. - if (amountSpent == 0) - continue; - - string address = this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, new Script(consumedOutputData.ScriptPubKeyBytes)); - - if (string.IsNullOrEmpty(address)) - { - // This condition need not be logged, as the address reader should be aware of all possible address formats already. - continue; - } - - this.ProcessBalanceChangeLocked(header.Height, address, amountSpent, false); - } - - // Process outputs. - foreach (Transaction tx in block.Transactions) - { - foreach (TxOut txOut in tx.Outputs) - { - Money amountReceived = txOut.Value; - - // Transactions that don't actually change the balance just bloat the database. - if (amountReceived == 0 || txOut.IsEmpty || txOut.ScriptPubKey.IsUnspendable) - continue; - - string address = this.scriptAddressReader.GetAddressFromScriptPubKey(this.network, txOut.ScriptPubKey); - - if (string.IsNullOrEmpty(address)) - { - // This condition need not be logged, as the address reader should be aware of all - // possible address formats already. - continue; - } - - this.ProcessBalanceChangeLocked(header.Height, address, amountReceived, true); - } - } - - this.outpointsRepository.RecordRewindData(rewindData); - - int purgeRewindDataThreshold = Math.Min(this.consensusManager.Tip.Height - this.compactionTriggerDistance, this.lastSavedHeight); - - if ((this.dateTimeProvider.GetUtcNow() - this.lastPurgeTime).TotalSeconds > PurgeIntervalSeconds) - { - this.outpointsRepository.PurgeOldRewindData(purgeRewindDataThreshold); - this.lastPurgeTime = this.dateTimeProvider.GetUtcNow(); - } - - // Remove outpoints that were consumed. - foreach (OutPoint consumedOutPoint in inputs.Select(x => x.PrevOut)) - this.outpointsRepository.RemoveOutPointData(consumedOutPoint); - } - - return true; - } - - /// Adds a new balance change entry to to the . - /// The height of the block this being processed. - /// The address receiving the funds. - /// The amount being received. - /// false if this is an output being spent, true otherwise. - /// Should be protected by . - private void ProcessBalanceChangeLocked(int height, string address, Money amount, bool deposited) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - // Record new balance change into the address index data. - indexData.BalanceChanges.Add(new AddressBalanceChange() - { - BalanceChangedHeight = height, - Satoshi = amount.Satoshi, - Deposited = deposited - }); - - // Anything less than that should be compacted. - int heightThreshold = this.consensusManager.Tip.Height - this.compactionTriggerDistance; - - bool compact = (indexData.BalanceChanges.Count > CompactingThreshold) && - (indexData.BalanceChanges[1].BalanceChangedHeight < heightThreshold); - - if (!compact) - { - this.logger.LogTrace("(-)[TOO_FEW_CHANGE_RECORDS]"); - return; - } - - var compacted = new List(CompactingThreshold / 2) - { - new AddressBalanceChange() - { - BalanceChangedHeight = 0, - Satoshi = 0, - Deposited = true - } - }; - - foreach (AddressBalanceChange change in indexData.BalanceChanges) - { - if (change.BalanceChangedHeight < heightThreshold) - { - this.logger.LogDebug("Balance change: {0} was selected for compaction. Compacted balance now: {1}.", change, compacted[0].Satoshi); - - if (change.Deposited) - compacted[0].Satoshi += change.Satoshi; - else - compacted[0].Satoshi -= change.Satoshi; - - this.logger.LogDebug("New compacted balance: {0}.", compacted[0].Satoshi); - } - else - compacted.Add(change); - } - - indexData.BalanceChanges = compacted; - this.addressIndexRepository.AddOrUpdate(indexData.Address, indexData, indexData.BalanceChanges.Count + 1); - } - - private bool IsSynced() - { - lock (this.lockObject) - { - return this.consensusManager.Tip.Height - this.IndexerTip.Height <= ConsiderSyncedMaxDistance; - } - } - - /// - /// This is currently not in use but will be required for exchange integration. - public AddressBalancesResult GetAddressBalances(string[] addresses, int minConfirmations = 1) - { - var (isQueryable, reason) = this.IsQueryable(); - - if (!isQueryable) - return AddressBalancesResult.RequestFailed(reason); - - var result = new AddressBalancesResult(); - - lock (this.lockObject) - { - foreach (var address in addresses) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - int maxAllowedHeight = this.consensusManager.Tip.Height - minConfirmations + 1; - - long balance = indexData.BalanceChanges.Where(x => x.BalanceChangedHeight <= maxAllowedHeight).CalculateBalance(); - - this.logger.LogDebug("Address: {0}, balance: {1}.", address, balance); - result.Balances.Add(new AddressBalanceResult(address, new Money(balance))); - } - - return result; - } - } - - /// - public VerboseAddressBalancesResult GetAddressIndexerState(string[] addresses) - { - var result = new VerboseAddressBalancesResult(this.consensusManager.Tip.Height); - - if (addresses.Length == 0) - return result; - - (bool isQueryable, string reason) = this.IsQueryable(); - - if (!isQueryable) - return VerboseAddressBalancesResult.RequestFailed(reason); - - lock (this.lockObject) - { - foreach (var address in addresses) - { - AddressIndexerData indexData = this.addressIndexRepository.GetOrCreateAddress(address); - - var copy = new AddressIndexerData() - { - Address = indexData.Address, - BalanceChanges = new List(indexData.BalanceChanges) - }; - - result.BalancesData.Add(copy); - } - } - - return result; - } - - private (bool isQueryable, string reason) IsQueryable() - { - if (this.addressIndexRepository == null) - { - this.logger.LogTrace("(-)[NOT_INITIALIZED]"); - return (false, "Address indexer is not initialized."); - } - - if (!this.IsSynced()) - { - this.logger.LogTrace("(-)[NOT_SYNCED]"); - return (false, "Address indexer is not synced."); - } - - return (true, string.Empty); - } - - /// - public void Dispose() - { - this.cancellation.Cancel(); - - this.indexingTask?.GetAwaiter().GetResult(); - - this.db?.Dispose(); - } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs deleted file mode 100644 index 73b7f93f318..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerOutpointsRepository.cs +++ /dev/null @@ -1,161 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using LiteDB; -using Microsoft.Extensions.Logging; -using NBitcoin; -using Stratis.Bitcoin.Utilities; - -namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing -{ - /// Repository for items with cache layer built in. - public sealed class AddressIndexerOutpointsRepository : MemoryCache - { - private const string DbOutputsDataKey = "OutputsData"; - - private const string DbOutputsRewindDataKey = "OutputsRewindData"; - - /// Represents the output collection. - /// Should be protected by - private readonly LiteCollection addressIndexerOutPointData; - - /// Represents the rewind data collection. - /// Should be protected by - private readonly LiteCollection addressIndexerRewindData; - - private readonly ILogger logger; - - private readonly int maxCacheItems; - - public AddressIndexerOutpointsRepository(LiteDatabase db, ILoggerFactory loggerFactory, int maxItems = 60_000) - { - this.logger = loggerFactory.CreateLogger(this.GetType().FullName); - this.addressIndexerOutPointData = db.GetCollection(DbOutputsDataKey); - this.addressIndexerRewindData = db.GetCollection(DbOutputsRewindDataKey); - this.maxCacheItems = maxItems; - } - - public double GetLoadPercentage() - { - return Math.Round(this.totalSize / (this.maxCacheItems / 100.0), 2); - } - - public void AddOutPointData(OutPointData outPointData) - { - this.AddOrUpdate(new CacheItem(outPointData.Outpoint, outPointData, 1)); - } - - public void RemoveOutPointData(OutPoint outPoint) - { - lock (this.LockObject) - { - if (this.Cache.TryGetValue(outPoint.ToString(), out LinkedListNode node)) - { - this.Cache.Remove(node.Value.Key); - this.Keys.Remove(node); - this.totalSize -= 1; - } - - if (!node.Value.Dirty) - this.addressIndexerOutPointData.Delete(outPoint.ToString()); - } - } - - protected override void ItemRemovedLocked(CacheItem item) - { - base.ItemRemovedLocked(item); - - if (item.Dirty) - this.addressIndexerOutPointData.Upsert(item.Value); - } - - public bool TryGetOutPointData(OutPoint outPoint, out OutPointData outPointData) - { - if (this.TryGetValue(outPoint.ToString(), out outPointData)) - { - this.logger.LogTrace("(-)[FOUND_IN_CACHE]:true"); - return true; - } - - // Not found in cache - try find it in database. - outPointData = this.addressIndexerOutPointData.FindById(outPoint.ToString()); - - if (outPointData != null) - { - this.AddOutPointData(outPointData); - this.logger.LogTrace("(-)[FOUND_IN_DATABASE]:true"); - return true; - } - - return false; - } - - public void SaveAllItems() - { - lock (this.LockObject) - { - CacheItem[] dirtyItems = this.Keys.Where(x => x.Dirty).ToArray(); - this.addressIndexerOutPointData.Upsert(dirtyItems.Select(x => x.Value)); - - foreach (CacheItem dirtyItem in dirtyItems) - dirtyItem.Dirty = false; - } - } - - /// Persists rewind data into the repository. - /// The data to be persisted. - public void RecordRewindData(AddressIndexerRewindData rewindData) - { - lock (this.LockObject) - { - this.addressIndexerRewindData.Upsert(rewindData); - } - } - - /// Deletes rewind data items that were originated at height lower than . - /// The threshold below which data will be deleted. - public void PurgeOldRewindData(int height) - { - lock (this.LockObject) - { - var itemsToPurge = this.addressIndexerRewindData.Find(x => x.BlockHeight < height).ToArray(); - - for (int i = 0; i < itemsToPurge.Count(); i++) - { - this.addressIndexerRewindData.Delete(itemsToPurge[i].BlockHash); - - if (i % 100 == 0) - this.logger.LogInformation("Purging {0}/{1} rewind data items.", i, itemsToPurge.Count()); - } - } - } - - /// Reverts changes made by processing blocks with height higher than . - /// The height above which to restore outpoints. - public void RewindDataAboveHeight(int height) - { - lock (this.LockObject) - { - IEnumerable toRestore = this.addressIndexerRewindData.Find(x => x.BlockHeight > height); - - this.logger.LogDebug("Restoring data for {0} blocks.", toRestore.Count()); - - foreach (AddressIndexerRewindData rewindData in toRestore) - { - // Put the spent outputs back into the cache. - foreach (OutPointData outPointData in rewindData.SpentOutputs) - this.AddOutPointData(outPointData); - - // This rewind data item should now be removed from the collection. - this.addressIndexerRewindData.Delete(rewindData.BlockHash); - } - } - } - - /// - protected override bool IsCacheFullLocked(CacheItem item) - { - return this.totalSize + 1 > this.maxCacheItems; - } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerTypes.cs b/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerTypes.cs deleted file mode 100644 index 337e8d158bd..00000000000 --- a/src/Stratis.Bitcoin.Features.BlockStore/AddressIndexing/AddressIndexerTypes.cs +++ /dev/null @@ -1,40 +0,0 @@ -using System.Collections.Generic; -using LiteDB; - -namespace Stratis.Bitcoin.Features.BlockStore.AddressIndexing -{ - public class AddressIndexerTipData - { - [BsonId] - public int Id { get; set; } - - public byte[] TipHashBytes { get; set; } - - public int Height { get; set; } - - public override string ToString() - { - return $"{nameof(this.Height)}:{this.Height}"; - } - } - - public class OutPointData - { - [BsonId] - public string Outpoint { get; set; } - - public byte[] ScriptPubKeyBytes { get; set; } - - public long Money { get; set; } - } - - public class AddressIndexerRewindData - { - [BsonId(false)] - public string BlockHash { get; set; } - - public int BlockHeight { get; set; } - - public List SpentOutputs { get; set; } - } -} diff --git a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs index 0b012db61ef..3058d37dace 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/BlockStoreFeature.cs @@ -10,7 +10,6 @@ using Stratis.Bitcoin.Configuration.Logging; using Stratis.Bitcoin.Connection; using Stratis.Bitcoin.Consensus; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.BlockStore.Pruning; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.P2P.Protocol.Payloads; @@ -48,8 +47,6 @@ public class BlockStoreFeature : FullNodeFeature private readonly IPrunedBlockRepository prunedBlockRepository; - private readonly IAddressIndexer addressIndexer; - public BlockStoreFeature( Network network, ChainIndexer chainIndexer, @@ -62,8 +59,7 @@ public BlockStoreFeature( INodeStats nodeStats, IConsensusManager consensusManager, ICheckpoints checkpoints, - IPrunedBlockRepository prunedBlockRepository, - IAddressIndexer addressIndexer) + IPrunedBlockRepository prunedBlockRepository) { this.network = network; this.chainIndexer = chainIndexer; @@ -77,7 +73,6 @@ public BlockStoreFeature( this.consensusManager = consensusManager; this.checkpoints = checkpoints; this.prunedBlockRepository = prunedBlockRepository; - this.addressIndexer = addressIndexer; nodeStats.RegisterStats(this.AddInlineStats, StatsType.Inline, this.GetType().Name, 900); } @@ -153,8 +148,6 @@ public override Task InitializeAsync() this.blockStoreSignaled.Initialize(); - this.addressIndexer.Initialize(); - return Task.CompletedTask; } @@ -169,9 +162,6 @@ public override void Dispose() this.logger.LogInformation("Stopping BlockStoreSignaled."); this.blockStoreSignaled.Dispose(); - - this.logger.LogInformation("Stopping AddressIndexer."); - this.addressIndexer.Dispose(); } } @@ -201,7 +191,6 @@ public static IFullNodeBuilder UseBlockStore(this IFullNodeBuilder fullNodeBuild services.AddSingleton(); services.AddSingleton(); - services.AddSingleton(); services.AddSingleton(); }); }); diff --git a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs index c1e59ed3fb8..d3799330b6d 100644 --- a/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs +++ b/src/Stratis.Bitcoin.Features.BlockStore/Controllers/BlockStoreController.cs @@ -5,7 +5,6 @@ using NBitcoin; using Stratis.Bitcoin.Base; using Stratis.Bitcoin.Controllers.Models; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; using Stratis.Bitcoin.Features.BlockStore.Models; using Stratis.Bitcoin.Interfaces; using Stratis.Bitcoin.Utilities; @@ -25,8 +24,6 @@ public static class BlockStoreRouteEndPoint [Route("api/[controller]")] public class BlockStoreController : Controller { - private readonly IAddressIndexer addressIndexer; - /// Provides access to the block store on disk. private readonly IBlockStore blockStore; @@ -47,15 +44,12 @@ public BlockStoreController( ILoggerFactory loggerFactory, IBlockStore blockStore, IChainState chainState, - ChainIndexer chainIndexer, - IAddressIndexer addressIndexer) + ChainIndexer chainIndexer) { Guard.NotNull(network, nameof(network)); Guard.NotNull(loggerFactory, nameof(loggerFactory)); Guard.NotNull(chainState, nameof(chainState)); - Guard.NotNull(addressIndexer, nameof(addressIndexer)); - this.addressIndexer = addressIndexer; this.network = network; this.blockStore = blockStore; this.chainState = chainState; diff --git a/src/Stratis.Bitcoin.IntegrationTests/BlockStore/AddressIndexerIntegrationTests.cs b/src/Stratis.Bitcoin.IntegrationTests/BlockStore/AddressIndexerIntegrationTests.cs deleted file mode 100644 index 88dbc07f932..00000000000 --- a/src/Stratis.Bitcoin.IntegrationTests/BlockStore/AddressIndexerIntegrationTests.cs +++ /dev/null @@ -1,257 +0,0 @@ -using NBitcoin; -using Stratis.Bitcoin.Features.BlockStore.AddressIndexing; -using Stratis.Bitcoin.IntegrationTests.Common; -using Stratis.Bitcoin.IntegrationTests.Common.EnvironmentMockUpHelpers; -using Stratis.Bitcoin.IntegrationTests.Common.ReadyData; -using Stratis.Bitcoin.IntegrationTests.Common.TestNetworks; -using Stratis.Bitcoin.Networks; -using Stratis.Bitcoin.Tests.Common; -using Xunit; - -namespace Stratis.Bitcoin.IntegrationTests.BlockStore -{ - public sealed class AddressIndexerIntegrationTests - { - [Fact] - public void IndexAddresses_All_Nodes_Synced() - { - using (NodeBuilder builder = NodeBuilder.Create(this)) - { - var network = new BitcoinRegTest(); - - var nodeConfig = new NodeConfigParameters - { - { "-addressindex", "1" } - }; - - CoreNode stratisNode1 = builder.CreateStratisPowNode(network, "ai-1-stratisNode1", configParameters: nodeConfig).WithDummyWallet().Start(); - CoreNode stratisNode2 = builder.CreateStratisPowNode(network, "ai-1-stratisNode2", configParameters: nodeConfig).WithDummyWallet().Start(); - CoreNode stratisNode3 = builder.CreateStratisPowNode(network, "ai-1-stratisNode3", configParameters: nodeConfig).WithDummyWallet().Start(); - - // Connect all the nodes. - TestHelper.Connect(stratisNode1, stratisNode2); - TestHelper.Connect(stratisNode1, stratisNode3); - TestHelper.Connect(stratisNode2, stratisNode3); - - // Mine up to a height of 100. - TestHelper.MineBlocks(stratisNode1, 100); - - TestBase.WaitLoop(() => stratisNode1.FullNode.NodeService().IndexerTip.Height == 100); - TestBase.WaitLoop(() => stratisNode2.FullNode.NodeService().IndexerTip.Height == 100); - TestBase.WaitLoop(() => stratisNode3.FullNode.NodeService().IndexerTip.Height == 100); - } - } - - [Fact] - public void IndexAddresses_All_Nodes_Synced_Reorg() - { - using (NodeBuilder builder = NodeBuilder.Create(this)) - { - var network = new BitcoinRegTest(); - - var nodeConfig = new NodeConfigParameters - { - { "-addressindex", "1" } - }; - - var minerA = builder.CreateStratisPowNode(network, "ai-2-minerA", configParameters: nodeConfig).WithReadyBlockchainData(ReadyBlockchain.BitcoinRegTest10Miner).Start(); - var minerB = builder.CreateStratisPowNode(network, "ai-2-minerB", configParameters: nodeConfig).WithDummyWallet().Start(); - var syncer = builder.CreateStratisPowNode(network, "ai-2-syncer", configParameters: nodeConfig).Start(); - - // Sync the network to height 10. - TestHelper.ConnectAndSync(syncer, minerA, minerB); - - // Disconnect syncer from miner B - TestHelper.Disconnect(syncer, minerB); - - // MinerA = 15 - // MinerB = 10 - // Syncer = 15 - TestHelper.MineBlocks(minerA, 5); - TestBase.WaitLoop(() => TestHelper.AreNodesSynced(syncer, minerA)); - - // MinerA = 15 - // MinerB = 15 - // Syncer = 15 - TestHelper.Connect(syncer, minerB); - - // Disconnect syncer from miner A - TestHelper.Disconnect(syncer, minerA); - - // MinerA = 15 - // MinerB = 25 - // Syncer = 25 - TestHelper.MineBlocks(minerB, 10); - TestBase.WaitLoop(() => TestHelper.AreNodesSynced(syncer, minerB)); - - // MinerA = 35 - // MinerB = 25 - // Syncer = 25 - TestHelper.MineBlocks(minerA, 20); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 35)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 25)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 25)); - - TestHelper.Connect(syncer, minerA); - - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 35)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 35)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 35)); - - TestBase.WaitLoop(() => minerA.FullNode.NodeService().IndexerTip.Height == 35); - TestBase.WaitLoop(() => minerB.FullNode.NodeService().IndexerTip.Height == 35); - TestBase.WaitLoop(() => syncer.FullNode.NodeService().IndexerTip.Height == 35); - } - } - - [Fact] - public void IndexAddresses_All_Nodes_Synced_Reorg_With_UTXOs() - { - using (NodeBuilder builder = NodeBuilder.Create(this)) - { - var network = new BitcoinRegTestOverrideCoinbaseMaturity(5); - - var nodeConfig = new NodeConfigParameters - { - { "-addressindex", "1" } - }; - - var minerA = builder.CreateStratisPowNode(network, "ai-3-minerA", configParameters: nodeConfig).WithWallet().Start(); - var minerB = builder.CreateStratisPowNode(network, "ai-3-minerB", configParameters: nodeConfig).WithWallet().Start(); - var syncer = builder.CreateStratisPowNode(network, "ai-3-syncer", configParameters: nodeConfig).WithWallet().Start(); - - // minerA mines to height 10 - // MinerA = 10 - // MinerB = 10 - // Syncer = 10 - TestHelper.MineBlocks(minerA, 10); - - // Sync the network to height 10. - TestHelper.ConnectAndSync(syncer, minerA); - TestHelper.ConnectAndSync(syncer, minerB); - - // Disconnect syncer from miner A - TestHelper.Disconnect(syncer, minerA); - TestHelper.Disconnect(syncer, minerB); - - // minerB mines to height 10 - // MinerA = 10 - // MinerB = 20 - // Syncer = 10 - TestHelper.MineBlocks(minerB, 10); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 10)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 20)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 10)); - - // Miner A mines on its own chain. - // MinerA = 25 - // MinerB = 20 - // Syncer = 10 - TestHelper.MineBlocks(minerA, 15); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 25)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 20)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 10)); - - // Reconnect syncer to minerA. - TestHelper.Connect(syncer, minerA); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 25)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 20)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 25)); - - // Spend some coins on minerA by sending 10 STRAT to syncer. - TestHelper.SendCoins(minerA, syncer, Money.Coins(10)); - - // Miner A mines the transaction and advances onto 35. - // MinerA = 40 - // MinerB = 20 - // Syncer = 20 - TestHelper.MineBlocks(minerA, 15); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 40)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 20)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 40)); - - // minerB mines to height 50 - // MinerA = 40 - // MinerB = 50 - // Syncer = 40 - TestHelper.MineBlocks(minerB, 40); - - // Reconnect minerB (the longer chain), this will trigger the reorg. - TestHelper.Connect(syncer, minerB); - - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(syncer, 60)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerA, 60)); - TestBase.WaitLoop(() => TestHelper.IsNodeSyncedAtHeight(minerB, 60)); - - TestBase.WaitLoop(() => minerA.FullNode.NodeService().IndexerTip.Height == 60); - TestBase.WaitLoop(() => minerB.FullNode.NodeService().IndexerTip.Height == 60); - TestBase.WaitLoop(() => syncer.FullNode.NodeService().IndexerTip.Height == 60); - - // The transaction got reverted. - TestHelper.CheckWalletBalance(syncer, 0); - } - } - - [Fact] - public void IndexAddresses_All_Nodes_Synced_Reorg_Connected() - { - using (NodeBuilder builder = NodeBuilder.Create(this)) - { - var network = new BitcoinRegTest(); - - var nodeConfig = new NodeConfigParameters - { - { "-addressindex", "1" } - }; - - var minerA = builder.CreateStratisPowNode(network, "ai-4-minerA", configParameters: nodeConfig).WithReadyBlockchainData(ReadyBlockchain.BitcoinRegTest10Miner).Start(); - var minerB = builder.CreateStratisPowNode(network, "ai-4-minerB", configParameters: nodeConfig).WithDummyWallet().Start(); - var syncer = builder.CreateStratisPowNode(network, "ai-4-syncer", configParameters: nodeConfig).Start(); - - // Sync the network to height 10. - TestHelper.ConnectAndSync(syncer, minerA); - TestHelper.ConnectAndSync(syncer, minerB); - - // Stop sending blocks from miner A to syncer - TestHelper.DisableBlockPropagation(minerA, syncer); - - // Stop sending blocks from miner B to syncer - TestHelper.DisableBlockPropagation(minerB, syncer); - - // Miner A advances 2 blocks [12] - // Syncer = 10 - // Miner A = 12 - // Miner B = 10 - TestHelper.MineBlocks(minerA, 2); - Assert.True(TestHelper.IsNodeSyncedAtHeight(syncer, 10)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerA, 12)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerB, 10)); - - // Miner B advances 1 block [11] - // Syncer = 10 - // Miner A = 12 - // Miner B = 11 - TestHelper.MineBlocks(minerB, 1); - Assert.True(TestHelper.IsNodeSyncedAtHeight(syncer, 10)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerA, 12)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerB, 11)); - - // Enable sending blocks from miner A to syncer - TestHelper.EnableBlockPropagation(minerA, syncer); - // Enable sending blocks from miner B to syncer - TestHelper.EnableBlockPropagation(minerB, syncer); - - // Miner B advances 2 blocks [13] - // Syncer = 13 - // Miner A = 13 - // Miner B = 13 - TestHelper.MineBlocks(minerA, 1, false); - TestHelper.MineBlocks(minerB, 1, false); - - Assert.True(TestHelper.IsNodeSyncedAtHeight(syncer, 13)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerA, 13)); - Assert.True(TestHelper.IsNodeSyncedAtHeight(minerB, 13)); - } - } - } -}