Skip to content
This repository has been archived by the owner on Aug 16, 2021. It is now read-only.

Commit

Permalink
[FS] Input consolidation when too many inputs at once. (#3705)
Browse files Browse the repository at this point in the history
* Input consolidation step 1

* Test for our specific issue

* Add comments to input-consolidator

* Extra comment

* Stage 2

* Add feature registration

* ConsolidatingTransaction

* More skeleton

* Skeleton adjustment

* fullySigned

* Technically completed, now to test.

* Remove 1 subscription

* 1 Test

* 1 Solid test

* 1 InputConsolidator test

* One more working test

* Add logging

* Ready for review

* Fixed one test

* All tests passing

* Importat missing line

* Special validation for consolidation transactions otherwise they won't pass validation

* Fees wrong way round

* Documentation

* try catch

* Fix to broadcast if we call again

* Slight documentation fix

* Fix test

* Consolidate Input Console Display

* Push gross fix

* Time

* Small changes

* Debugging lines

* Attempt to see what's failing

* another try catch

* Even more logging

* Checking for deadlock

* Use events to remove deadlock

* Remove more error checks

* Actually async

* fee change

* Cleaning up

* skeleton moving forward

* Namespace fix

* Multiple consolidation txs at a time

* Append -> AppendLine

* Last number for performance

* Performance

* Use partial transaction requester for consolidation transactions too

* Consolidation transactions one at a time.

* Changes based on feedback

* Changes as requested

* Task.Run around blocking lock

* Get benefit of transactionbuilder PR

* Changes based on feedback

* Fix async parts

* Wasn't returning where it should be

* Fix time for PoS networks

* Changes as requested

* Changes as requested

* CheckTemplateAndSign

* Introduce lock for task

* Updating documentation
  • Loading branch information
codingupastorm authored and fassadlr committed Jun 21, 2019
1 parent 65667f5 commit d9702be
Show file tree
Hide file tree
Showing 29 changed files with 1,088 additions and 156 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@ public class CrossChainTestBase
protected IFederationWalletSyncManager federationWalletSyncManager;
protected IFederationWalletTransactionHandler FederationWalletTransactionHandler;
protected IWithdrawalTransactionBuilder withdrawalTransactionBuilder;
protected IInputConsolidator inputConsolidator;
protected IFederatedPegBroadcaster federatedPegBroadcaster;
protected IStateRepositoryRoot stateRepositoryRoot;
protected DataFolder dataFolder;
protected IWalletFeePolicy walletFeePolicy;
Expand Down Expand Up @@ -97,6 +99,8 @@ public CrossChainTestBase(Network network = null, Network counterChainNetwork =
this.FederationWalletTransactionHandler = Substitute.For<IFederationWalletTransactionHandler>();
this.walletFeePolicy = Substitute.For<IWalletFeePolicy>();
this.connectionManager = Substitute.For<IConnectionManager>();
this.federatedPegBroadcaster = Substitute.For<IFederatedPegBroadcaster>();
this.inputConsolidator = Substitute.For<IInputConsolidator>();
this.dBreezeSerializer = new DBreezeSerializer(this.network.Consensus.ConsensusFactory);
this.ibdState = Substitute.For<IInitialBlockDownloadState>();
this.wallet = null;
Expand Down Expand Up @@ -218,7 +222,7 @@ protected void Init(DataFolder dataFolder)
// TODO: The transaction builder, cross-chain store and fed wallet tx handler should be tested individually.
this.FederationWalletTransactionHandler = new FederationWalletTransactionHandler(this.loggerFactory, this.federationWalletManager, this.walletFeePolicy, this.network, this.federatedPegSettings);
this.stateRepositoryRoot = Substitute.For<IStateRepositoryRoot>();
this.withdrawalTransactionBuilder = new WithdrawalTransactionBuilder(this.loggerFactory, this.network, this.federationWalletManager, this.FederationWalletTransactionHandler, this.federatedPegSettings);
this.withdrawalTransactionBuilder = new WithdrawalTransactionBuilder(this.loggerFactory, this.network, this.federationWalletManager, this.FederationWalletTransactionHandler, this.federatedPegSettings, this.signals);

var storeSettings = (StoreSettings)FormatterServices.GetUninitializedObject(typeof(StoreSettings));

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
using Stratis.Bitcoin.P2P.Peer;
using Stratis.Bitcoin.Tests.Common;
using Stratis.Features.FederatedPeg.CounterChain;
using Stratis.Features.FederatedPeg.Events;
using Stratis.Features.FederatedPeg.Interfaces;
using Stratis.Features.FederatedPeg.Models;
using Stratis.Features.FederatedPeg.Payloads;
Expand Down Expand Up @@ -446,7 +447,7 @@ public async Task StoreMergesSignaturesAsExpectedAsync()

BitcoinAddress address = (new Key()).PubKey.Hash.GetAddress(this.network);

var deposit = new Deposit(0, new Money(160m, MoneyUnit.BTC), address.ToString(), cctsInstanceOne.NextMatureDepositHeight, 1);
var deposit = new Deposit(1, new Money(160m, MoneyUnit.BTC), address.ToString(), cctsInstanceOne.NextMatureDepositHeight, 1);

MaturedBlockDepositsModel[] blockDeposits = new[] { new MaturedBlockDepositsModel(
new MaturedBlockInfoModel() {
Expand Down Expand Up @@ -545,8 +546,8 @@ public async Task StoredPartialTransactionsTriggerSignatureRequestAsync()
BitcoinAddress address1 = (new Key()).PubKey.Hash.GetAddress(this.network);
BitcoinAddress address2 = (new Key()).PubKey.Hash.GetAddress(this.network);

var deposit1 = new Deposit(0, new Money(160m, MoneyUnit.BTC), address1.ToString(), crossChainTransferStore.NextMatureDepositHeight, 1);
var deposit2 = new Deposit(1, new Money(60m, MoneyUnit.BTC), address2.ToString(), crossChainTransferStore.NextMatureDepositHeight, 1);
var deposit1 = new Deposit(1, new Money(160m, MoneyUnit.BTC), address1.ToString(), crossChainTransferStore.NextMatureDepositHeight, 1);
var deposit2 = new Deposit(2, new Money(60m, MoneyUnit.BTC), address2.ToString(), crossChainTransferStore.NextMatureDepositHeight, 1);

MaturedBlockDepositsModel[] blockDeposits = new[] { new MaturedBlockDepositsModel(
new MaturedBlockInfoModel() {
Expand All @@ -560,32 +561,22 @@ public async Task StoredPartialTransactionsTriggerSignatureRequestAsync()
ICrossChainTransfer[] transactions = crossChainTransferStore.GetTransfersByStatus(new[] { CrossChainTransferStatus.Partial });

var requester = new PartialTransactionRequester(this.loggerFactory, crossChainTransferStore, this.asyncProvider,
this.nodeLifetime, this.connectionManager, this.federatedPegSettings, this.ibdState, this.federationWalletManager);

var peerEndPoint = new IPEndPoint(System.Net.IPAddress.Parse("1.2.3.4"), 5);
INetworkPeer peer = Substitute.For<INetworkPeer>();
peer.RemoteSocketAddress.Returns(peerEndPoint.Address);
peer.RemoteSocketPort.Returns(peerEndPoint.Port);
peer.PeerEndPoint.Returns(peerEndPoint);
peer.IsConnected.Returns(true);

var peers = new NetworkPeerCollection();
peers.Add(peer);

this.federatedPegSettings.FederationNodeIpEndPoints.Returns(new[] { peerEndPoint });

this.connectionManager.ConnectedPeers.Returns(peers);
this.nodeLifetime, this.federatedPegBroadcaster, this.ibdState, this.federationWalletManager, this.inputConsolidator);

requester.Start();

Thread.Sleep(2000);

// Receives all of the requests. We broadcast multiple at a time.
peer.Received().SendMessageAsync(Arg.Is<RequestPartialTransactionPayload>(o =>
o.DepositId == 0 && o.PartialTransaction.GetHash() == transactions[0].PartialTransaction.GetHash())).GetAwaiter().GetResult();

peer.Received().SendMessageAsync(Arg.Is<RequestPartialTransactionPayload>(o =>
o.DepositId == 1 && o.PartialTransaction.GetHash() == transactions[1].PartialTransaction.GetHash())).GetAwaiter().GetResult();
this.federatedPegBroadcaster.Received().BroadcastAsync(Arg.Is<RequestPartialTransactionPayload>(o =>
o.DepositId == 1 && o.PartialTransaction.GetHash() ==
transactions[0].PartialTransaction.GetHash()))
.GetAwaiter().GetResult();

this.federatedPegBroadcaster.Received().BroadcastAsync(Arg.Is<RequestPartialTransactionPayload>(o =>
o.DepositId == 2 && o.PartialTransaction.GetHash() ==
transactions[1].PartialTransaction.GetHash()))
.GetAwaiter().GetResult();
}
}

Expand Down Expand Up @@ -961,6 +952,67 @@ public async Task ReorgSetsAllInProgressToSuspended()
}
}

[Fact]
public async Task CrossChainTransferStoreDoesntCreateMassiveTransactions()
{
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));

this.Init(dataFolder);
this.AddFunding();
this.AppendBlocks(WithdrawalTransactionBuilder.MinConfirmations);

using (ICrossChainTransferStore crossChainTransferStore = this.CreateStore())
{
crossChainTransferStore.Initialize();
crossChainTransferStore.Start();

TestBase.WaitLoopMessage(() => (
this.ChainIndexer.Tip.Height == crossChainTransferStore.TipHashAndHeight.Height,
$"ChainIndexer.Height:{this.ChainIndexer.Tip.Height} Store.TipHashHeight:{crossChainTransferStore.TipHashAndHeight.Height}"));
Assert.Equal(this.ChainIndexer.Tip.HashBlock, crossChainTransferStore.TipHashAndHeight.HashBlock);

// Lets set the funding transactions to many really small outputs
const int numUtxos = FederatedPegSettings.MaxInputs * 2;
const decimal individualAmount = 0.1m;
const decimal depositAmount = numUtxos * individualAmount - 1; // Large amount minus some for fees.
BitcoinAddress address = new Script("").Hash.GetAddress(this.network);

this.wallet.MultiSigAddress.Transactions.Clear();
this.fundingTransactions.Clear();

Money[] funding = new Money[numUtxos];

for (int i = 0; i < funding.Length; i++)
{
funding[i] = new Money(individualAmount, MoneyUnit.BTC);
}

this.AddFundingTransaction(funding);

Deposit deposit = new Deposit(1uL, new Money(depositAmount, MoneyUnit.BTC), address.ToString(), crossChainTransferStore.NextMatureDepositHeight, 1);

var blockDeposits = new Dictionary<int, MaturedBlockDepositsModel[]>();

blockDeposits[crossChainTransferStore.NextMatureDepositHeight] = new[]
{
new MaturedBlockDepositsModel(
new MaturedBlockInfoModel
{
BlockHash = 1,
BlockHeight = crossChainTransferStore.NextMatureDepositHeight
},
new []{deposit})
};

RecordLatestMatureDepositsResult recordMatureDepositResult = await crossChainTransferStore.RecordLatestMatureDepositsAsync(blockDeposits[crossChainTransferStore.NextMatureDepositHeight]);

// The CCTS won't create any transactions until the InputConsolidator consolidates some inputs
Assert.Empty(recordMatureDepositResult.WithDrawalTransactions);

this.signals.Received().Publish(Arg.Any<WalletNeedsConsolidation>());
}
}

private Q Post<T, Q>(string url, T body)
{
// Request is sent to mainchain user.
Expand Down
79 changes: 79 additions & 0 deletions src/Stratis.Features.FederatedPeg.Tests/InputConsolidatorTests.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
using System.Collections.Generic;
using Moq;
using NBitcoin;
using Stratis.Bitcoin.Configuration;
using Stratis.Bitcoin.Features.Wallet.Interfaces;
using Stratis.Bitcoin.Tests.Common;
using Stratis.Features.FederatedPeg.InputConsolidation;
using Xunit;

namespace Stratis.Features.FederatedPeg.Tests
{
public class InputConsolidatorTests : CrossChainTestBase
{
[Fact]
public void ConsolidationTransactionOnlyBuiltWhenWalletHasManyInputs()
{
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));

this.Init(dataFolder);

var broadcasterManager = new Mock<IBroadcasterManager>();

var inputConsolidator = new InputConsolidator(
this.FederationWalletTransactionHandler,
this.federationWalletManager,
broadcasterManager.Object,
this.federatedPegSettings,
this.loggerFactory,
this.signals,
this.asyncProvider,
this.network);

Assert.Null(inputConsolidator.CreateRequiredConsolidationTransactions(Money.Coins(100m)));
}

[Fact]
public void ConsolidationTransactionBuildsCorrectly()
{
var dataFolder = new DataFolder(TestBase.CreateTestDir(this));

this.Init(dataFolder);

var broadcasterManager = new Mock<IBroadcasterManager>();

var inputConsolidator = new InputConsolidator(
this.FederationWalletTransactionHandler,
this.federationWalletManager,
broadcasterManager.Object,
this.federatedPegSettings,
this.loggerFactory,
this.signals,
this.asyncProvider,
this.network);

// Lets set the funding transactions to many really small outputs
const int numUtxos = FederatedPegSettings.MaxInputs * 2;
const decimal individualAmount = 0.1m;
const decimal depositAmount = numUtxos * individualAmount - 1; // Large amount minus some for fees.
BitcoinAddress address = new Script("").Hash.GetAddress(this.network);

this.wallet.MultiSigAddress.Transactions.Clear();
this.fundingTransactions.Clear();

Money[] funding = new Money[numUtxos];

for (int i = 0; i < funding.Length; i++)
{
funding[i] = new Money(individualAmount, MoneyUnit.BTC);
}

this.AddFundingTransaction(funding);

List<ConsolidationTransaction> transactions = inputConsolidator.CreateRequiredConsolidationTransactions(Money.Coins(depositAmount));

Assert.Equal(2, transactions.Count);
Assert.Equal(this.federatedPegSettings.MultiSigAddress.ScriptPubKey, transactions[0].PartialTransaction.Outputs[0].ScriptPubKey);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
using Microsoft.Extensions.Logging;
using NSubstitute;
using Stratis.Bitcoin.AsyncWork;
using Stratis.Bitcoin.Connection;
using Stratis.Bitcoin.Interfaces;
using Stratis.Bitcoin.Utilities;
using Stratis.Features.FederatedPeg.Interfaces;
Expand All @@ -16,10 +15,10 @@ public class PartialTransactionsRequesterTests
private readonly ILogger logger;
private readonly ILoggerFactory loggerFactory;
private readonly ICrossChainTransferStore store;
private readonly IFederatedPegSettings federatedPegSettings;
private readonly IAsyncProvider asyncProvider;
private readonly INodeLifetime nodeLifetime;
private readonly IConnectionManager connectionManager;
private readonly IFederatedPegBroadcaster federatedPegBroadcaster;
private readonly IInputConsolidator inputConsolidator;

private readonly IInitialBlockDownloadState ibdState;
private readonly IFederationWalletManager federationWalletManager;
Expand All @@ -28,12 +27,12 @@ public PartialTransactionsRequesterTests()
{
this.loggerFactory = Substitute.For<ILoggerFactory>();
this.logger = Substitute.For<ILogger>();
this.federatedPegSettings = Substitute.For<IFederatedPegSettings>();
this.loggerFactory.CreateLogger(null).ReturnsForAnyArgs(this.logger);
this.store = Substitute.For<ICrossChainTransferStore>();
this.asyncProvider = Substitute.For<IAsyncProvider>();
this.nodeLifetime = Substitute.For<INodeLifetime>();

this.federatedPegBroadcaster = Substitute.For<IFederatedPegBroadcaster>();
this.inputConsolidator = Substitute.For<IInputConsolidator>();
this.ibdState = Substitute.For<IInitialBlockDownloadState>();
this.federationWalletManager = Substitute.For<IFederationWalletManager>();
this.federationWalletManager.IsFederationWalletActive().Returns(true);
Expand All @@ -50,10 +49,10 @@ public async Task DoesntBroadcastInIBD()
this.store,
this.asyncProvider,
this.nodeLifetime,
this.connectionManager,
this.federatedPegSettings,
this.federatedPegBroadcaster,
this.ibdState,
this.federationWalletManager);
this.federationWalletManager,
this.inputConsolidator);

await partialRequester.BroadcastPartialTransactionsAsync();

Expand All @@ -70,10 +69,10 @@ public async Task DoesntBroadcastWithInactiveFederation()
this.store,
this.asyncProvider,
this.nodeLifetime,
this.connectionManager,
this.federatedPegSettings,
this.federatedPegBroadcaster,
this.ibdState,
this.federationWalletManager);
this.federationWalletManager,
this.inputConsolidator);

await partialRequester.BroadcastPartialTransactionsAsync();

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Moq;
using NBitcoin;
using Stratis.Bitcoin.Features.Wallet;
using Stratis.Bitcoin.Signals;
using Stratis.Features.FederatedPeg.Interfaces;
using Stratis.Features.FederatedPeg.TargetChain;
using Stratis.Features.FederatedPeg.Wallet;
Expand All @@ -24,6 +25,7 @@ public class WithdrawalTransactionBuilderTests
private readonly Mock<IFederationWalletManager> federationWalletManager;
private readonly Mock<IFederationWalletTransactionHandler> federationWalletTransactionHandler;
private readonly Mock<IFederatedPegSettings> federationGatewaySettings;
private readonly Mock<ISignals> signals;

public WithdrawalTransactionBuilderTests()
{
Expand All @@ -32,6 +34,7 @@ public WithdrawalTransactionBuilderTests()
this.federationWalletManager = new Mock<IFederationWalletManager>();
this.federationWalletTransactionHandler = new Mock<IFederationWalletTransactionHandler>();
this.federationGatewaySettings = new Mock<IFederatedPegSettings>();
this.signals = new Mock<ISignals>();

this.logger = new Mock<ILogger>();
this.loggerFactory.Setup(x => x.CreateLogger(It.IsAny<string>()))
Expand Down Expand Up @@ -82,7 +85,8 @@ public void FeeIsTakenFromRecipient()
this.network,
this.federationWalletManager.Object,
this.federationWalletTransactionHandler.Object,
this.federationGatewaySettings.Object
this.federationGatewaySettings.Object,
this.signals.Object
);

var recipient = new Recipient
Expand Down Expand Up @@ -116,7 +120,8 @@ public void NoSpendableTransactionsLogWarning()
this.network,
this.federationWalletManager.Object,
this.federationWalletTransactionHandler.Object,
this.federationGatewaySettings.Object
this.federationGatewaySettings.Object,
this.signals.Object
);

var recipient = new Recipient
Expand All @@ -143,7 +148,8 @@ public void NotEnoughFundsLogWarning()
this.network,
this.federationWalletManager.Object,
this.federationWalletTransactionHandler.Object,
this.federationGatewaySettings.Object
this.federationGatewaySettings.Object,
this.signals.Object
);

var recipient = new Recipient
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
using System;
using NBitcoin;
using Stratis.Bitcoin.EventBus;

namespace Stratis.Features.FederatedPeg.Events
{
public class WalletNeedsConsolidation : EventBase
{
/// <summary>
/// The amount required for the next withdrawal transaction..
/// </summary>
public Money Amount { get; }

public WalletNeedsConsolidation(Money amount)
{
this.Amount = amount;
}
}
}
Loading

0 comments on commit d9702be

Please sign in to comment.