From dfa6dc673c08f1e35127175f7e0f8d502019e223 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 16:14:32 +0200 Subject: [PATCH 01/41] Draft --- neo.UnitTests/UT_Policy.cs | 41 +++++++++++++++++++++- neo/Consensus/ConsensusContext.cs | 37 +++++++++++++++++-- neo/SmartContract/Native/PolicyContract.cs | 26 ++++++++++++++ 3 files changed, 100 insertions(+), 4 deletions(-) diff --git a/neo.UnitTests/UT_Policy.cs b/neo.UnitTests/UT_Policy.cs index 09a09b20ba..1170b11cdd 100644 --- a/neo.UnitTests/UT_Policy.cs +++ b/neo.UnitTests/UT_Policy.cs @@ -32,12 +32,16 @@ public void Check_Initialize() NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - (keyCount + 3).Should().Be(snapshot.Storages.GetChangeSet().Count()); + (keyCount + 4).Should().Be(snapshot.Storages.GetChangeSet().Count()); var ret = NativeContract.Policy.Call(snapshot, "getMaxTransactionsPerBlock"); ret.Should().BeOfType(); ret.GetBigInteger().Should().Be(512); + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + ret = NativeContract.Policy.Call(snapshot, "getFeePerByte"); ret.Should().BeOfType(); ret.GetBigInteger().Should().Be(1000); @@ -47,6 +51,41 @@ public void Check_Initialize() ((VM.Types.Array)ret).Count.Should().Be(0); } + [TestMethod] + public void Check_SetMaxBlockSize() + { + var snapshot = Store.GetSnapshot().Clone(); + + // Fake blockchain + + snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; + snapshot.Blocks.Add(UInt256.Zero, new Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); + + NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); + + // Without signature + + var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeFalse(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + + // With signature + + ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeTrue(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024); + } + [TestMethod] public void Check_SetMaxTransactionsPerBlock() { diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 270536fc89..5c994879bb 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -210,14 +210,45 @@ public ConsensusPayload MakePrepareRequest() { byte[] buffer = new byte[sizeof(ulong)]; random.NextBytes(buffer); + Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); + IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); foreach (IPolicyPlugin plugin in Plugin.Policies) memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); + List transactions = memoryPoolTransactions.ToList(); - TransactionHashes = transactions.Select(p => p.Hash).ToArray(); - Transactions = transactions.ToDictionary(p => p.Hash); + + uint maxBlockSize; + Transactions = new Dictionary(); + using (var snapshot = Blockchain.Singleton.GetSnapshot()) + { + maxBlockSize = NativeContract.Policy.GetMaxBlockSize(snapshot); + TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(snapshot)]; + } + + // Prevent that block exceed the max size + + Block.Transactions = new Transaction[0]; + var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size + + for (int x = 0; x < transactions.Count; x++) + { + var tx = transactions[x]; + + // Check if exceed + if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; + + TransactionHashes[x] = tx.Hash; + Transactions.Add(tx.Hash, tx); + } + + // Truncate null values + + Array.Resize(ref TransactionHashes, Transactions.Count); + + // Create valid request + Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); - Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest { Timestamp = Block.Timestamp, diff --git a/neo/SmartContract/Native/PolicyContract.cs b/neo/SmartContract/Native/PolicyContract.cs index 2109e8b65d..ffa23a7ca6 100644 --- a/neo/SmartContract/Native/PolicyContract.cs +++ b/neo/SmartContract/Native/PolicyContract.cs @@ -20,6 +20,7 @@ public sealed class PolicyContract : NativeContract private const byte Prefix_MaxTransactionsPerBlock = 23; private const byte Prefix_FeePerByte = 10; private const byte Prefix_BlockedAccounts = 15; + private const byte Prefix_MaxBlockSize = 16; public PolicyContract() { @@ -36,6 +37,10 @@ private bool CheckValidators(ApplicationEngine engine) internal override bool Initialize(ApplicationEngine engine) { if (!base.Initialize(engine)) return false; + engine.Snapshot.Storages.Add(CreateStorageKey(Prefix_MaxBlockSize), new StorageItem + { + Value = BitConverter.GetBytes(1024u * 256u) + }); engine.Snapshot.Storages.Add(CreateStorageKey(Prefix_MaxTransactionsPerBlock), new StorageItem { Value = BitConverter.GetBytes(512u) @@ -62,6 +67,17 @@ public uint GetMaxTransactionsPerBlock(Snapshot snapshot) return BitConverter.ToUInt32(snapshot.Storages[CreateStorageKey(Prefix_MaxTransactionsPerBlock)].Value, 0); } + [ContractMethod(0_01000000, ContractParameterType.Integer, SafeMethod = true)] + private StackItem GetMaxBlockSize(ApplicationEngine engine, VMArray args) + { + return GetMaxBlockSize(engine.Snapshot); + } + + public uint GetMaxBlockSize(Snapshot snapshot) + { + return BitConverter.ToUInt32(snapshot.Storages[CreateStorageKey(Prefix_MaxBlockSize)].Value, 0); + } + [ContractMethod(0_01000000, ContractParameterType.Integer, SafeMethod = true)] private StackItem GetFeePerByte(ApplicationEngine engine, VMArray args) { @@ -84,6 +100,16 @@ public UInt160[] GetBlockedAccounts(Snapshot snapshot) return snapshot.Storages[CreateStorageKey(Prefix_BlockedAccounts)].Value.AsSerializableArray(); } + [ContractMethod(0_03000000, ContractParameterType.Boolean, ParameterTypes = new[] { ContractParameterType.Integer }, ParameterNames = new[] { "value" })] + private StackItem SetMaxBlockSize(ApplicationEngine engine, VMArray args) + { + if (!CheckValidators(engine)) return false; + uint value = (uint)args[0].GetBigInteger(); + StorageItem storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_MaxBlockSize)); + storage.Value = BitConverter.GetBytes(value); + return true; + } + [ContractMethod(0_03000000, ContractParameterType.Boolean, ParameterTypes = new[] { ContractParameterType.Integer }, ParameterNames = new[] { "value" })] private StackItem SetMaxTransactionsPerBlock(ApplicationEngine engine, VMArray args) { From d1616d8c25ab12a6721544c1da788846a484f532 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 24 Jul 2019 16:18:54 +0200 Subject: [PATCH 02/41] Take into account p2p max payload --- neo/Consensus/ConsensusContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 5c994879bb..6b47a70b46 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -222,7 +222,7 @@ public ConsensusPayload MakePrepareRequest() Transactions = new Dictionary(); using (var snapshot = Blockchain.Singleton.GetSnapshot()) { - maxBlockSize = NativeContract.Policy.GetMaxBlockSize(snapshot); + maxBlockSize = Math.Min(Network.P2P.Message.PayloadMaxSize, NativeContract.Policy.GetMaxBlockSize(snapshot)); TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(snapshot)]; } From 89fe5b0ae2929055ec7405fae488f5c2ad4bd28e Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 25 Jul 2019 14:34:10 +0200 Subject: [PATCH 03/41] Move max allowed to policy --- neo.UnitTests/UT_Policy.cs | 11 +++++++++++ neo/Consensus/ConsensusContext.cs | 2 +- neo/SmartContract/Native/PolicyContract.cs | 1 + 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/neo.UnitTests/UT_Policy.cs b/neo.UnitTests/UT_Policy.cs index 1170b11cdd..70e8e77167 100644 --- a/neo.UnitTests/UT_Policy.cs +++ b/neo.UnitTests/UT_Policy.cs @@ -74,6 +74,17 @@ public void Check_SetMaxBlockSize() ret.Should().BeOfType(); ret.GetBigInteger().Should().Be(1024 * 256); + // More than expected + + ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = Network.P2P.Message.PayloadMaxSize }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeFalse(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + // With signature ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 6b47a70b46..5c994879bb 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -222,7 +222,7 @@ public ConsensusPayload MakePrepareRequest() Transactions = new Dictionary(); using (var snapshot = Blockchain.Singleton.GetSnapshot()) { - maxBlockSize = Math.Min(Network.P2P.Message.PayloadMaxSize, NativeContract.Policy.GetMaxBlockSize(snapshot)); + maxBlockSize = NativeContract.Policy.GetMaxBlockSize(snapshot); TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(snapshot)]; } diff --git a/neo/SmartContract/Native/PolicyContract.cs b/neo/SmartContract/Native/PolicyContract.cs index ffa23a7ca6..428e2cab74 100644 --- a/neo/SmartContract/Native/PolicyContract.cs +++ b/neo/SmartContract/Native/PolicyContract.cs @@ -105,6 +105,7 @@ private StackItem SetMaxBlockSize(ApplicationEngine engine, VMArray args) { if (!CheckValidators(engine)) return false; uint value = (uint)args[0].GetBigInteger(); + if (Network.P2P.Message.PayloadMaxSize <= value) return false; StorageItem storage = engine.Snapshot.Storages.GetAndChange(CreateStorageKey(Prefix_MaxBlockSize)); storage.Value = BitConverter.GetBytes(value); return true; From 0ddf1e9c33d28656bf5dc3b28b49c45fdbc706da Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 25 Jul 2019 14:39:14 +0200 Subject: [PATCH 04/41] Check block size on prepResponse --- neo/Consensus/ConsensusService.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 2e850807e4..b1f644625d 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -8,6 +8,7 @@ using Neo.Network.P2P.Payloads; using Neo.Persistence; using Neo.Plugins; +using Neo.SmartContract.Native; using Neo.Wallets; using System; using System.Collections.Generic; @@ -79,6 +80,19 @@ private bool AddTransaction(Transaction tx, bool verify) // previously sent prepare request, then we don't want to send a prepare response. if (context.IsPrimary || context.WatchOnly) return true; + // Check policy + using (var snapshot = Blockchain.Singleton.GetSnapshot()) + { + var block = context.CreateBlock(); + + if (block.Size > NativeContract.Policy.GetMaxBlockSize(snapshot)) + { + Log($"rejected block: {block.Hash}{Environment.NewLine} The size '{block.Size}' exceed the policy", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxRejectedByPolicy); + return false; + } + } + // Timeout extension due to prepare response sent // around 2*15/M=30.0/5 ~ 40% block time (for M=5) ExtendTimerByFactor(2); From 5d97f71f8780315c9d3ef8fa4f58c143ef0611ab Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 25 Jul 2019 14:40:19 +0200 Subject: [PATCH 05/41] Change reason --- neo/Consensus/ChangeViewReason.cs | 1 + neo/Consensus/ConsensusService.cs | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/neo/Consensus/ChangeViewReason.cs b/neo/Consensus/ChangeViewReason.cs index 64a2a6053e..eb06b7494a 100644 --- a/neo/Consensus/ChangeViewReason.cs +++ b/neo/Consensus/ChangeViewReason.cs @@ -7,5 +7,6 @@ public enum ChangeViewReason : byte TxNotFound = 0x2, TxRejectedByPolicy = 0x3, TxInvalid = 0x4, + BlockRejectedByPolicy = 0x5 } } \ No newline at end of file diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index b1f644625d..e7860ec323 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -88,7 +88,7 @@ private bool AddTransaction(Transaction tx, bool verify) if (block.Size > NativeContract.Policy.GetMaxBlockSize(snapshot)) { Log($"rejected block: {block.Hash}{Environment.NewLine} The size '{block.Size}' exceed the policy", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxRejectedByPolicy); + RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); return false; } } From f622e5753ac30307422852fde7a39fb8e819f2bc Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 25 Jul 2019 14:44:57 +0200 Subject: [PATCH 06/41] Prevent overflow --- neo/Consensus/ConsensusContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 5c994879bb..9daa7402f7 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -231,7 +231,7 @@ public ConsensusPayload MakePrepareRequest() Block.Transactions = new Transaction[0]; var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size - for (int x = 0; x < transactions.Count; x++) + for (int x = 0, max = Math.Min(Transactions.Count, transactions.Count); x < max; x++) { var tx = transactions[x]; From 4a111bcabcbc329ad3edf3f2fbd933c49a686d9e Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 26 Jul 2019 10:05:43 +0200 Subject: [PATCH 07/41] Optimization --- neo/Consensus/ConsensusContext.cs | 10 +++------- neo/Consensus/ConsensusService.cs | 23 ++++++++++------------- 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 9daa7402f7..583fdc3aa8 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -218,13 +218,9 @@ public ConsensusPayload MakePrepareRequest() List transactions = memoryPoolTransactions.ToList(); - uint maxBlockSize; - Transactions = new Dictionary(); - using (var snapshot = Blockchain.Singleton.GetSnapshot()) - { - maxBlockSize = NativeContract.Policy.GetMaxBlockSize(snapshot); - TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(snapshot)]; - } + Transactions = new Dictionary(); + uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot)]; // Prevent that block exceed the max size diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index e7860ec323..b82fdb4e1d 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -78,19 +78,16 @@ private bool AddTransaction(Transaction tx, bool verify) { // if we are the primary for this view, but acting as a backup because we recovered our own // previously sent prepare request, then we don't want to send a prepare response. - if (context.IsPrimary || context.WatchOnly) return true; - - // Check policy - using (var snapshot = Blockchain.Singleton.GetSnapshot()) - { - var block = context.CreateBlock(); - - if (block.Size > NativeContract.Policy.GetMaxBlockSize(snapshot)) - { - Log($"rejected block: {block.Hash}{Environment.NewLine} The size '{block.Size}' exceed the policy", LogLevel.Warning); - RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); - return false; - } + if (context.IsPrimary || context.WatchOnly) return true; + + // Check policy + var block = context.CreateBlock(); + + if (block.Size > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + { + Log($"rejected block: {block.Hash}{Environment.NewLine} The size '{block.Size}' exceed the policy", LogLevel.Warning); + RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); + return false; } // Timeout extension due to prepare response sent From a013394beaa227b9e2d3a1a2cec7a6d41204fd17 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 26 Jul 2019 10:09:59 +0200 Subject: [PATCH 08/41] Reduce the length of the array --- neo/Consensus/ConsensusContext.cs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 583fdc3aa8..d527e3d029 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -220,14 +220,14 @@ public ConsensusPayload MakePrepareRequest() Transactions = new Dictionary(); uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); - TransactionHashes = new UInt256[NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot)]; + TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; // Prevent that block exceed the max size Block.Transactions = new Transaction[0]; var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size - for (int x = 0, max = Math.Min(Transactions.Count, transactions.Count); x < max; x++) + for (int x = 0, max = TransactionHashes.Length; x < max; x++) { var tx = transactions[x]; @@ -240,7 +240,10 @@ public ConsensusPayload MakePrepareRequest() // Truncate null values - Array.Resize(ref TransactionHashes, Transactions.Count); + if (TransactionHashes.Length > Transactions.Count) + { + Array.Resize(ref TransactionHashes, Transactions.Count); + } // Create valid request From 94a29dc9dc18f9a3836db773d652e72c5e688222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Tue, 30 Jul 2019 18:56:08 -0300 Subject: [PATCH 09/41] Organizing consensus code --- neo/Consensus/ConsensusContext.cs | 790 +++++++++++++++--------------- 1 file changed, 392 insertions(+), 398 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d527e3d029..058ade225a 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -1,402 +1,396 @@ -using Neo.Cryptography; -using Neo.Cryptography.ECC; -using Neo.IO; -using Neo.Ledger; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract; -using Neo.SmartContract.Native; -using Neo.Wallets; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; - -namespace Neo.Consensus -{ - internal class ConsensusContext : IDisposable, ISerializable - { - /// - /// Prefix for saving consensus state. - /// - public const byte CN_Context = 0xf4; - - public Block Block; - public byte ViewNumber; - public ECPoint[] Validators; - public int MyIndex; - public UInt256[] TransactionHashes; - public Dictionary Transactions; - public ConsensusPayload[] PreparationPayloads; - public ConsensusPayload[] CommitPayloads; - public ConsensusPayload[] ChangeViewPayloads; - public ConsensusPayload[] LastChangeViewPayloads; - // LastSeenMessage array stores the height of the last seen message, for each validator. - // if this node never heard from validator i, LastSeenMessage[i] will be -1. - public int[] LastSeenMessage; - - public Snapshot Snapshot { get; private set; } - private KeyPair keyPair; - private readonly Wallet wallet; - private readonly Store store; - private readonly Random random = new Random(); - - public int F => (Validators.Length - 1) / 3; - public int M => Validators.Length - F; - public bool IsPrimary => MyIndex == Block.ConsensusData.PrimaryIndex; - public bool IsBackup => MyIndex >= 0 && MyIndex != Block.ConsensusData.PrimaryIndex; - public bool WatchOnly => MyIndex < 0; - public Header PrevHeader => Snapshot.GetHeader(Block.PrevHash); - public int CountCommitted => CommitPayloads.Count(p => p != null); - public int CountFailed => LastSeenMessage.Count(p => p < (((int)Block.Index) - 1)); - - #region Consensus States - public bool RequestSentOrReceived => PreparationPayloads[Block.ConsensusData.PrimaryIndex] != null; - public bool ResponseSent => !WatchOnly && PreparationPayloads[MyIndex] != null; - public bool CommitSent => !WatchOnly && CommitPayloads[MyIndex] != null; - public bool BlockSent => Block.Transactions != null; - public bool ViewChanging => !WatchOnly && ChangeViewPayloads[MyIndex]?.GetDeserializedMessage().NewViewNumber > ViewNumber; - public bool NotAcceptingPayloadsDueToViewChanging => ViewChanging && !MoreThanFNodesCommittedOrLost; - // A possible attack can happen if the last node to commit is malicious and either sends change view after his - // commit to stall nodes in a higher view, or if he refuses to send recovery messages. In addition, if a node - // asking change views loses network or crashes and comes back when nodes are committed in more than one higher - // numbered view, it is possible for the node accepting recovery to commit in any of the higher views, thus - // potentially splitting nodes among views and stalling the network. - public bool MoreThanFNodesCommittedOrLost => (CountCommitted + CountFailed) > F; - #endregion - - public int Size => throw new NotImplementedException(); - - public ConsensusContext(Wallet wallet, Store store) - { - this.wallet = wallet; - this.store = store; - } - - public Block CreateBlock() - { - Contract contract = Contract.CreateMultiSigContract(M, Validators); - ContractParametersContext sc = new ContractParametersContext(Block); - for (int i = 0, j = 0; i < Validators.Length && j < M; i++) - { - if (CommitPayloads[i]?.ConsensusMessage.ViewNumber != ViewNumber) continue; - sc.AddSignature(contract, Validators[i], CommitPayloads[i].GetDeserializedMessage().Signature); - j++; - } - Block.Witness = sc.GetWitnesses()[0]; - Block.Transactions = TransactionHashes.Select(p => Transactions[p]).ToArray(); - return Block; - } - - public void Deserialize(BinaryReader reader) - { - Reset(0); - if (reader.ReadUInt32() != Block.Version) throw new FormatException(); - if (reader.ReadUInt32() != Block.Index) throw new InvalidOperationException(); - Block.Timestamp = reader.ReadUInt64(); - Block.NextConsensus = reader.ReadSerializable(); - if (Block.NextConsensus.Equals(UInt160.Zero)) - Block.NextConsensus = null; - Block.ConsensusData = reader.ReadSerializable(); - ViewNumber = reader.ReadByte(); - TransactionHashes = reader.ReadSerializableArray(); - if (TransactionHashes.Length == 0) - TransactionHashes = null; - Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); - Transactions = transactions.Length == 0 ? null : transactions.ToDictionary(p => p.Hash); - PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < PreparationPayloads.Length; i++) - PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - CommitPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < CommitPayloads.Length; i++) - CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < ChangeViewPayloads.Length; i++) - ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < LastChangeViewPayloads.Length; i++) - LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - } - - public void Dispose() - { - Snapshot?.Dispose(); - } - - public Block EnsureHeader() - { - if (TransactionHashes == null) return null; - if (Block.MerkleRoot is null) - Block.MerkleRoot = Block.CalculateMerkleRoot(Block.ConsensusData.Hash, TransactionHashes); - return Block; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public uint GetPrimaryIndex(byte viewNumber) - { - int p = ((int)Block.Index - viewNumber) % Validators.Length; - return p >= 0 ? (uint)p : (uint)(p + Validators.Length); - } - - public bool Load() - { - byte[] data = store.Get(CN_Context, new byte[0]); - if (data is null || data.Length == 0) return false; - using (MemoryStream ms = new MemoryStream(data, false)) - using (BinaryReader reader = new BinaryReader(ms)) - { - try - { - Deserialize(reader); - } - catch - { - return false; - } - return true; - } - } - - public ConsensusPayload MakeChangeView(ChangeViewReason reason) - { - return ChangeViewPayloads[MyIndex] = MakeSignedPayload(new ChangeView - { - Reason = reason, - Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() - }); - } - - public ConsensusPayload MakeCommit() - { - return CommitPayloads[MyIndex] ?? (CommitPayloads[MyIndex] = MakeSignedPayload(new Commit - { - Signature = EnsureHeader().Sign(keyPair) - })); - } - - private ConsensusPayload MakeSignedPayload(ConsensusMessage message) - { - message.ViewNumber = ViewNumber; - ConsensusPayload payload = new ConsensusPayload - { - Version = Block.Version, - PrevHash = Block.PrevHash, - BlockIndex = Block.Index, - ValidatorIndex = (ushort)MyIndex, - ConsensusMessage = message - }; - SignPayload(payload); - return payload; - } - - private void SignPayload(ConsensusPayload payload) - { - ContractParametersContext sc; - try - { - sc = new ContractParametersContext(payload); - wallet.Sign(sc); - } - catch (InvalidOperationException) - { - return; - } - payload.Witness = sc.GetWitnesses()[0]; - } - - public ConsensusPayload MakePrepareRequest() - { - byte[] buffer = new byte[sizeof(ulong)]; - random.NextBytes(buffer); - Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); - - IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); - foreach (IPolicyPlugin plugin in Plugin.Policies) - memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); - - List transactions = memoryPoolTransactions.ToList(); - +using Neo.Cryptography; +using Neo.Cryptography.ECC; +using Neo.IO; +using Neo.Ledger; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Neo.Consensus +{ + internal class ConsensusContext : IDisposable, ISerializable + { + /// + /// Prefix for saving consensus state. + /// + public const byte CN_Context = 0xf4; + + public Block Block; + public byte ViewNumber; + public ECPoint[] Validators; + public int MyIndex; + public UInt256[] TransactionHashes; + public Dictionary Transactions; + public ConsensusPayload[] PreparationPayloads; + public ConsensusPayload[] CommitPayloads; + public ConsensusPayload[] ChangeViewPayloads; + public ConsensusPayload[] LastChangeViewPayloads; + // LastSeenMessage array stores the height of the last seen message, for each validator. + // if this node never heard from validator i, LastSeenMessage[i] will be -1. + public int[] LastSeenMessage; + + public Snapshot Snapshot { get; private set; } + private KeyPair keyPair; + private readonly Wallet wallet; + private readonly Store store; + private readonly Random random = new Random(); + + public int F => (Validators.Length - 1) / 3; + public int M => Validators.Length - F; + public bool IsPrimary => MyIndex == Block.ConsensusData.PrimaryIndex; + public bool IsBackup => MyIndex >= 0 && MyIndex != Block.ConsensusData.PrimaryIndex; + public bool WatchOnly => MyIndex < 0; + public Header PrevHeader => Snapshot.GetHeader(Block.PrevHash); + public int CountCommitted => CommitPayloads.Count(p => p != null); + public int CountFailed => LastSeenMessage.Count(p => p < (((int)Block.Index) - 1)); + + #region Consensus States + public bool RequestSentOrReceived => PreparationPayloads[Block.ConsensusData.PrimaryIndex] != null; + public bool ResponseSent => !WatchOnly && PreparationPayloads[MyIndex] != null; + public bool CommitSent => !WatchOnly && CommitPayloads[MyIndex] != null; + public bool BlockSent => Block.Transactions != null; + public bool ViewChanging => !WatchOnly && ChangeViewPayloads[MyIndex]?.GetDeserializedMessage().NewViewNumber > ViewNumber; + public bool NotAcceptingPayloadsDueToViewChanging => ViewChanging && !MoreThanFNodesCommittedOrLost; + // A possible attack can happen if the last node to commit is malicious and either sends change view after his + // commit to stall nodes in a higher view, or if he refuses to send recovery messages. In addition, if a node + // asking change views loses network or crashes and comes back when nodes are committed in more than one higher + // numbered view, it is possible for the node accepting recovery to commit in any of the higher views, thus + // potentially splitting nodes among views and stalling the network. + public bool MoreThanFNodesCommittedOrLost => (CountCommitted + CountFailed) > F; + #endregion + + public int Size => throw new NotImplementedException(); + + public ConsensusContext(Wallet wallet, Store store) + { + this.wallet = wallet; + this.store = store; + } + + public Block CreateBlock() + { + Contract contract = Contract.CreateMultiSigContract(M, Validators); + ContractParametersContext sc = new ContractParametersContext(Block); + for (int i = 0, j = 0; i < Validators.Length && j < M; i++) + { + if (CommitPayloads[i]?.ConsensusMessage.ViewNumber != ViewNumber) continue; + sc.AddSignature(contract, Validators[i], CommitPayloads[i].GetDeserializedMessage().Signature); + j++; + } + Block.Witness = sc.GetWitnesses()[0]; + Block.Transactions = TransactionHashes.Select(p => Transactions[p]).ToArray(); + return Block; + } + + public void Deserialize(BinaryReader reader) + { + Reset(0); + if (reader.ReadUInt32() != Block.Version) throw new FormatException(); + if (reader.ReadUInt32() != Block.Index) throw new InvalidOperationException(); + Block.Timestamp = reader.ReadUInt64(); + Block.NextConsensus = reader.ReadSerializable(); + if (Block.NextConsensus.Equals(UInt160.Zero)) + Block.NextConsensus = null; + Block.ConsensusData = reader.ReadSerializable(); + ViewNumber = reader.ReadByte(); + TransactionHashes = reader.ReadSerializableArray(); + if (TransactionHashes.Length == 0) + TransactionHashes = null; + Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); + Transactions = transactions.Length == 0 ? null : transactions.ToDictionary(p => p.Hash); + PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < PreparationPayloads.Length; i++) + PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + CommitPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < CommitPayloads.Length; i++) + CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < ChangeViewPayloads.Length; i++) + ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + } + + public void Dispose() + { + Snapshot?.Dispose(); + } + + public Block EnsureHeader() + { + if (TransactionHashes == null) return null; + if (Block.MerkleRoot is null) + Block.MerkleRoot = Block.CalculateMerkleRoot(Block.ConsensusData.Hash, TransactionHashes); + return Block; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint GetPrimaryIndex(byte viewNumber) + { + int p = ((int)Block.Index - viewNumber) % Validators.Length; + return p >= 0 ? (uint)p : (uint)(p + Validators.Length); + } + + public bool Load() + { + byte[] data = store.Get(CN_Context, new byte[0]); + if (data is null || data.Length == 0) return false; + using (MemoryStream ms = new MemoryStream(data, false)) + using (BinaryReader reader = new BinaryReader(ms)) + { + try + { + Deserialize(reader); + } + catch + { + return false; + } + return true; + } + } + + public ConsensusPayload MakeChangeView(ChangeViewReason reason) + { + return ChangeViewPayloads[MyIndex] = MakeSignedPayload(new ChangeView + { + Reason = reason, + Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() + }); + } + + public ConsensusPayload MakeCommit() + { + return CommitPayloads[MyIndex] ?? (CommitPayloads[MyIndex] = MakeSignedPayload(new Commit + { + Signature = EnsureHeader().Sign(keyPair) + })); + } + + private ConsensusPayload MakeSignedPayload(ConsensusMessage message) + { + message.ViewNumber = ViewNumber; + ConsensusPayload payload = new ConsensusPayload + { + Version = Block.Version, + PrevHash = Block.PrevHash, + BlockIndex = Block.Index, + ValidatorIndex = (ushort)MyIndex, + ConsensusMessage = message + }; + SignPayload(payload); + return payload; + } + + private void SignPayload(ConsensusPayload payload) + { + ContractParametersContext sc; + try + { + sc = new ContractParametersContext(payload); + wallet.Sign(sc); + } + catch (InvalidOperationException) + { + return; + } + payload.Witness = sc.GetWitnesses()[0]; + } + + public ConsensusPayload MakePrepareRequest() + { + byte[] buffer = new byte[sizeof(ulong)]; + random.NextBytes(buffer); + Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); + + IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); + foreach (IPolicyPlugin plugin in Plugin.Policies) + memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); + + List transactions = memoryPoolTransactions.ToList(); + // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool + TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + Transactions = new Dictionary(); + // Prevent that block exceed the max size + Block.Transactions = new Transaction[0]; uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); - TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; - - // Prevent that block exceed the max size - - Block.Transactions = new Transaction[0]; - var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size - - for (int x = 0, max = TransactionHashes.Length; x < max; x++) - { - var tx = transactions[x]; - - // Check if exceed - if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; - - TransactionHashes[x] = tx.Hash; - Transactions.Add(tx.Hash, tx); - } - - // Truncate null values - + // Ensure that the var size grows without exceed the max size + var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); + for (int x = 0, max = TransactionHashes.Length; x < max; x++) + { + var tx = transactions[x]; + + // Check if maximum block size has been already exceeded with the current selected set + if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; + + TransactionHashes[x] = tx.Hash; + Transactions.Add(tx.Hash, tx); + } + + // Truncate null values if (TransactionHashes.Length > Transactions.Count) + Array.Resize(ref TransactionHashes, Transactions.Count); + + Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest + { + Timestamp = Block.Timestamp, + Nonce = Block.ConsensusData.Nonce, + TransactionHashes = TransactionHashes + }); + } + + public ConsensusPayload MakeRecoveryRequest() + { + return MakeSignedPayload(new RecoveryRequest + { + Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() + }); + } + + public ConsensusPayload MakeRecoveryMessage() + { + PrepareRequest prepareRequestMessage = null; + if (TransactionHashes != null) + { + prepareRequestMessage = new PrepareRequest + { + ViewNumber = ViewNumber, + Timestamp = Block.Timestamp, + Nonce = Block.ConsensusData.Nonce, + TransactionHashes = TransactionHashes + }; + } + return MakeSignedPayload(new RecoveryMessage() + { + ChangeViewMessages = LastChangeViewPayloads.Where(p => p != null).Select(p => RecoveryMessage.ChangeViewPayloadCompact.FromPayload(p)).Take(M).ToDictionary(p => (int)p.ValidatorIndex), + PrepareRequestMessage = prepareRequestMessage, + // We only need a PreparationHash set if we don't have the PrepareRequest information. + PreparationHash = TransactionHashes == null ? PreparationPayloads.Where(p => p != null).GroupBy(p => p.GetDeserializedMessage().PreparationHash, (k, g) => new { Hash = k, Count = g.Count() }).OrderByDescending(p => p.Count).Select(p => p.Hash).FirstOrDefault() : null, + PreparationMessages = PreparationPayloads.Where(p => p != null).Select(p => RecoveryMessage.PreparationPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex), + CommitMessages = CommitSent + ? CommitPayloads.Where(p => p != null).Select(p => RecoveryMessage.CommitPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex) + : new Dictionary() + }); + } + + public ConsensusPayload MakePrepareResponse() + { + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareResponse + { + PreparationHash = PreparationPayloads[Block.ConsensusData.PrimaryIndex].Hash + }); + } + + public void Reset(byte viewNumber) + { + if (viewNumber == 0) + { + Snapshot?.Dispose(); + Snapshot = Blockchain.Singleton.GetSnapshot(); + Block = new Block + { + PrevHash = Snapshot.CurrentBlockHash, + Index = Snapshot.Height + 1, + NextConsensus = Blockchain.GetConsensusAddress(NativeContract.NEO.GetValidators(Snapshot).ToArray()), + ConsensusData = new ConsensusData() + }; + Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); + MyIndex = -1; + ChangeViewPayloads = new ConsensusPayload[Validators.Length]; + LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; + CommitPayloads = new ConsensusPayload[Validators.Length]; + if (LastSeenMessage == null) + { + LastSeenMessage = new int[Validators.Length]; + for (int i = 0; i < Validators.Length; i++) + LastSeenMessage[i] = -1; + } + keyPair = null; + for (int i = 0; i < Validators.Length; i++) + { + WalletAccount account = wallet?.GetAccount(Validators[i]); + if (account?.HasKey != true) continue; + MyIndex = i; + keyPair = account.GetKey(); + break; + } + } + else + { + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + if (ChangeViewPayloads[i]?.GetDeserializedMessage().NewViewNumber >= viewNumber) + LastChangeViewPayloads[i] = ChangeViewPayloads[i]; + else + LastChangeViewPayloads[i] = null; + } + ViewNumber = viewNumber; + Block.ConsensusData.PrimaryIndex = GetPrimaryIndex(viewNumber); + Block.MerkleRoot = null; + Block.Timestamp = 0; + Block.Transactions = null; + TransactionHashes = null; + PreparationPayloads = new ConsensusPayload[Validators.Length]; + if (MyIndex >= 0) LastSeenMessage[MyIndex] = (int)Block.Index; + } + + public void Save() + { + store.PutSync(CN_Context, new byte[0], this.ToArray()); + } + + public void Serialize(BinaryWriter writer) + { + writer.Write(Block.Version); + writer.Write(Block.Index); + writer.Write(Block.Timestamp); + writer.Write(Block.NextConsensus ?? UInt160.Zero); + writer.Write(Block.ConsensusData); + writer.Write(ViewNumber); + writer.Write(TransactionHashes ?? new UInt256[0]); + writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); + writer.WriteVarInt(PreparationPayloads.Length); + foreach (var payload in PreparationPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(CommitPayloads.Length); + foreach (var payload in CommitPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(ChangeViewPayloads.Length); + foreach (var payload in ChangeViewPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(LastChangeViewPayloads.Length); + foreach (var payload in LastChangeViewPayloads) { - Array.Resize(ref TransactionHashes, Transactions.Count); - } - - // Create valid request - - Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); - return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest - { - Timestamp = Block.Timestamp, - Nonce = Block.ConsensusData.Nonce, - TransactionHashes = TransactionHashes - }); - } - - public ConsensusPayload MakeRecoveryRequest() - { - return MakeSignedPayload(new RecoveryRequest - { - Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() - }); - } - - public ConsensusPayload MakeRecoveryMessage() - { - PrepareRequest prepareRequestMessage = null; - if (TransactionHashes != null) - { - prepareRequestMessage = new PrepareRequest - { - ViewNumber = ViewNumber, - Timestamp = Block.Timestamp, - Nonce = Block.ConsensusData.Nonce, - TransactionHashes = TransactionHashes - }; - } - return MakeSignedPayload(new RecoveryMessage() - { - ChangeViewMessages = LastChangeViewPayloads.Where(p => p != null).Select(p => RecoveryMessage.ChangeViewPayloadCompact.FromPayload(p)).Take(M).ToDictionary(p => (int)p.ValidatorIndex), - PrepareRequestMessage = prepareRequestMessage, - // We only need a PreparationHash set if we don't have the PrepareRequest information. - PreparationHash = TransactionHashes == null ? PreparationPayloads.Where(p => p != null).GroupBy(p => p.GetDeserializedMessage().PreparationHash, (k, g) => new { Hash = k, Count = g.Count() }).OrderByDescending(p => p.Count).Select(p => p.Hash).FirstOrDefault() : null, - PreparationMessages = PreparationPayloads.Where(p => p != null).Select(p => RecoveryMessage.PreparationPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex), - CommitMessages = CommitSent - ? CommitPayloads.Where(p => p != null).Select(p => RecoveryMessage.CommitPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex) - : new Dictionary() - }); - } - - public ConsensusPayload MakePrepareResponse() - { - return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareResponse - { - PreparationHash = PreparationPayloads[Block.ConsensusData.PrimaryIndex].Hash - }); - } - - public void Reset(byte viewNumber) - { - if (viewNumber == 0) - { - Snapshot?.Dispose(); - Snapshot = Blockchain.Singleton.GetSnapshot(); - Block = new Block - { - PrevHash = Snapshot.CurrentBlockHash, - Index = Snapshot.Height + 1, - NextConsensus = Blockchain.GetConsensusAddress(NativeContract.NEO.GetValidators(Snapshot).ToArray()), - ConsensusData = new ConsensusData() - }; - Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); - MyIndex = -1; - ChangeViewPayloads = new ConsensusPayload[Validators.Length]; - LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; - CommitPayloads = new ConsensusPayload[Validators.Length]; - if (LastSeenMessage == null) - { - LastSeenMessage = new int[Validators.Length]; - for (int i = 0; i < Validators.Length; i++) - LastSeenMessage[i] = -1; - } - keyPair = null; - for (int i = 0; i < Validators.Length; i++) - { - WalletAccount account = wallet?.GetAccount(Validators[i]); - if (account?.HasKey != true) continue; - MyIndex = i; - keyPair = account.GetKey(); - break; - } - } - else - { - for (int i = 0; i < LastChangeViewPayloads.Length; i++) - if (ChangeViewPayloads[i]?.GetDeserializedMessage().NewViewNumber >= viewNumber) - LastChangeViewPayloads[i] = ChangeViewPayloads[i]; - else - LastChangeViewPayloads[i] = null; - } - ViewNumber = viewNumber; - Block.ConsensusData.PrimaryIndex = GetPrimaryIndex(viewNumber); - Block.MerkleRoot = null; - Block.Timestamp = 0; - Block.Transactions = null; - TransactionHashes = null; - PreparationPayloads = new ConsensusPayload[Validators.Length]; - if (MyIndex >= 0) LastSeenMessage[MyIndex] = (int)Block.Index; - } - - public void Save() - { - store.PutSync(CN_Context, new byte[0], this.ToArray()); - } - - public void Serialize(BinaryWriter writer) - { - writer.Write(Block.Version); - writer.Write(Block.Index); - writer.Write(Block.Timestamp); - writer.Write(Block.NextConsensus ?? UInt160.Zero); - writer.Write(Block.ConsensusData); - writer.Write(ViewNumber); - writer.Write(TransactionHashes ?? new UInt256[0]); - writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); - writer.WriteVarInt(PreparationPayloads.Length); - foreach (var payload in PreparationPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(CommitPayloads.Length); - foreach (var payload in CommitPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(ChangeViewPayloads.Length); - foreach (var payload in ChangeViewPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(LastChangeViewPayloads.Length); - foreach (var payload in LastChangeViewPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - } - } -} + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + } + } +} From dba973392238beef0848bdac2a80e0efb768bfc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Tue, 30 Jul 2019 18:56:08 -0300 Subject: [PATCH 10/41] Revert "Organizing consensus code" This reverts commit 94a29dc9dc18f9a3836db773d652e72c5e688222. --- neo/Consensus/ConsensusContext.cs | 790 +++++++++++++++--------------- 1 file changed, 398 insertions(+), 392 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 058ade225a..d527e3d029 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -1,396 +1,402 @@ -using Neo.Cryptography; -using Neo.Cryptography.ECC; -using Neo.IO; -using Neo.Ledger; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract; -using Neo.SmartContract.Native; -using Neo.Wallets; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Runtime.CompilerServices; - -namespace Neo.Consensus -{ - internal class ConsensusContext : IDisposable, ISerializable - { - /// - /// Prefix for saving consensus state. - /// - public const byte CN_Context = 0xf4; - - public Block Block; - public byte ViewNumber; - public ECPoint[] Validators; - public int MyIndex; - public UInt256[] TransactionHashes; - public Dictionary Transactions; - public ConsensusPayload[] PreparationPayloads; - public ConsensusPayload[] CommitPayloads; - public ConsensusPayload[] ChangeViewPayloads; - public ConsensusPayload[] LastChangeViewPayloads; - // LastSeenMessage array stores the height of the last seen message, for each validator. - // if this node never heard from validator i, LastSeenMessage[i] will be -1. - public int[] LastSeenMessage; - - public Snapshot Snapshot { get; private set; } - private KeyPair keyPair; - private readonly Wallet wallet; - private readonly Store store; - private readonly Random random = new Random(); - - public int F => (Validators.Length - 1) / 3; - public int M => Validators.Length - F; - public bool IsPrimary => MyIndex == Block.ConsensusData.PrimaryIndex; - public bool IsBackup => MyIndex >= 0 && MyIndex != Block.ConsensusData.PrimaryIndex; - public bool WatchOnly => MyIndex < 0; - public Header PrevHeader => Snapshot.GetHeader(Block.PrevHash); - public int CountCommitted => CommitPayloads.Count(p => p != null); - public int CountFailed => LastSeenMessage.Count(p => p < (((int)Block.Index) - 1)); - - #region Consensus States - public bool RequestSentOrReceived => PreparationPayloads[Block.ConsensusData.PrimaryIndex] != null; - public bool ResponseSent => !WatchOnly && PreparationPayloads[MyIndex] != null; - public bool CommitSent => !WatchOnly && CommitPayloads[MyIndex] != null; - public bool BlockSent => Block.Transactions != null; - public bool ViewChanging => !WatchOnly && ChangeViewPayloads[MyIndex]?.GetDeserializedMessage().NewViewNumber > ViewNumber; - public bool NotAcceptingPayloadsDueToViewChanging => ViewChanging && !MoreThanFNodesCommittedOrLost; - // A possible attack can happen if the last node to commit is malicious and either sends change view after his - // commit to stall nodes in a higher view, or if he refuses to send recovery messages. In addition, if a node - // asking change views loses network or crashes and comes back when nodes are committed in more than one higher - // numbered view, it is possible for the node accepting recovery to commit in any of the higher views, thus - // potentially splitting nodes among views and stalling the network. - public bool MoreThanFNodesCommittedOrLost => (CountCommitted + CountFailed) > F; - #endregion - - public int Size => throw new NotImplementedException(); - - public ConsensusContext(Wallet wallet, Store store) - { - this.wallet = wallet; - this.store = store; - } - - public Block CreateBlock() - { - Contract contract = Contract.CreateMultiSigContract(M, Validators); - ContractParametersContext sc = new ContractParametersContext(Block); - for (int i = 0, j = 0; i < Validators.Length && j < M; i++) - { - if (CommitPayloads[i]?.ConsensusMessage.ViewNumber != ViewNumber) continue; - sc.AddSignature(contract, Validators[i], CommitPayloads[i].GetDeserializedMessage().Signature); - j++; - } - Block.Witness = sc.GetWitnesses()[0]; - Block.Transactions = TransactionHashes.Select(p => Transactions[p]).ToArray(); - return Block; - } - - public void Deserialize(BinaryReader reader) - { - Reset(0); - if (reader.ReadUInt32() != Block.Version) throw new FormatException(); - if (reader.ReadUInt32() != Block.Index) throw new InvalidOperationException(); - Block.Timestamp = reader.ReadUInt64(); - Block.NextConsensus = reader.ReadSerializable(); - if (Block.NextConsensus.Equals(UInt160.Zero)) - Block.NextConsensus = null; - Block.ConsensusData = reader.ReadSerializable(); - ViewNumber = reader.ReadByte(); - TransactionHashes = reader.ReadSerializableArray(); - if (TransactionHashes.Length == 0) - TransactionHashes = null; - Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); - Transactions = transactions.Length == 0 ? null : transactions.ToDictionary(p => p.Hash); - PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < PreparationPayloads.Length; i++) - PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - CommitPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < CommitPayloads.Length; i++) - CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < ChangeViewPayloads.Length; i++) - ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; - for (int i = 0; i < LastChangeViewPayloads.Length; i++) - LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; - } - - public void Dispose() - { - Snapshot?.Dispose(); - } - - public Block EnsureHeader() - { - if (TransactionHashes == null) return null; - if (Block.MerkleRoot is null) - Block.MerkleRoot = Block.CalculateMerkleRoot(Block.ConsensusData.Hash, TransactionHashes); - return Block; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public uint GetPrimaryIndex(byte viewNumber) - { - int p = ((int)Block.Index - viewNumber) % Validators.Length; - return p >= 0 ? (uint)p : (uint)(p + Validators.Length); - } - - public bool Load() - { - byte[] data = store.Get(CN_Context, new byte[0]); - if (data is null || data.Length == 0) return false; - using (MemoryStream ms = new MemoryStream(data, false)) - using (BinaryReader reader = new BinaryReader(ms)) - { - try - { - Deserialize(reader); - } - catch - { - return false; - } - return true; - } - } - - public ConsensusPayload MakeChangeView(ChangeViewReason reason) - { - return ChangeViewPayloads[MyIndex] = MakeSignedPayload(new ChangeView - { - Reason = reason, - Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() - }); - } - - public ConsensusPayload MakeCommit() - { - return CommitPayloads[MyIndex] ?? (CommitPayloads[MyIndex] = MakeSignedPayload(new Commit - { - Signature = EnsureHeader().Sign(keyPair) - })); - } - - private ConsensusPayload MakeSignedPayload(ConsensusMessage message) - { - message.ViewNumber = ViewNumber; - ConsensusPayload payload = new ConsensusPayload - { - Version = Block.Version, - PrevHash = Block.PrevHash, - BlockIndex = Block.Index, - ValidatorIndex = (ushort)MyIndex, - ConsensusMessage = message - }; - SignPayload(payload); - return payload; - } - - private void SignPayload(ConsensusPayload payload) - { - ContractParametersContext sc; - try - { - sc = new ContractParametersContext(payload); - wallet.Sign(sc); - } - catch (InvalidOperationException) - { - return; - } - payload.Witness = sc.GetWitnesses()[0]; - } - - public ConsensusPayload MakePrepareRequest() - { - byte[] buffer = new byte[sizeof(ulong)]; - random.NextBytes(buffer); - Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); - - IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); - foreach (IPolicyPlugin plugin in Plugin.Policies) - memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); - - List transactions = memoryPoolTransactions.ToList(); - // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool - TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; - +using Neo.Cryptography; +using Neo.Cryptography.ECC; +using Neo.IO; +using Neo.Ledger; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; + +namespace Neo.Consensus +{ + internal class ConsensusContext : IDisposable, ISerializable + { + /// + /// Prefix for saving consensus state. + /// + public const byte CN_Context = 0xf4; + + public Block Block; + public byte ViewNumber; + public ECPoint[] Validators; + public int MyIndex; + public UInt256[] TransactionHashes; + public Dictionary Transactions; + public ConsensusPayload[] PreparationPayloads; + public ConsensusPayload[] CommitPayloads; + public ConsensusPayload[] ChangeViewPayloads; + public ConsensusPayload[] LastChangeViewPayloads; + // LastSeenMessage array stores the height of the last seen message, for each validator. + // if this node never heard from validator i, LastSeenMessage[i] will be -1. + public int[] LastSeenMessage; + + public Snapshot Snapshot { get; private set; } + private KeyPair keyPair; + private readonly Wallet wallet; + private readonly Store store; + private readonly Random random = new Random(); + + public int F => (Validators.Length - 1) / 3; + public int M => Validators.Length - F; + public bool IsPrimary => MyIndex == Block.ConsensusData.PrimaryIndex; + public bool IsBackup => MyIndex >= 0 && MyIndex != Block.ConsensusData.PrimaryIndex; + public bool WatchOnly => MyIndex < 0; + public Header PrevHeader => Snapshot.GetHeader(Block.PrevHash); + public int CountCommitted => CommitPayloads.Count(p => p != null); + public int CountFailed => LastSeenMessage.Count(p => p < (((int)Block.Index) - 1)); + + #region Consensus States + public bool RequestSentOrReceived => PreparationPayloads[Block.ConsensusData.PrimaryIndex] != null; + public bool ResponseSent => !WatchOnly && PreparationPayloads[MyIndex] != null; + public bool CommitSent => !WatchOnly && CommitPayloads[MyIndex] != null; + public bool BlockSent => Block.Transactions != null; + public bool ViewChanging => !WatchOnly && ChangeViewPayloads[MyIndex]?.GetDeserializedMessage().NewViewNumber > ViewNumber; + public bool NotAcceptingPayloadsDueToViewChanging => ViewChanging && !MoreThanFNodesCommittedOrLost; + // A possible attack can happen if the last node to commit is malicious and either sends change view after his + // commit to stall nodes in a higher view, or if he refuses to send recovery messages. In addition, if a node + // asking change views loses network or crashes and comes back when nodes are committed in more than one higher + // numbered view, it is possible for the node accepting recovery to commit in any of the higher views, thus + // potentially splitting nodes among views and stalling the network. + public bool MoreThanFNodesCommittedOrLost => (CountCommitted + CountFailed) > F; + #endregion + + public int Size => throw new NotImplementedException(); + + public ConsensusContext(Wallet wallet, Store store) + { + this.wallet = wallet; + this.store = store; + } + + public Block CreateBlock() + { + Contract contract = Contract.CreateMultiSigContract(M, Validators); + ContractParametersContext sc = new ContractParametersContext(Block); + for (int i = 0, j = 0; i < Validators.Length && j < M; i++) + { + if (CommitPayloads[i]?.ConsensusMessage.ViewNumber != ViewNumber) continue; + sc.AddSignature(contract, Validators[i], CommitPayloads[i].GetDeserializedMessage().Signature); + j++; + } + Block.Witness = sc.GetWitnesses()[0]; + Block.Transactions = TransactionHashes.Select(p => Transactions[p]).ToArray(); + return Block; + } + + public void Deserialize(BinaryReader reader) + { + Reset(0); + if (reader.ReadUInt32() != Block.Version) throw new FormatException(); + if (reader.ReadUInt32() != Block.Index) throw new InvalidOperationException(); + Block.Timestamp = reader.ReadUInt64(); + Block.NextConsensus = reader.ReadSerializable(); + if (Block.NextConsensus.Equals(UInt160.Zero)) + Block.NextConsensus = null; + Block.ConsensusData = reader.ReadSerializable(); + ViewNumber = reader.ReadByte(); + TransactionHashes = reader.ReadSerializableArray(); + if (TransactionHashes.Length == 0) + TransactionHashes = null; + Transaction[] transactions = reader.ReadSerializableArray(Block.MaxTransactionsPerBlock); + Transactions = transactions.Length == 0 ? null : transactions.ToDictionary(p => p.Hash); + PreparationPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < PreparationPayloads.Length; i++) + PreparationPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + CommitPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < CommitPayloads.Length; i++) + CommitPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + ChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < ChangeViewPayloads.Length; i++) + ChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + LastChangeViewPayloads = new ConsensusPayload[reader.ReadVarInt(Blockchain.MaxValidators)]; + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + LastChangeViewPayloads[i] = reader.ReadBoolean() ? reader.ReadSerializable() : null; + } + + public void Dispose() + { + Snapshot?.Dispose(); + } + + public Block EnsureHeader() + { + if (TransactionHashes == null) return null; + if (Block.MerkleRoot is null) + Block.MerkleRoot = Block.CalculateMerkleRoot(Block.ConsensusData.Hash, TransactionHashes); + return Block; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public uint GetPrimaryIndex(byte viewNumber) + { + int p = ((int)Block.Index - viewNumber) % Validators.Length; + return p >= 0 ? (uint)p : (uint)(p + Validators.Length); + } + + public bool Load() + { + byte[] data = store.Get(CN_Context, new byte[0]); + if (data is null || data.Length == 0) return false; + using (MemoryStream ms = new MemoryStream(data, false)) + using (BinaryReader reader = new BinaryReader(ms)) + { + try + { + Deserialize(reader); + } + catch + { + return false; + } + return true; + } + } + + public ConsensusPayload MakeChangeView(ChangeViewReason reason) + { + return ChangeViewPayloads[MyIndex] = MakeSignedPayload(new ChangeView + { + Reason = reason, + Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() + }); + } + + public ConsensusPayload MakeCommit() + { + return CommitPayloads[MyIndex] ?? (CommitPayloads[MyIndex] = MakeSignedPayload(new Commit + { + Signature = EnsureHeader().Sign(keyPair) + })); + } + + private ConsensusPayload MakeSignedPayload(ConsensusMessage message) + { + message.ViewNumber = ViewNumber; + ConsensusPayload payload = new ConsensusPayload + { + Version = Block.Version, + PrevHash = Block.PrevHash, + BlockIndex = Block.Index, + ValidatorIndex = (ushort)MyIndex, + ConsensusMessage = message + }; + SignPayload(payload); + return payload; + } + + private void SignPayload(ConsensusPayload payload) + { + ContractParametersContext sc; + try + { + sc = new ContractParametersContext(payload); + wallet.Sign(sc); + } + catch (InvalidOperationException) + { + return; + } + payload.Witness = sc.GetWitnesses()[0]; + } + + public ConsensusPayload MakePrepareRequest() + { + byte[] buffer = new byte[sizeof(ulong)]; + random.NextBytes(buffer); + Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); + + IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); + foreach (IPolicyPlugin plugin in Plugin.Policies) + memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); + + List transactions = memoryPoolTransactions.ToList(); + Transactions = new Dictionary(); - // Prevent that block exceed the max size - Block.Transactions = new Transaction[0]; uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); - // Ensure that the var size grows without exceed the max size - var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); - for (int x = 0, max = TransactionHashes.Length; x < max; x++) - { - var tx = transactions[x]; - - // Check if maximum block size has been already exceeded with the current selected set - if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; - - TransactionHashes[x] = tx.Hash; - Transactions.Add(tx.Hash, tx); - } - - // Truncate null values + TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + + // Prevent that block exceed the max size + + Block.Transactions = new Transaction[0]; + var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size + + for (int x = 0, max = TransactionHashes.Length; x < max; x++) + { + var tx = transactions[x]; + + // Check if exceed + if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; + + TransactionHashes[x] = tx.Hash; + Transactions.Add(tx.Hash, tx); + } + + // Truncate null values + if (TransactionHashes.Length > Transactions.Count) - Array.Resize(ref TransactionHashes, Transactions.Count); - - Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); - return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest - { - Timestamp = Block.Timestamp, - Nonce = Block.ConsensusData.Nonce, - TransactionHashes = TransactionHashes - }); - } - - public ConsensusPayload MakeRecoveryRequest() - { - return MakeSignedPayload(new RecoveryRequest - { - Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() - }); - } - - public ConsensusPayload MakeRecoveryMessage() - { - PrepareRequest prepareRequestMessage = null; - if (TransactionHashes != null) - { - prepareRequestMessage = new PrepareRequest - { - ViewNumber = ViewNumber, - Timestamp = Block.Timestamp, - Nonce = Block.ConsensusData.Nonce, - TransactionHashes = TransactionHashes - }; - } - return MakeSignedPayload(new RecoveryMessage() - { - ChangeViewMessages = LastChangeViewPayloads.Where(p => p != null).Select(p => RecoveryMessage.ChangeViewPayloadCompact.FromPayload(p)).Take(M).ToDictionary(p => (int)p.ValidatorIndex), - PrepareRequestMessage = prepareRequestMessage, - // We only need a PreparationHash set if we don't have the PrepareRequest information. - PreparationHash = TransactionHashes == null ? PreparationPayloads.Where(p => p != null).GroupBy(p => p.GetDeserializedMessage().PreparationHash, (k, g) => new { Hash = k, Count = g.Count() }).OrderByDescending(p => p.Count).Select(p => p.Hash).FirstOrDefault() : null, - PreparationMessages = PreparationPayloads.Where(p => p != null).Select(p => RecoveryMessage.PreparationPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex), - CommitMessages = CommitSent - ? CommitPayloads.Where(p => p != null).Select(p => RecoveryMessage.CommitPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex) - : new Dictionary() - }); - } - - public ConsensusPayload MakePrepareResponse() - { - return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareResponse - { - PreparationHash = PreparationPayloads[Block.ConsensusData.PrimaryIndex].Hash - }); - } - - public void Reset(byte viewNumber) - { - if (viewNumber == 0) - { - Snapshot?.Dispose(); - Snapshot = Blockchain.Singleton.GetSnapshot(); - Block = new Block - { - PrevHash = Snapshot.CurrentBlockHash, - Index = Snapshot.Height + 1, - NextConsensus = Blockchain.GetConsensusAddress(NativeContract.NEO.GetValidators(Snapshot).ToArray()), - ConsensusData = new ConsensusData() - }; - Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); - MyIndex = -1; - ChangeViewPayloads = new ConsensusPayload[Validators.Length]; - LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; - CommitPayloads = new ConsensusPayload[Validators.Length]; - if (LastSeenMessage == null) - { - LastSeenMessage = new int[Validators.Length]; - for (int i = 0; i < Validators.Length; i++) - LastSeenMessage[i] = -1; - } - keyPair = null; - for (int i = 0; i < Validators.Length; i++) - { - WalletAccount account = wallet?.GetAccount(Validators[i]); - if (account?.HasKey != true) continue; - MyIndex = i; - keyPair = account.GetKey(); - break; - } - } - else - { - for (int i = 0; i < LastChangeViewPayloads.Length; i++) - if (ChangeViewPayloads[i]?.GetDeserializedMessage().NewViewNumber >= viewNumber) - LastChangeViewPayloads[i] = ChangeViewPayloads[i]; - else - LastChangeViewPayloads[i] = null; - } - ViewNumber = viewNumber; - Block.ConsensusData.PrimaryIndex = GetPrimaryIndex(viewNumber); - Block.MerkleRoot = null; - Block.Timestamp = 0; - Block.Transactions = null; - TransactionHashes = null; - PreparationPayloads = new ConsensusPayload[Validators.Length]; - if (MyIndex >= 0) LastSeenMessage[MyIndex] = (int)Block.Index; - } - - public void Save() - { - store.PutSync(CN_Context, new byte[0], this.ToArray()); - } - - public void Serialize(BinaryWriter writer) - { - writer.Write(Block.Version); - writer.Write(Block.Index); - writer.Write(Block.Timestamp); - writer.Write(Block.NextConsensus ?? UInt160.Zero); - writer.Write(Block.ConsensusData); - writer.Write(ViewNumber); - writer.Write(TransactionHashes ?? new UInt256[0]); - writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); - writer.WriteVarInt(PreparationPayloads.Length); - foreach (var payload in PreparationPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(CommitPayloads.Length); - foreach (var payload in CommitPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(ChangeViewPayloads.Length); - foreach (var payload in ChangeViewPayloads) - { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - writer.WriteVarInt(LastChangeViewPayloads.Length); - foreach (var payload in LastChangeViewPayloads) { - bool hasPayload = !(payload is null); - writer.Write(hasPayload); - if (!hasPayload) continue; - writer.Write(payload); - } - } - } -} + Array.Resize(ref TransactionHashes, Transactions.Count); + } + + // Create valid request + + Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest + { + Timestamp = Block.Timestamp, + Nonce = Block.ConsensusData.Nonce, + TransactionHashes = TransactionHashes + }); + } + + public ConsensusPayload MakeRecoveryRequest() + { + return MakeSignedPayload(new RecoveryRequest + { + Timestamp = TimeProvider.Current.UtcNow.ToTimestampMS() + }); + } + + public ConsensusPayload MakeRecoveryMessage() + { + PrepareRequest prepareRequestMessage = null; + if (TransactionHashes != null) + { + prepareRequestMessage = new PrepareRequest + { + ViewNumber = ViewNumber, + Timestamp = Block.Timestamp, + Nonce = Block.ConsensusData.Nonce, + TransactionHashes = TransactionHashes + }; + } + return MakeSignedPayload(new RecoveryMessage() + { + ChangeViewMessages = LastChangeViewPayloads.Where(p => p != null).Select(p => RecoveryMessage.ChangeViewPayloadCompact.FromPayload(p)).Take(M).ToDictionary(p => (int)p.ValidatorIndex), + PrepareRequestMessage = prepareRequestMessage, + // We only need a PreparationHash set if we don't have the PrepareRequest information. + PreparationHash = TransactionHashes == null ? PreparationPayloads.Where(p => p != null).GroupBy(p => p.GetDeserializedMessage().PreparationHash, (k, g) => new { Hash = k, Count = g.Count() }).OrderByDescending(p => p.Count).Select(p => p.Hash).FirstOrDefault() : null, + PreparationMessages = PreparationPayloads.Where(p => p != null).Select(p => RecoveryMessage.PreparationPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex), + CommitMessages = CommitSent + ? CommitPayloads.Where(p => p != null).Select(p => RecoveryMessage.CommitPayloadCompact.FromPayload(p)).ToDictionary(p => (int)p.ValidatorIndex) + : new Dictionary() + }); + } + + public ConsensusPayload MakePrepareResponse() + { + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareResponse + { + PreparationHash = PreparationPayloads[Block.ConsensusData.PrimaryIndex].Hash + }); + } + + public void Reset(byte viewNumber) + { + if (viewNumber == 0) + { + Snapshot?.Dispose(); + Snapshot = Blockchain.Singleton.GetSnapshot(); + Block = new Block + { + PrevHash = Snapshot.CurrentBlockHash, + Index = Snapshot.Height + 1, + NextConsensus = Blockchain.GetConsensusAddress(NativeContract.NEO.GetValidators(Snapshot).ToArray()), + ConsensusData = new ConsensusData() + }; + Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); + MyIndex = -1; + ChangeViewPayloads = new ConsensusPayload[Validators.Length]; + LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; + CommitPayloads = new ConsensusPayload[Validators.Length]; + if (LastSeenMessage == null) + { + LastSeenMessage = new int[Validators.Length]; + for (int i = 0; i < Validators.Length; i++) + LastSeenMessage[i] = -1; + } + keyPair = null; + for (int i = 0; i < Validators.Length; i++) + { + WalletAccount account = wallet?.GetAccount(Validators[i]); + if (account?.HasKey != true) continue; + MyIndex = i; + keyPair = account.GetKey(); + break; + } + } + else + { + for (int i = 0; i < LastChangeViewPayloads.Length; i++) + if (ChangeViewPayloads[i]?.GetDeserializedMessage().NewViewNumber >= viewNumber) + LastChangeViewPayloads[i] = ChangeViewPayloads[i]; + else + LastChangeViewPayloads[i] = null; + } + ViewNumber = viewNumber; + Block.ConsensusData.PrimaryIndex = GetPrimaryIndex(viewNumber); + Block.MerkleRoot = null; + Block.Timestamp = 0; + Block.Transactions = null; + TransactionHashes = null; + PreparationPayloads = new ConsensusPayload[Validators.Length]; + if (MyIndex >= 0) LastSeenMessage[MyIndex] = (int)Block.Index; + } + + public void Save() + { + store.PutSync(CN_Context, new byte[0], this.ToArray()); + } + + public void Serialize(BinaryWriter writer) + { + writer.Write(Block.Version); + writer.Write(Block.Index); + writer.Write(Block.Timestamp); + writer.Write(Block.NextConsensus ?? UInt160.Zero); + writer.Write(Block.ConsensusData); + writer.Write(ViewNumber); + writer.Write(TransactionHashes ?? new UInt256[0]); + writer.Write(Transactions?.Values.ToArray() ?? new Transaction[0]); + writer.WriteVarInt(PreparationPayloads.Length); + foreach (var payload in PreparationPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(CommitPayloads.Length); + foreach (var payload in CommitPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(ChangeViewPayloads.Length); + foreach (var payload in ChangeViewPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + writer.WriteVarInt(LastChangeViewPayloads.Length); + foreach (var payload in LastChangeViewPayloads) + { + bool hasPayload = !(payload is null); + writer.Write(hasPayload); + if (!hasPayload) continue; + writer.Write(payload); + } + } + } +} From b38c9a7aff3ed84c255fb1854255126e31c77680 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 09:16:56 +0200 Subject: [PATCH 11/41] Remove Policy UT --- neo.UnitTests/UT_Policy.cs | 229 ------------------------------------- 1 file changed, 229 deletions(-) delete mode 100644 neo.UnitTests/UT_Policy.cs diff --git a/neo.UnitTests/UT_Policy.cs b/neo.UnitTests/UT_Policy.cs deleted file mode 100644 index 70e8e77167..0000000000 --- a/neo.UnitTests/UT_Policy.cs +++ /dev/null @@ -1,229 +0,0 @@ -using FluentAssertions; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.SmartContract; -using Neo.SmartContract.Native; -using Neo.UnitTests.Extensions; -using System.Linq; - -namespace Neo.UnitTests -{ - [TestClass] - public class UT_Policy - { - Store Store; - - [TestInitialize] - public void TestSetup() - { - TestBlockchain.InitializeMockNeoSystem(); - Store = TestBlockchain.GetStore(); - } - - [TestMethod] - public void Check_SupportedStandards() => NativeContract.Policy.SupportedStandards().Should().BeEquivalentTo(new string[] { "NEP-10" }); - - [TestMethod] - public void Check_Initialize() - { - var snapshot = Store.GetSnapshot().Clone(); - var keyCount = snapshot.Storages.GetChangeSet().Count(); - - NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - - (keyCount + 4).Should().Be(snapshot.Storages.GetChangeSet().Count()); - - var ret = NativeContract.Policy.Call(snapshot, "getMaxTransactionsPerBlock"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(512); - - ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1024 * 256); - - ret = NativeContract.Policy.Call(snapshot, "getFeePerByte"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1000); - - ret = NativeContract.Policy.Call(snapshot, "getBlockedAccounts"); - ret.Should().BeOfType(); - ((VM.Types.Array)ret).Count.Should().Be(0); - } - - [TestMethod] - public void Check_SetMaxBlockSize() - { - var snapshot = Store.GetSnapshot().Clone(); - - // Fake blockchain - - snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; - snapshot.Blocks.Add(UInt256.Zero, new Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); - - NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - - // Without signature - - var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), - "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1024 * 256); - - // More than expected - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = Network.P2P.Message.PayloadMaxSize }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1024 * 256); - - // With signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeTrue(); - - ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1024); - } - - [TestMethod] - public void Check_SetMaxTransactionsPerBlock() - { - var snapshot = Store.GetSnapshot().Clone(); - - // Fake blockchain - - snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; - snapshot.Blocks.Add(UInt256.Zero, new Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); - - NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - - // Without signature - - var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), - "setMaxTransactionsPerBlock", new ContractParameter(ContractParameterType.Integer) { Value = 1 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getMaxTransactionsPerBlock"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(512); - - // With signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "setMaxTransactionsPerBlock", new ContractParameter(ContractParameterType.Integer) { Value = 1 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeTrue(); - - ret = NativeContract.Policy.Call(snapshot, "getMaxTransactionsPerBlock"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1); - } - - [TestMethod] - public void Check_SetFeePerByte() - { - var snapshot = Store.GetSnapshot().Clone(); - - // Fake blockchain - - snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; - snapshot.Blocks.Add(UInt256.Zero, new Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); - - NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - - // Without signature - - var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), - "setFeePerByte", new ContractParameter(ContractParameterType.Integer) { Value = 1 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getFeePerByte"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1000); - - // With signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "setFeePerByte", new ContractParameter(ContractParameterType.Integer) { Value = 1 }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeTrue(); - - ret = NativeContract.Policy.Call(snapshot, "getFeePerByte"); - ret.Should().BeOfType(); - ret.GetBigInteger().Should().Be(1); - } - - [TestMethod] - public void Check_Block_UnblockAccount() - { - var snapshot = Store.GetSnapshot().Clone(); - - // Fake blockchain - - snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; - snapshot.Blocks.Add(UInt256.Zero, new Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); - - NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - - // Block without signature - - var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), - "blockAccount", new ContractParameter(ContractParameterType.Hash160) { Value = UInt160.Zero }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getBlockedAccounts"); - ret.Should().BeOfType(); - ((VM.Types.Array)ret).Count.Should().Be(0); - - // Block with signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "blockAccount", new ContractParameter(ContractParameterType.Hash160) { Value = UInt160.Zero }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeTrue(); - - ret = NativeContract.Policy.Call(snapshot, "getBlockedAccounts"); - ret.Should().BeOfType(); - ((VM.Types.Array)ret).Count.Should().Be(1); - ((VM.Types.Array)ret)[0].GetByteArray().ShouldBeEquivalentTo(UInt160.Zero.ToArray()); - - // Unblock without signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), - "unblockAccount", new ContractParameter(ContractParameterType.Hash160) { Value = UInt160.Zero }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeFalse(); - - ret = NativeContract.Policy.Call(snapshot, "getBlockedAccounts"); - ret.Should().BeOfType(); - ((VM.Types.Array)ret).Count.Should().Be(1); - ((VM.Types.Array)ret)[0].GetByteArray().ShouldBeEquivalentTo(UInt160.Zero.ToArray()); - - // Unblock with signature - - ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), - "unblockAccount", new ContractParameter(ContractParameterType.Hash160) { Value = UInt160.Zero }); - ret.Should().BeOfType(); - ret.GetBoolean().Should().BeTrue(); - - ret = NativeContract.Policy.Call(snapshot, "getBlockedAccounts"); - ret.Should().BeOfType(); - ((VM.Types.Array)ret).Count.Should().Be(0); - } - } -} From b05fc3982f83f8339999782fcb3f1afd72eaf10a Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 09:19:17 +0200 Subject: [PATCH 12/41] Resolve Policy conflicts --- .../SmartContract/Native/UT_PolicyContract.cs | 52 ++++++++++++++++++- 1 file changed, 51 insertions(+), 1 deletion(-) diff --git a/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs b/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs index 0f353e37da..cb6c50a66c 100644 --- a/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs +++ b/neo.UnitTests/SmartContract/Native/UT_PolicyContract.cs @@ -32,12 +32,16 @@ public void Check_Initialize() NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); - (keyCount + 3).Should().Be(snapshot.Storages.GetChangeSet().Count()); + (keyCount + 4).Should().Be(snapshot.Storages.GetChangeSet().Count()); var ret = NativeContract.Policy.Call(snapshot, "getMaxTransactionsPerBlock"); ret.Should().BeOfType(); ret.GetBigInteger().Should().Be(512); + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + ret = NativeContract.Policy.Call(snapshot, "getFeePerByte"); ret.Should().BeOfType(); ret.GetBigInteger().Should().Be(1000); @@ -47,6 +51,52 @@ public void Check_Initialize() ((VM.Types.Array)ret).Count.Should().Be(0); } + [TestMethod] + public void Check_SetMaxBlockSize() + { + var snapshot = Store.GetSnapshot().Clone(); + + // Fake blockchain + + snapshot.PersistingBlock = new Block() { Index = 1000, PrevHash = UInt256.Zero }; + snapshot.Blocks.Add(UInt256.Zero, new Neo.Ledger.TrimmedBlock() { NextConsensus = UInt160.Zero }); + + NativeContract.Policy.Initialize(new ApplicationEngine(TriggerType.Application, null, snapshot, 0)).Should().BeTrue(); + + // Without signature + + var ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(null), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeFalse(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + + // More than expected + + ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = Neo.Network.P2P.Message.PayloadMaxSize }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeFalse(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024 * 256); + + // With signature + + ret = NativeContract.Policy.Call(snapshot, new Nep5NativeContractExtensions.ManualWitness(UInt160.Zero), + "setMaxBlockSize", new ContractParameter(ContractParameterType.Integer) { Value = 1024 }); + ret.Should().BeOfType(); + ret.GetBoolean().Should().BeTrue(); + + ret = NativeContract.Policy.Call(snapshot, "getMaxBlockSize"); + ret.Should().BeOfType(); + ret.GetBigInteger().Should().Be(1024); + } + [TestMethod] public void Check_SetMaxTransactionsPerBlock() { From 2d548747c37828edc6bdbd7ef23bdad418b82915 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 09:33:38 +0200 Subject: [PATCH 13/41] prepare unit test --- neo/Consensus/ConsensusContext.cs | 33 ++++++++++++++++++------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d527e3d029..6a34116367 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -206,20 +206,12 @@ private void SignPayload(ConsensusPayload payload) payload.Witness = sc.GetWitnesses()[0]; } - public ConsensusPayload MakePrepareRequest() + internal void EnsureMaxBlockSize(IEnumerable txs) { - byte[] buffer = new byte[sizeof(ulong)]; - random.NextBytes(buffer); - Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); + var transactions = txs.ToList(); - IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); - foreach (IPolicyPlugin plugin in Plugin.Policies) - memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); - - List transactions = memoryPoolTransactions.ToList(); - - Transactions = new Dictionary(); - uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + Transactions = new Dictionary(); + uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; // Prevent that block exceed the max size @@ -240,10 +232,23 @@ public ConsensusPayload MakePrepareRequest() // Truncate null values - if (TransactionHashes.Length > Transactions.Count) - { + if (TransactionHashes.Length > Transactions.Count) + { Array.Resize(ref TransactionHashes, Transactions.Count); } + } + + public ConsensusPayload MakePrepareRequest() + { + byte[] buffer = new byte[sizeof(ulong)]; + random.NextBytes(buffer); + Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); + + IEnumerable memoryPoolTransactions = Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions(); + foreach (IPolicyPlugin plugin in Plugin.Policies) + memoryPoolTransactions = plugin.FilterForBlock(memoryPoolTransactions); + + EnsureMaxBlockSize(memoryPoolTransactions); // Create valid request From 3c35703c094a9ca370af2a567e9d3681ee1cc411 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 10:35:56 +0200 Subject: [PATCH 14/41] Small unit test --- .../Consensus/UT_ConsensusContext.cs | 85 +++++++++++++++++++ neo/Consensus/ConsensusContext.cs | 14 ++- neo/Network/P2P/Payloads/BlockBase.cs | 6 +- 3 files changed, 100 insertions(+), 5 deletions(-) create mode 100644 neo.UnitTests/Consensus/UT_ConsensusContext.cs diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs new file mode 100644 index 0000000000..e751ba6a4f --- /dev/null +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -0,0 +1,85 @@ +using Akka.TestKit.Xunit2; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Consensus; +using Neo.Network.P2P.Payloads; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System.Linq; + +namespace Neo.UnitTests.Consensus +{ + + [TestClass] + public class UT_ConsensusContext : TestKit + { + [TestInitialize] + public void TestSetup() + { + TestBlockchain.InitializeMockNeoSystem(); + } + + [TestCleanup] + public void Cleanup() + { + Shutdown(); + } + + [TestMethod] + public void TestMaxBlockSize() + { + var mockWallet = new Mock(); + + mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); + ConsensusContext context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()); + context.Reset(0); + + // Only one tx + + var tx1 = CreateTransactionWithSize(200); + context.EnsureMaxBlockSize(new Transaction[] { tx1 }); + EnsureContext(context, tx1); + + // Two tx, the last one exceed + + var tx2 = CreateTransactionWithSize(256 * 1024); + context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); + EnsureContext(context, tx1); + } + + private Transaction CreateTransactionWithSize(int v) + { + var tx = new Transaction() + { + Attributes = new TransactionAttribute[0], + NetworkFee = 0, + Nonce = 0, + Script = new byte[0], + Sender = UInt160.Zero, + SystemFee = 0, + ValidUntilBlock = 0, + Version = 0, + Witnesses = new Witness[0], + }; + + // Could be higher (few bytes) if varSize grows + tx.Script = new byte[v - tx.Size]; + return tx; + } + + private void EnsureContext(ConsensusContext context, params Transaction[] expected) + { + // Check all tx + + Assert.IsTrue(expected.All(tx => context.Transactions.ContainsKey(tx.Hash))); + Assert.IsTrue(expected.All(tx => context.TransactionHashes.Count(t => t == tx.Hash) == 1)); + + // Ensure length + + // TODO: Should check the size? we need to mock the signatures + + //var block = context.CreateBlock(); + //Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); + } + } +} diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 6a34116367..d041101e8b 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -206,6 +206,10 @@ private void SignPayload(ConsensusPayload payload) payload.Witness = sc.GetWitnesses()[0]; } + /// + /// Prevent that block exceed the max size + /// + /// Ordered transactions internal void EnsureMaxBlockSize(IEnumerable txs) { var transactions = txs.ToList(); @@ -213,11 +217,17 @@ internal void EnsureMaxBlockSize(IEnumerable txs) Transactions = new Dictionary(); uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + Block.Transactions = new Transaction[0]; - // Prevent that block exceed the max size + // TODO: Real expected witness + Block.Witness = new Witness() + { + InvocationScript = new byte[0], + VerificationScript = new byte[0] + }; - Block.Transactions = new Transaction[0]; var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size + Block.Witness = null; for (int x = 0, max = TransactionHashes.Length; x < max; x++) { diff --git a/neo/Network/P2P/Payloads/BlockBase.cs b/neo/Network/P2P/Payloads/BlockBase.cs index 8820c95538..20a0c98211 100644 --- a/neo/Network/P2P/Payloads/BlockBase.cs +++ b/neo/Network/P2P/Payloads/BlockBase.cs @@ -35,11 +35,11 @@ public UInt256 Hash public virtual int Size => sizeof(uint) + //Version - PrevHash.Size + //PrevHash - MerkleRoot.Size + //MerkleRoot + UInt256.Length + //PrevHash + UInt256.Length + //MerkleRoot sizeof(ulong) + //Timestamp sizeof(uint) + //Index - NextConsensus.Size + //NextConsensus + UInt160.Length + //NextConsensus 1 + // Witness.Size; //Witness From 63d64c81e03cf58ebabacf5612cdf459065c9aab Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 10:52:44 +0200 Subject: [PATCH 15/41] More ut --- .../Consensus/UT_ConsensusContext.cs | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index e751ba6a4f..d28c1f14d2 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -5,6 +5,7 @@ using Neo.Network.P2P.Payloads; using Neo.SmartContract.Native; using Neo.Wallets; +using System; using System.Linq; namespace Neo.UnitTests.Consensus @@ -45,19 +46,37 @@ public void TestMaxBlockSize() var tx2 = CreateTransactionWithSize(256 * 1024); context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); EnsureContext(context, tx1); + + // Exceed txs + + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(context.Snapshot); + var txs = new Transaction[max + 1]; + + for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); + + context.EnsureMaxBlockSize(txs); + EnsureContext(context, txs.Take(max).ToArray()); + + // All txs + + txs = txs.Take(max).ToArray(); + + context.EnsureMaxBlockSize(txs); + EnsureContext(context, txs); } private Transaction CreateTransactionWithSize(int v) { + var r = new Random(); var tx = new Transaction() { Attributes = new TransactionAttribute[0], NetworkFee = 0, - Nonce = 0, + Nonce = (uint)Environment.TickCount, Script = new byte[0], Sender = UInt160.Zero, SystemFee = 0, - ValidUntilBlock = 0, + ValidUntilBlock = (uint)r.Next(), Version = 0, Witnesses = new Witness[0], }; From 9419e4a82d613595a44923102f2e9e73f3fc568c Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 10:55:37 +0200 Subject: [PATCH 16/41] Add one check --- neo.UnitTests/Consensus/UT_ConsensusContext.cs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index d28c1f14d2..209bc810a6 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -90,7 +90,10 @@ private void EnsureContext(ConsensusContext context, params Transaction[] expect { // Check all tx + Assert.AreEqual(expected.Length, context.Transactions.Count); Assert.IsTrue(expected.All(tx => context.Transactions.ContainsKey(tx.Hash))); + + Assert.AreEqual(expected.Length, context.TransactionHashes.Length); Assert.IsTrue(expected.All(tx => context.TransactionHashes.Count(t => t == tx.Hash) == 1)); // Ensure length From 138a3423a0c3fc39aaea9c6cbb2832552ec092a5 Mon Sep 17 00:00:00 2001 From: Shargon Date: Wed, 31 Jul 2019 11:03:17 +0200 Subject: [PATCH 17/41] Clean using --- neo/Network/P2P/Payloads/Block.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/neo/Network/P2P/Payloads/Block.cs b/neo/Network/P2P/Payloads/Block.cs index 3f29bb9428..75e5d7c424 100644 --- a/neo/Network/P2P/Payloads/Block.cs +++ b/neo/Network/P2P/Payloads/Block.cs @@ -2,7 +2,6 @@ using Neo.IO; using Neo.IO.Json; using Neo.Ledger; -using Neo.Wallets; using System; using System.Collections.Generic; using System.IO; From 5d4ebe8c200eaa2bb4d164de1f67b120e77c1115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Wed, 31 Jul 2019 06:45:11 -0300 Subject: [PATCH 18/41] Organizing consensus and comments --- neo/Consensus/ConsensusContext.cs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d041101e8b..7cedffdfd2 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -213,12 +213,10 @@ private void SignPayload(ConsensusPayload payload) internal void EnsureMaxBlockSize(IEnumerable txs) { var transactions = txs.ToList(); - - Transactions = new Dictionary(); - uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + Block.Transactions = new Transaction[0]; - // TODO: Real expected witness Block.Witness = new Witness() { @@ -226,14 +224,15 @@ internal void EnsureMaxBlockSize(IEnumerable txs) VerificationScript = new byte[0] }; + Transactions = new Dictionary(); + uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size Block.Witness = null; - for (int x = 0, max = TransactionHashes.Length; x < max; x++) { var tx = transactions[x]; - // Check if exceed + // Check if maximum block size has been already exceeded with the current selected set if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; TransactionHashes[x] = tx.Hash; @@ -241,11 +240,8 @@ internal void EnsureMaxBlockSize(IEnumerable txs) } // Truncate null values - if (TransactionHashes.Length > Transactions.Count) - { Array.Resize(ref TransactionHashes, Transactions.Count); - } } public ConsensusPayload MakePrepareRequest() From aa6cbeee23692f0869cf98f655a06caf8d5d7199 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 2 Aug 2019 10:21:39 +0200 Subject: [PATCH 19/41] Split unit test --- .../Consensus/UT_ConsensusContext.cs | 38 +++++++++++++------ 1 file changed, 26 insertions(+), 12 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 209bc810a6..214d9fed30 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -27,7 +27,7 @@ public void Cleanup() } [TestMethod] - public void TestMaxBlockSize() + public void TestMaxBlockSize_Good() { var mockWallet = new Mock(); @@ -35,19 +35,40 @@ public void TestMaxBlockSize() ConsensusContext context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()); context.Reset(0); - // Only one tx + // Only one tx, is included var tx1 = CreateTransactionWithSize(200); context.EnsureMaxBlockSize(new Transaction[] { tx1 }); - EnsureContext(context, tx1); + EnsureContext(context, tx1); + + // All txs included + + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(context.Snapshot); + var txs = new Transaction[max]; + + for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); + + context.EnsureMaxBlockSize(txs); + EnsureContext(context, txs); + } + + [TestMethod] + public void TestMaxBlockSize_Exceed() + { + var mockWallet = new Mock(); + + mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); + ConsensusContext context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()); + context.Reset(0); - // Two tx, the last one exceed + // Two tx, the last one exceed the size rule, only the first will be included + var tx1 = CreateTransactionWithSize(200); var tx2 = CreateTransactionWithSize(256 * 1024); context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); EnsureContext(context, tx1); - // Exceed txs + // Exceed txs number, just MaxTransactionsPerBlock included var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(context.Snapshot); var txs = new Transaction[max + 1]; @@ -56,13 +77,6 @@ public void TestMaxBlockSize() context.EnsureMaxBlockSize(txs); EnsureContext(context, txs.Take(max).ToArray()); - - // All txs - - txs = txs.Take(max).ToArray(); - - context.EnsureMaxBlockSize(txs); - EnsureContext(context, txs); } private Transaction CreateTransactionWithSize(int v) From a942fc5db4cc5f657cacae743d36491502490647 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 2 Aug 2019 10:48:49 +0200 Subject: [PATCH 20/41] UT check block size --- .../Consensus/UT_ConsensusContext.cs | 81 ++++++++++++------- neo/Consensus/ConsensusContext.cs | 17 ++-- 2 files changed, 65 insertions(+), 33 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 214d9fed30..5f56d3b5ab 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -2,6 +2,8 @@ using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; using Neo.Consensus; +using Neo.Cryptography.ECC; +using Neo.IO; using Neo.Network.P2P.Payloads; using Neo.SmartContract.Native; using Neo.Wallets; @@ -14,10 +16,34 @@ namespace Neo.UnitTests.Consensus [TestClass] public class UT_ConsensusContext : TestKit { + ConsensusContext _context; + KeyPair[] _validatorKeys; + [TestInitialize] public void TestSetup() { TestBlockchain.InitializeMockNeoSystem(); + + var rand = new Random(); + var mockWallet = new Mock(); + mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); + + // Create dummy validators + + _validatorKeys = new KeyPair[7]; + for (int x = 0; x < _validatorKeys.Length; x++) + { + var pk = new byte[32]; + rand.NextBytes(pk); + + _validatorKeys[x] = new KeyPair(pk); + } + + _context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()) + { + Validators = _validatorKeys.Select(u => u.PublicKey).ToArray() + }; + _context.Reset(0); } [TestCleanup] @@ -29,54 +55,42 @@ public void Cleanup() [TestMethod] public void TestMaxBlockSize_Good() { - var mockWallet = new Mock(); - - mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); - ConsensusContext context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()); - context.Reset(0); - // Only one tx, is included var tx1 = CreateTransactionWithSize(200); - context.EnsureMaxBlockSize(new Transaction[] { tx1 }); - EnsureContext(context, tx1); + _context.EnsureMaxBlockSize(new Transaction[] { tx1 }); + EnsureContext(_context, tx1); // All txs included - var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(context.Snapshot); + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); var txs = new Transaction[max]; for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); - context.EnsureMaxBlockSize(txs); - EnsureContext(context, txs); + _context.EnsureMaxBlockSize(txs); + EnsureContext(_context, txs); } [TestMethod] public void TestMaxBlockSize_Exceed() - { - var mockWallet = new Mock(); - - mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); - ConsensusContext context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()); - context.Reset(0); - + { // Two tx, the last one exceed the size rule, only the first will be included var tx1 = CreateTransactionWithSize(200); var tx2 = CreateTransactionWithSize(256 * 1024); - context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); - EnsureContext(context, tx1); + _context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); + EnsureContext(_context, tx1); // Exceed txs number, just MaxTransactionsPerBlock included - var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(context.Snapshot); + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); var txs = new Transaction[max + 1]; for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); - context.EnsureMaxBlockSize(txs); - EnsureContext(context, txs.Take(max).ToArray()); + _context.EnsureMaxBlockSize(txs); + EnsureContext(_context, txs.Take(max).ToArray()); } private Transaction CreateTransactionWithSize(int v) @@ -98,6 +112,20 @@ private Transaction CreateTransactionWithSize(int v) // Could be higher (few bytes) if varSize grows tx.Script = new byte[v - tx.Size]; return tx; + } + + private void SignBlock(ConsensusContext context) + { + context.Block.MerkleRoot = null; + + for (int x = 0; x < _validatorKeys.Length; x++) + { + _context.MyIndex = x; + _context.SetKeyPar(_validatorKeys[x]); + + var com = _context.MakeCommit(); + _context.CommitPayloads[_context.MyIndex] = com; + } } private void EnsureContext(ConsensusContext context, params Transaction[] expected) @@ -112,10 +140,9 @@ private void EnsureContext(ConsensusContext context, params Transaction[] expect // Ensure length - // TODO: Should check the size? we need to mock the signatures - - //var block = context.CreateBlock(); - //Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); + SignBlock(context); + var block = context.CreateBlock(); + Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); } } } diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 0400e4a643..d6c5fd2deb 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -213,8 +213,8 @@ internal void EnsureMaxBlockSize(IEnumerable txs) { var transactions = txs.ToList(); // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool - TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; - + TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + Block.Transactions = new Transaction[0]; // TODO: Real expected witness Block.Witness = new Witness() @@ -249,8 +249,8 @@ public ConsensusPayload MakePrepareRequest() random.NextBytes(buffer); Block.ConsensusData.Nonce = BitConverter.ToUInt64(buffer, 0); EnsureMaxBlockSize(Blockchain.Singleton.MemPool.GetSortedVerifiedTransactions()); - Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); - + Block.Timestamp = Math.Max(TimeProvider.Current.UtcNow.ToTimestampMS(), PrevHeader.Timestamp + 1); + return PreparationPayloads[MyIndex] = MakeSignedPayload(new PrepareRequest { Timestamp = Block.Timestamp, @@ -301,6 +301,11 @@ public ConsensusPayload MakePrepareResponse() }); } + internal void SetKeyPar(KeyPair key) + { + keyPair = key; + } + public void Reset(byte viewNumber) { if (viewNumber == 0) @@ -325,13 +330,13 @@ public void Reset(byte viewNumber) for (int i = 0; i < Validators.Length; i++) LastSeenMessage[i] = -1; } - keyPair = null; + SetKeyPar(null); for (int i = 0; i < Validators.Length; i++) { WalletAccount account = wallet?.GetAccount(Validators[i]); if (account?.HasKey != true) continue; MyIndex = i; - keyPair = account.GetKey(); + SetKeyPar(account.GetKey()); break; } } From 6821ee5aea505a9f2cb38bc8f7ec7797d566d861 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 2 Aug 2019 10:49:51 +0200 Subject: [PATCH 21/41] Clean --- neo/Consensus/ConsensusContext.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index d6c5fd2deb..f4a7ae94a2 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -214,8 +214,9 @@ internal void EnsureMaxBlockSize(IEnumerable txs) var transactions = txs.ToList(); // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; - + Transactions = new Dictionary(); Block.Transactions = new Transaction[0]; + // TODO: Real expected witness Block.Witness = new Witness() { @@ -223,7 +224,6 @@ internal void EnsureMaxBlockSize(IEnumerable txs) VerificationScript = new byte[0] }; - Transactions = new Dictionary(); uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size Block.Witness = null; From f9dcbeb08567281a38dc50e926ff71c0eeed2cca Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 2 Aug 2019 17:18:11 +0200 Subject: [PATCH 22/41] Expected witness size --- neo/Consensus/ConsensusContext.cs | 41 +++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index f4a7ae94a2..8397a95b17 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -6,6 +6,7 @@ using Neo.Persistence; using Neo.SmartContract; using Neo.SmartContract.Native; +using Neo.VM; using Neo.Wallets; using System; using System.Collections.Generic; @@ -41,6 +42,10 @@ internal class ConsensusContext : IDisposable, ISerializable private readonly Wallet wallet; private readonly Store store; private readonly Random random = new Random(); + /// + /// Used for know what is the expected size of the witness + /// + private Witness _fakeWitness; public int F => (Validators.Length - 1) / 3; public int M => Validators.Length - F; @@ -212,21 +217,19 @@ private void SignPayload(ConsensusPayload payload) internal void EnsureMaxBlockSize(IEnumerable txs) { var transactions = txs.ToList(); + uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; Transactions = new Dictionary(); - Block.Transactions = new Transaction[0]; - - // TODO: Real expected witness - Block.Witness = new Witness() - { - InvocationScript = new byte[0], - VerificationScript = new byte[0] - }; - - uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + Block.Transactions = new Transaction[0]; + + // We need to know the expected header size + + Block.Witness = _fakeWitness; var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size Block.Witness = null; + for (int x = 0, max = TransactionHashes.Length; x < max; x++) { var tx = transactions[x]; @@ -319,7 +322,25 @@ public void Reset(byte viewNumber) NextConsensus = Blockchain.GetConsensusAddress(NativeContract.NEO.GetValidators(Snapshot).ToArray()), ConsensusData = new ConsensusData() }; + var pv = Validators; Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); + if (_fakeWitness == null || pv != null && pv.Length != Validators.Length) + { + // If there are no fake witness or validators change, we need to compute the fakeWitness + Contract contract = Contract.CreateMultiSigContract(M, Validators); + using (ScriptBuilder sb = new ScriptBuilder()) + { + for (int x = 0; x < M; x++) + { + sb.EmitPush(new byte[64]); + } + _fakeWitness = new Witness + { + InvocationScript = sb.ToArray(), + VerificationScript = contract.Script ?? new byte[0] + }; + } + } MyIndex = -1; ChangeViewPayloads = new ConsensusPayload[Validators.Length]; LastChangeViewPayloads = new ConsensusPayload[Validators.Length]; From 5ad4500ef85e7bf87249e4c33f1fc9b7ea7a40d2 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Mon, 5 Aug 2019 10:37:42 +0800 Subject: [PATCH 23/41] optimize --- neo/Consensus/ConsensusContext.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 8397a95b17..6297b75671 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -327,7 +327,6 @@ public void Reset(byte viewNumber) if (_fakeWitness == null || pv != null && pv.Length != Validators.Length) { // If there are no fake witness or validators change, we need to compute the fakeWitness - Contract contract = Contract.CreateMultiSigContract(M, Validators); using (ScriptBuilder sb = new ScriptBuilder()) { for (int x = 0; x < M; x++) @@ -337,7 +336,7 @@ public void Reset(byte viewNumber) _fakeWitness = new Witness { InvocationScript = sb.ToArray(), - VerificationScript = contract.Script ?? new byte[0] + VerificationScript = Contract.CreateMultiSigRedeemScript(M, Validators) }; } } From 720950336f25f55b96d651a950082fa37044a2fe Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 5 Aug 2019 21:07:42 +0200 Subject: [PATCH 24/41] Remove fakeWitness --- neo/Consensus/ConsensusContext.cs | 34 +++++++++++++++++++------------ 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 6297b75671..95d7b6750c 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -39,14 +39,11 @@ internal class ConsensusContext : IDisposable, ISerializable public Snapshot Snapshot { get; private set; } private KeyPair keyPair; + private int _witnessSize; private readonly Wallet wallet; private readonly Store store; private readonly Random random = new Random(); - /// - /// Used for know what is the expected size of the witness - /// - private Witness _fakeWitness; - + public int F => (Validators.Length - 1) / 3; public int M => Validators.Length - F; public bool IsPrimary => MyIndex == Block.ConsensusData.PrimaryIndex; @@ -226,16 +223,27 @@ internal void EnsureMaxBlockSize(IEnumerable txs) // We need to know the expected header size - Block.Witness = _fakeWitness; - var fixedSize = Block.Size + IO.Helper.GetVarSize(TransactionHashes.Length); // ensure that the var size grows without exceed the max size - Block.Witness = null; + var fixedSize = + // BlockBase + sizeof(uint) + //Version + UInt256.Length + //PrevHash + UInt256.Length + //MerkleRoot + sizeof(ulong) + //Timestamp + sizeof(uint) + //Index + UInt160.Length + //NextConsensus + 1 + // + _witnessSize + //Witness + // Block + Block.ConsensusData.Size + //ConsensusData + IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size for (int x = 0, max = TransactionHashes.Length; x < max; x++) { var tx = transactions[x]; // Check if maximum block size has been already exceeded with the current selected set - if (fixedSize + UInt256.Length + tx.Size > maxBlockSize) break; + fixedSize += tx.Size; + if (fixedSize > maxBlockSize) break; TransactionHashes[x] = tx.Hash; Transactions.Add(tx.Hash, tx); @@ -324,20 +332,20 @@ public void Reset(byte viewNumber) }; var pv = Validators; Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); - if (_fakeWitness == null || pv != null && pv.Length != Validators.Length) + if (_witnessSize == 0 || pv != null && pv.Length != Validators.Length) { - // If there are no fake witness or validators change, we need to compute the fakeWitness + // Compute the expected size of the witness using (ScriptBuilder sb = new ScriptBuilder()) { for (int x = 0; x < M; x++) { sb.EmitPush(new byte[64]); } - _fakeWitness = new Witness + _witnessSize = new Witness { InvocationScript = sb.ToArray(), VerificationScript = Contract.CreateMultiSigRedeemScript(M, Validators) - }; + }.Size; } } MyIndex = -1; From 165fe96a4f02993e552620a677697223aa8fa07e Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 5 Aug 2019 21:10:13 +0200 Subject: [PATCH 25/41] Format comments --- neo/Consensus/ConsensusContext.cs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 95d7b6750c..c3885f32dd 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -232,8 +232,10 @@ internal void EnsureMaxBlockSize(IEnumerable txs) sizeof(uint) + //Index UInt160.Length + //NextConsensus 1 + // - _witnessSize + //Witness - // Block + _witnessSize; //Witness + + fixedSize += + // Block Block.ConsensusData.Size + //ConsensusData IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size From 94b8fc9e9c108e558cf5d67cbc43a3f0c796d8d2 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 5 Aug 2019 21:12:40 +0200 Subject: [PATCH 26/41] rename var --- neo/Consensus/ConsensusContext.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index c3885f32dd..ba06359dd3 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -221,9 +221,9 @@ internal void EnsureMaxBlockSize(IEnumerable txs) Transactions = new Dictionary(); Block.Transactions = new Transaction[0]; - // We need to know the expected header size + // We need to know the expected block size - var fixedSize = + var blockSize = // BlockBase sizeof(uint) + //Version UInt256.Length + //PrevHash @@ -234,24 +234,27 @@ internal void EnsureMaxBlockSize(IEnumerable txs) 1 + // _witnessSize; //Witness - fixedSize += + blockSize += // Block Block.ConsensusData.Size + //ConsensusData IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size + // Iterate transaction until reach the size + for (int x = 0, max = TransactionHashes.Length; x < max; x++) { var tx = transactions[x]; // Check if maximum block size has been already exceeded with the current selected set - fixedSize += tx.Size; - if (fixedSize > maxBlockSize) break; + blockSize += tx.Size; + if (blockSize > maxBlockSize) break; TransactionHashes[x] = tx.Hash; Transactions.Add(tx.Hash, tx); } - // Truncate null values + // Truncate null values [Tx1,Tx2,null,null,null] + if (TransactionHashes.Length > Transactions.Count) Array.Resize(ref TransactionHashes, Transactions.Count); } From 7a26d2ea75eae3e5469ae5e2158e4416fb9b1f83 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 5 Aug 2019 21:15:11 +0200 Subject: [PATCH 27/41] Add (..) --- neo/Consensus/ConsensusContext.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index ba06359dd3..f1f585315b 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -337,7 +337,7 @@ public void Reset(byte viewNumber) }; var pv = Validators; Validators = NativeContract.NEO.GetNextBlockValidators(Snapshot); - if (_witnessSize == 0 || pv != null && pv.Length != Validators.Length) + if (_witnessSize == 0 || (pv != null && pv.Length != Validators.Length)) { // Compute the expected size of the witness using (ScriptBuilder sb = new ScriptBuilder()) From 2ef7c2f8f4bfdffc331e910281325ca75c6b9148 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 6 Aug 2019 14:37:28 +0200 Subject: [PATCH 28/41] Remove SetKey method --- .../Consensus/UT_ConsensusContext.cs | 252 +++++++++--------- neo/Consensus/ConsensusContext.cs | 9 +- 2 files changed, 134 insertions(+), 127 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 5f56d3b5ab..8c4b9e2486 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -1,148 +1,160 @@ -using Akka.TestKit.Xunit2; -using Microsoft.VisualStudio.TestTools.UnitTesting; -using Moq; -using Neo.Consensus; -using Neo.Cryptography.ECC; +using Akka.TestKit.Xunit2; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Neo.Consensus; using Neo.IO; -using Neo.Network.P2P.Payloads; -using Neo.SmartContract.Native; -using Neo.Wallets; -using System; -using System.Linq; - -namespace Neo.UnitTests.Consensus -{ - - [TestClass] - public class UT_ConsensusContext : TestKit - { - ConsensusContext _context; - KeyPair[] _validatorKeys; - - [TestInitialize] - public void TestSetup() - { - TestBlockchain.InitializeMockNeoSystem(); - - var rand = new Random(); +using Neo.Network.P2P.Payloads; +using Neo.SmartContract; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System; +using System.Linq; + +namespace Neo.UnitTests.Consensus +{ + + [TestClass] + public class UT_ConsensusContext : TestKit + { + ConsensusContext _context; + KeyPair[] _validatorKeys; + + [TestInitialize] + public void TestSetup() + { + TestBlockchain.InitializeMockNeoSystem(); + + var rand = new Random(); var mockWallet = new Mock(); - mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); - - // Create dummy validators - - _validatorKeys = new KeyPair[7]; + mockWallet.Setup(p => p.GetAccount(It.IsAny())).Returns(p => new TestWalletAccount(p)); + + // Create dummy validators + + _validatorKeys = new KeyPair[7]; for (int x = 0; x < _validatorKeys.Length; x++) { var pk = new byte[32]; rand.NextBytes(pk); _validatorKeys[x] = new KeyPair(pk); - } - + } + _context = new ConsensusContext(mockWallet.Object, TestBlockchain.GetStore()) { Validators = _validatorKeys.Select(u => u.PublicKey).ToArray() - }; - _context.Reset(0); - } - - [TestCleanup] - public void Cleanup() - { - Shutdown(); - } - - [TestMethod] - public void TestMaxBlockSize_Good() - { - // Only one tx, is included - - var tx1 = CreateTransactionWithSize(200); - _context.EnsureMaxBlockSize(new Transaction[] { tx1 }); + }; + _context.Reset(0); + } + + [TestCleanup] + public void Cleanup() + { + Shutdown(); + } + + [TestMethod] + public void TestMaxBlockSize_Good() + { + // Only one tx, is included + + var tx1 = CreateTransactionWithSize(200); + _context.EnsureMaxBlockSize(new Transaction[] { tx1 }); EnsureContext(_context, tx1); // All txs included - var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); - var txs = new Transaction[max]; - - for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); - - _context.EnsureMaxBlockSize(txs); - EnsureContext(_context, txs); - } - - [TestMethod] - public void TestMaxBlockSize_Exceed() + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); + var txs = new Transaction[max]; + + for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); + + _context.EnsureMaxBlockSize(txs); + EnsureContext(_context, txs); + } + + [TestMethod] + public void TestMaxBlockSize_Exceed() { - // Two tx, the last one exceed the size rule, only the first will be included - - var tx1 = CreateTransactionWithSize(200); - var tx2 = CreateTransactionWithSize(256 * 1024); - _context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); - EnsureContext(_context, tx1); - - // Exceed txs number, just MaxTransactionsPerBlock included - - var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); - var txs = new Transaction[max + 1]; - - for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); - - _context.EnsureMaxBlockSize(txs); - EnsureContext(_context, txs.Take(max).ToArray()); - } - - private Transaction CreateTransactionWithSize(int v) - { - var r = new Random(); - var tx = new Transaction() - { - Attributes = new TransactionAttribute[0], - NetworkFee = 0, - Nonce = (uint)Environment.TickCount, - Script = new byte[0], - Sender = UInt160.Zero, - SystemFee = 0, - ValidUntilBlock = (uint)r.Next(), - Version = 0, - Witnesses = new Witness[0], - }; - - // Could be higher (few bytes) if varSize grows - tx.Script = new byte[v - tx.Size]; - return tx; + // Two tx, the last one exceed the size rule, only the first will be included + + var tx1 = CreateTransactionWithSize(200); + var tx2 = CreateTransactionWithSize(256 * 1024); + _context.EnsureMaxBlockSize(new Transaction[] { tx1, tx2 }); + EnsureContext(_context, tx1); + + // Exceed txs number, just MaxTransactionsPerBlock included + + var max = (int)NativeContract.Policy.GetMaxTransactionsPerBlock(_context.Snapshot); + var txs = new Transaction[max + 1]; + + for (int x = 0; x < max; x++) txs[x] = CreateTransactionWithSize(100); + + _context.EnsureMaxBlockSize(txs); + EnsureContext(_context, txs.Take(max).ToArray()); } - private void SignBlock(ConsensusContext context) + private Transaction CreateTransactionWithSize(int v) + { + var r = new Random(); + var tx = new Transaction() + { + Attributes = new TransactionAttribute[0], + NetworkFee = 0, + Nonce = (uint)Environment.TickCount, + Script = new byte[0], + Sender = UInt160.Zero, + SystemFee = 0, + ValidUntilBlock = (uint)r.Next(), + Version = 0, + Witnesses = new Witness[0], + }; + + // Could be higher (few bytes) if varSize grows + tx.Script = new byte[v - tx.Size]; + return tx; + } + + private Block SignBlock(ConsensusContext context) { context.Block.MerkleRoot = null; for (int x = 0; x < _validatorKeys.Length; x++) { _context.MyIndex = x; - _context.SetKeyPar(_validatorKeys[x]); var com = _context.MakeCommit(); _context.CommitPayloads[_context.MyIndex] = com; } - } - - private void EnsureContext(ConsensusContext context, params Transaction[] expected) - { - // Check all tx - - Assert.AreEqual(expected.Length, context.Transactions.Count); - Assert.IsTrue(expected.All(tx => context.Transactions.ContainsKey(tx.Hash))); - - Assert.AreEqual(expected.Length, context.TransactionHashes.Length); - Assert.IsTrue(expected.All(tx => context.TransactionHashes.Count(t => t == tx.Hash) == 1)); - - // Ensure length - - SignBlock(context); - var block = context.CreateBlock(); - Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); - } - } -} + + // Manual block sign + + Contract contract = Contract.CreateMultiSigContract(context.M, context.Validators); + ContractParametersContext sc = new ContractParametersContext(context.Block); + for (int i = 0, j = 0; i < context.Validators.Length && j < context.M; i++) + { + if (context.CommitPayloads[i]?.ConsensusMessage.ViewNumber != context.ViewNumber) continue; + sc.AddSignature(contract, context.Validators[i], context.CommitPayloads[i].GetDeserializedMessage().Signature); + j++; + } + context.Block.Witness = sc.GetWitnesses()[0]; + context.Block.Transactions = context.TransactionHashes.Select(p => context.Transactions[p]).ToArray(); + return context.Block; + } + + private void EnsureContext(ConsensusContext context, params Transaction[] expected) + { + // Check all tx + + Assert.AreEqual(expected.Length, context.Transactions.Count); + Assert.IsTrue(expected.All(tx => context.Transactions.ContainsKey(tx.Hash))); + + Assert.AreEqual(expected.Length, context.TransactionHashes.Length); + Assert.IsTrue(expected.All(tx => context.TransactionHashes.Count(t => t == tx.Hash) == 1)); + + // Ensure length + + var block = SignBlock(context); + Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); + } + } +} diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index f1f585315b..b1f78d255a 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -317,11 +317,6 @@ public ConsensusPayload MakePrepareResponse() }); } - internal void SetKeyPar(KeyPair key) - { - keyPair = key; - } - public void Reset(byte viewNumber) { if (viewNumber == 0) @@ -363,13 +358,13 @@ public void Reset(byte viewNumber) for (int i = 0; i < Validators.Length; i++) LastSeenMessage[i] = -1; } - SetKeyPar(null); + keyPair = null; for (int i = 0; i < Validators.Length; i++) { WalletAccount account = wallet?.GetAccount(Validators[i]); if (account?.HasKey != true) continue; MyIndex = i; - SetKeyPar(account.GetKey()); + keyPair = account.GetKey(); break; } } From 7a799db57357954007d784b1b4ad41e36c3fa9fe Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 6 Aug 2019 14:49:04 +0200 Subject: [PATCH 29/41] Centralize expected block size --- neo/Consensus/ConsensusContext.cs | 50 +++++++++++++++++++++---------- neo/Consensus/ConsensusService.cs | 5 ++-- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index b1f78d255a..a9a64e3276 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -207,6 +207,40 @@ private void SignPayload(ConsensusPayload payload) payload.Witness = sc.GetWitnesses()[0]; } + /// + /// Return the expected block size + /// + internal int GetExpectedBlockSize(IEnumerable txs = null) + { + var blockSize = + // BlockBase + sizeof(uint) + //Version + UInt256.Length + //PrevHash + UInt256.Length + //MerkleRoot + sizeof(ulong) + //Timestamp + sizeof(uint) + //Index + UInt160.Length + //NextConsensus + 1 + // + _witnessSize; //Witness + + blockSize += + // Block + Block.ConsensusData.Size; //ConsensusData + + // Txs + + if (txs != null) + { + blockSize += txs.Sum(u => u.Size)+ IO.Helper.GetVarSize(txs.Count() + 1); + } + else + { + blockSize += IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size + } + + return blockSize; + } + /// /// Prevent that block exceed the max size /// @@ -223,21 +257,7 @@ internal void EnsureMaxBlockSize(IEnumerable txs) // We need to know the expected block size - var blockSize = - // BlockBase - sizeof(uint) + //Version - UInt256.Length + //PrevHash - UInt256.Length + //MerkleRoot - sizeof(ulong) + //Timestamp - sizeof(uint) + //Index - UInt160.Length + //NextConsensus - 1 + // - _witnessSize; //Witness - - blockSize += - // Block - Block.ConsensusData.Size + //ConsensusData - IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size + var blockSize = GetExpectedBlockSize(); // Iterate transaction until reach the size diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index e3c96d6838..3aa3d566ee 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -81,11 +81,10 @@ private bool AddTransaction(Transaction tx, bool verify) if (context.IsPrimary || context.WatchOnly) return true; // Check policy - var block = context.CreateBlock(); - if (block.Size > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + if (context.GetExpectedBlockSize(context.Transactions.Values) > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { - Log($"rejected block: {block.Hash}{Environment.NewLine} The size '{block.Size}' exceed the policy", LogLevel.Warning); + Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); return false; } From 1dca8d5c557c6e7e74488be83958440aeb23cd80 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 6 Aug 2019 15:02:07 +0200 Subject: [PATCH 30/41] Optimize --- neo/Consensus/ConsensusContext.cs | 17 ++++++++--------- neo/Consensus/ConsensusService.cs | 2 +- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index a9a64e3276..9dc5f2bb92 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -210,7 +210,8 @@ private void SignPayload(ConsensusPayload payload) /// /// Return the expected block size /// - internal int GetExpectedBlockSize(IEnumerable txs = null) + /// Compute the block size with the Transactions + internal int GetExpectedBlockSize(bool computeTransactionSize) { var blockSize = // BlockBase @@ -225,17 +226,15 @@ internal int GetExpectedBlockSize(IEnumerable txs = null) blockSize += // Block - Block.ConsensusData.Size; //ConsensusData + Block.ConsensusData.Size + //ConsensusData + IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Transactions count + // Txs - if (txs != null) - { - blockSize += txs.Sum(u => u.Size)+ IO.Helper.GetVarSize(txs.Count() + 1); - } - else + if (computeTransactionSize) { - blockSize += IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Tx => ensure that the var size grows without exceed the max size + blockSize += Transactions.Values.Sum(u => u.Size) + IO.Helper.GetVarSize(Transactions.Count() + 1); } return blockSize; @@ -257,7 +256,7 @@ internal void EnsureMaxBlockSize(IEnumerable txs) // We need to know the expected block size - var blockSize = GetExpectedBlockSize(); + var blockSize = GetExpectedBlockSize(false); // Iterate transaction until reach the size diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 3aa3d566ee..beab42f07c 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -82,7 +82,7 @@ private bool AddTransaction(Transaction tx, bool verify) // Check policy - if (context.GetExpectedBlockSize(context.Transactions.Values) > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + if (context.GetExpectedBlockSize(true) > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); From dbda619a7e0ff15ad5e087890fb8f76b9844b0f7 Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 6 Aug 2019 15:02:28 +0200 Subject: [PATCH 31/41] Fix --- neo/Consensus/ConsensusContext.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 9dc5f2bb92..279d2f7f56 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -229,12 +229,11 @@ internal int GetExpectedBlockSize(bool computeTransactionSize) Block.ConsensusData.Size + //ConsensusData IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Transactions count - // Txs if (computeTransactionSize) { - blockSize += Transactions.Values.Sum(u => u.Size) + IO.Helper.GetVarSize(Transactions.Count() + 1); + blockSize += Transactions.Values.Sum(u => u.Size); } return blockSize; From 8d385a8e5b0f6e3e49ab0fcb101518dc7046f70b Mon Sep 17 00:00:00 2001 From: Shargon Date: Tue, 6 Aug 2019 15:05:09 +0200 Subject: [PATCH 32/41] Add one test --- neo.UnitTests/Consensus/UT_ConsensusContext.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 8c4b9e2486..481fb9571b 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -154,6 +154,8 @@ private void EnsureContext(ConsensusContext context, params Transaction[] expect // Ensure length var block = SignBlock(context); + + Assert.AreEqual(context.GetExpectedBlockSize(true), block.Size); Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); } } From accbf72476ed68ac3a6ca43388225770f19c8988 Mon Sep 17 00:00:00 2001 From: erikzhang Date: Thu, 8 Aug 2019 15:04:34 +0800 Subject: [PATCH 33/41] Optimize `EnsureMaxBlockSize()` --- neo/Consensus/ConsensusContext.cs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 279d2f7f56..5be1849b9f 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -245,11 +245,12 @@ internal int GetExpectedBlockSize(bool computeTransactionSize) /// Ordered transactions internal void EnsureMaxBlockSize(IEnumerable txs) { - var transactions = txs.ToList(); uint maxBlockSize = NativeContract.Policy.GetMaxBlockSize(Snapshot); + uint maxTransactionsPerBlock = NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot); // Limit Speaker proposal to the limit `MaxTransactionsPerBlock` or all available transactions of the mempool - TransactionHashes = new UInt256[Math.Min(transactions.Count, NativeContract.Policy.GetMaxTransactionsPerBlock(Snapshot))]; + txs = txs.Take((int)maxTransactionsPerBlock); + List hashes = new List(); Transactions = new Dictionary(); Block.Transactions = new Transaction[0]; @@ -259,22 +260,17 @@ internal void EnsureMaxBlockSize(IEnumerable txs) // Iterate transaction until reach the size - for (int x = 0, max = TransactionHashes.Length; x < max; x++) + foreach (Transaction tx in txs) { - var tx = transactions[x]; - // Check if maximum block size has been already exceeded with the current selected set blockSize += tx.Size; if (blockSize > maxBlockSize) break; - TransactionHashes[x] = tx.Hash; + hashes.Add(tx.Hash); Transactions.Add(tx.Hash, tx); } - - // Truncate null values [Tx1,Tx2,null,null,null] - - if (TransactionHashes.Length > Transactions.Count) - Array.Resize(ref TransactionHashes, Transactions.Count); + + TransactionHashes = hashes.ToArray(); } public ConsensusPayload MakePrepareRequest() From b8e80b0c1f95470aefe129f5a19e8213aaabb1a0 Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 8 Aug 2019 11:07:32 +0200 Subject: [PATCH 34/41] Fix unit tests --- .../Consensus/UT_ConsensusContext.cs | 2 +- neo/Consensus/ConsensusContext.cs | 27 +++++++++++-------- neo/Consensus/ConsensusService.cs | 2 +- 3 files changed, 18 insertions(+), 13 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 481fb9571b..324c608917 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -155,7 +155,7 @@ private void EnsureContext(ConsensusContext context, params Transaction[] expect var block = SignBlock(context); - Assert.AreEqual(context.GetExpectedBlockSize(true), block.Size); + Assert.AreEqual(context.GetExpectedBlockSizeWithTransactions(), block.Size); Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); } } diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 5be1849b9f..060e3eede1 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -210,8 +210,20 @@ private void SignPayload(ConsensusPayload payload) /// /// Return the expected block size /// - /// Compute the block size with the Transactions - internal int GetExpectedBlockSize(bool computeTransactionSize) + internal int GetExpectedBlockSizeWithTransactions() + { + var blockSize = GetExpectedBlockSizeWithoutTransactions(Transactions.Count); + + // Sum Txs + + return blockSize + Transactions.Values.Sum(u => u.Size); + } + + /// + /// Return the expected block size without txs + /// + /// Expected transactions + internal int GetExpectedBlockSizeWithoutTransactions(int expectedTransactions) { var blockSize = // BlockBase @@ -227,14 +239,7 @@ internal int GetExpectedBlockSize(bool computeTransactionSize) blockSize += // Block Block.ConsensusData.Size + //ConsensusData - IO.Helper.GetVarSize(TransactionHashes.Length + 1); // Transactions count - - // Txs - - if (computeTransactionSize) - { - blockSize += Transactions.Values.Sum(u => u.Size); - } + IO.Helper.GetVarSize(expectedTransactions + 1); // Transactions count return blockSize; } @@ -256,7 +261,7 @@ internal void EnsureMaxBlockSize(IEnumerable txs) // We need to know the expected block size - var blockSize = GetExpectedBlockSize(false); + var blockSize = GetExpectedBlockSizeWithoutTransactions(txs.Count()); // Iterate transaction until reach the size diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index beab42f07c..1dc9215ea5 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -82,7 +82,7 @@ private bool AddTransaction(Transaction tx, bool verify) // Check policy - if (context.GetExpectedBlockSize(true) > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + if (context.GetExpectedBlockSizeWithTransactions() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); From 44de1516940b8803693e856c98efcbb3010aab7a Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 8 Aug 2019 11:08:32 +0200 Subject: [PATCH 35/41] Rename --- neo.UnitTests/Consensus/UT_ConsensusContext.cs | 2 +- neo/Consensus/ConsensusContext.cs | 2 +- neo/Consensus/ConsensusService.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 324c608917..9f0c6e8531 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -155,7 +155,7 @@ private void EnsureContext(ConsensusContext context, params Transaction[] expect var block = SignBlock(context); - Assert.AreEqual(context.GetExpectedBlockSizeWithTransactions(), block.Size); + Assert.AreEqual(context.GetExpectedBlockSize(), block.Size); Assert.IsTrue(block.Size < NativeContract.Policy.GetMaxBlockSize(context.Snapshot)); } } diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 060e3eede1..84c2dc86b4 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -210,7 +210,7 @@ private void SignPayload(ConsensusPayload payload) /// /// Return the expected block size /// - internal int GetExpectedBlockSizeWithTransactions() + internal int GetExpectedBlockSize() { var blockSize = GetExpectedBlockSizeWithoutTransactions(Transactions.Count); diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index 1dc9215ea5..a17e0195ad 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -82,7 +82,7 @@ private bool AddTransaction(Transaction tx, bool verify) // Check policy - if (context.GetExpectedBlockSizeWithTransactions() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + if (context.GetExpectedBlockSize() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); From fa5b74398404706f924e646c546ffef1596ece2b Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 8 Aug 2019 11:11:22 +0200 Subject: [PATCH 36/41] Indent --- neo/Consensus/ConsensusContext.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 84c2dc86b4..18eb357589 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -238,8 +238,8 @@ internal int GetExpectedBlockSizeWithoutTransactions(int expectedTransactions) blockSize += // Block - Block.ConsensusData.Size + //ConsensusData - IO.Helper.GetVarSize(expectedTransactions + 1); // Transactions count + Block.ConsensusData.Size + //ConsensusData + IO.Helper.GetVarSize(expectedTransactions + 1); //Transactions count return blockSize; } From 9aa49b454d4e55a0a7f4d082718db7b07e5a2435 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 12 Aug 2019 09:54:52 +0200 Subject: [PATCH 37/41] Vitor suggestion --- neo/Consensus/ConsensusContext.cs | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/neo/Consensus/ConsensusContext.cs b/neo/Consensus/ConsensusContext.cs index 18eb357589..a06c6c965b 100644 --- a/neo/Consensus/ConsensusContext.cs +++ b/neo/Consensus/ConsensusContext.cs @@ -205,18 +205,15 @@ private void SignPayload(ConsensusPayload payload) return; } payload.Witness = sc.GetWitnesses()[0]; - } - - /// - /// Return the expected block size - /// + } + + /// + /// Return the expected block size + /// internal int GetExpectedBlockSize() { - var blockSize = GetExpectedBlockSizeWithoutTransactions(Transactions.Count); - - // Sum Txs - - return blockSize + Transactions.Values.Sum(u => u.Size); + return GetExpectedBlockSizeWithoutTransactions(Transactions.Count) + // Base size + Transactions.Values.Sum(u => u.Size); // Sum Txs } /// From dc9fd12ef5b748bcfe42cc843ae4fc9b77eb2234 Mon Sep 17 00:00:00 2001 From: Shargon Date: Mon, 12 Aug 2019 12:06:51 +0200 Subject: [PATCH 38/41] Merge with Scoped signatures --- neo.UnitTests/Consensus/UT_ConsensusContext.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/neo.UnitTests/Consensus/UT_ConsensusContext.cs b/neo.UnitTests/Consensus/UT_ConsensusContext.cs index 9f0c6e8531..696d6aaa0a 100644 --- a/neo.UnitTests/Consensus/UT_ConsensusContext.cs +++ b/neo.UnitTests/Consensus/UT_ConsensusContext.cs @@ -98,6 +98,7 @@ private Transaction CreateTransactionWithSize(int v) var r = new Random(); var tx = new Transaction() { + Cosigners = new Cosigner[0], Attributes = new TransactionAttribute[0], NetworkFee = 0, Nonce = (uint)Environment.TickCount, From e1348813a488e3be736f177e20df79e1efa01c6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Vitor=20Naz=C3=A1rio=20Coelho?= Date: Mon, 12 Aug 2019 11:06:37 -0300 Subject: [PATCH 39/41] Remove extra line --- neo/Consensus/ConsensusService.cs | 1307 ++++++++++++++--------------- 1 file changed, 653 insertions(+), 654 deletions(-) diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index cb6e6e1dc4..eb30880022 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -1,661 +1,660 @@ -using Akka.Actor; -using Akka.Configuration; -using Neo.Cryptography; -using Neo.IO; -using Neo.IO.Actors; -using Neo.Ledger; -using Neo.Network.P2P; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract.Native; -using Neo.Wallets; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace Neo.Consensus -{ - public sealed class ConsensusService : UntypedActor - { - public class Start { public bool IgnoreRecoveryLogs; } - public class SetViewNumber { public byte ViewNumber; } - internal class Timer { public uint Height; public byte ViewNumber; } - - private readonly ConsensusContext context; - private readonly IActorRef localNode; - private readonly IActorRef taskManager; - private ICancelable timer_token; - private DateTime block_received_time; - private bool started = false; - - /// - /// This will record the information from last scheduled timer - /// - private DateTime clock_started = TimeProvider.Current.UtcNow; - private TimeSpan expected_delay = TimeSpan.Zero; - - /// - /// This will be cleared every block (so it will not grow out of control, but is used to prevent repeatedly - /// responding to the same message. - /// - private readonly HashSet knownHashes = new HashSet(); - /// - /// This variable is only true during OnRecoveryMessageReceived - /// - private bool isRecovering = false; - - public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) - : this(localNode, taskManager, new ConsensusContext(wallet, store)) - { - } - - internal ConsensusService(IActorRef localNode, IActorRef taskManager, ConsensusContext context) - { - this.localNode = localNode; - this.taskManager = taskManager; - this.context = context; - Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); - } - - private bool AddTransaction(Transaction tx, bool verify) - { - if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values)) - { - Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxInvalid); - return false; - } - if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot)) - { - Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxRejectedByPolicy); - return false; - } - context.Transactions[tx.Hash] = tx; - if (context.TransactionHashes.Length == context.Transactions.Count) - { - // if we are the primary for this view, but acting as a backup because we recovered our own - // previously sent prepare request, then we don't want to send a prepare response. - if (context.IsPrimary || context.WatchOnly) return true; +using Akka.Actor; +using Akka.Configuration; +using Neo.Cryptography; +using Neo.IO; +using Neo.IO.Actors; +using Neo.Ledger; +using Neo.Network.P2P; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Neo.Consensus +{ + public sealed class ConsensusService : UntypedActor + { + public class Start { public bool IgnoreRecoveryLogs; } + public class SetViewNumber { public byte ViewNumber; } + internal class Timer { public uint Height; public byte ViewNumber; } + + private readonly ConsensusContext context; + private readonly IActorRef localNode; + private readonly IActorRef taskManager; + private ICancelable timer_token; + private DateTime block_received_time; + private bool started = false; + + /// + /// This will record the information from last scheduled timer + /// + private DateTime clock_started = TimeProvider.Current.UtcNow; + private TimeSpan expected_delay = TimeSpan.Zero; + + /// + /// This will be cleared every block (so it will not grow out of control, but is used to prevent repeatedly + /// responding to the same message. + /// + private readonly HashSet knownHashes = new HashSet(); + /// + /// This variable is only true during OnRecoveryMessageReceived + /// + private bool isRecovering = false; - // Check policy + public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) + : this(localNode, taskManager, new ConsensusContext(wallet, store)) + { + } + internal ConsensusService(IActorRef localNode, IActorRef taskManager, ConsensusContext context) + { + this.localNode = localNode; + this.taskManager = taskManager; + this.context = context; + Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); + } + + private bool AddTransaction(Transaction tx, bool verify) + { + if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values)) + { + Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxInvalid); + return false; + } + if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot)) + { + Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxRejectedByPolicy); + return false; + } + context.Transactions[tx.Hash] = tx; + if (context.TransactionHashes.Length == context.Transactions.Count) + { + // if we are the primary for this view, but acting as a backup because we recovered our own + // previously sent prepare request, then we don't want to send a prepare response. + if (context.IsPrimary || context.WatchOnly) return true; + + // Check maximum block size via Native.Contract policy if (context.GetExpectedBlockSize() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); return false; - } - - // Timeout extension due to prepare response sent - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - Log($"send prepare response"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse() }); - CheckPreparations(); - } - return true; - } - - private void ChangeTimer(TimeSpan delay) - { - clock_started = TimeProvider.Current.UtcNow; - expected_delay = delay; - timer_token.CancelIfNotNull(); - timer_token = Context.System.Scheduler.ScheduleTellOnceCancelable(delay, Self, new Timer - { - Height = context.Block.Index, - ViewNumber = context.ViewNumber - }, ActorRefs.NoSender); - } - - private void CheckCommits() - { - if (context.CommitPayloads.Count(p => p?.ConsensusMessage.ViewNumber == context.ViewNumber) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) - { - Block block = context.CreateBlock(); - Log($"relay block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); - localNode.Tell(new LocalNode.Relay { Inventory = block }); - } - } - - private void CheckExpectedView(byte viewNumber) - { - if (context.ViewNumber >= viewNumber) return; - // if there are `M` change view payloads with NewViewNumber greater than viewNumber, then, it is safe to move - if (context.ChangeViewPayloads.Count(p => p != null && p.GetDeserializedMessage().NewViewNumber >= viewNumber) >= context.M) - { - if (!context.WatchOnly) - { - ChangeView message = context.ChangeViewPayloads[context.MyIndex]?.GetDeserializedMessage(); - // Communicate the network about my agreement to move to `viewNumber` - // if my last change view payload, `message`, has NewViewNumber lower than current view to change - if (message is null || message.NewViewNumber < viewNumber) - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(ChangeViewReason.ChangeAgreement) }); - } - InitializeConsensus(viewNumber); - } - } - - private void CheckPreparations() - { - if (context.PreparationPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) - { - ConsensusPayload payload = context.MakeCommit(); - Log($"send commit"); - context.Save(); - localNode.Tell(new LocalNode.SendDirectly { Inventory = payload }); - // Set timer, so we will resend the commit in case of a networking issue - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock)); - CheckCommits(); - } - } - - private void InitializeConsensus(byte viewNumber) - { - context.Reset(viewNumber); - if (viewNumber > 0) - Log($"changeview: view={viewNumber} primary={context.Validators[context.GetPrimaryIndex((byte)(viewNumber - 1u))]}", LogLevel.Warning); - Log($"initialize: height={context.Block.Index} view={viewNumber} index={context.MyIndex} role={(context.IsPrimary ? "Primary" : context.WatchOnly ? "WatchOnly" : "Backup")}"); - if (context.WatchOnly) return; - if (context.IsPrimary) - { - if (isRecovering) - { - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); - } - else - { - TimeSpan span = TimeProvider.Current.UtcNow - block_received_time; - if (span >= Blockchain.TimePerBlock) - ChangeTimer(TimeSpan.Zero); - else - ChangeTimer(Blockchain.TimePerBlock - span); - } - } - else - { - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); - } - } - - private void Log(string message, LogLevel level = LogLevel.Info) - { - Console.WriteLine($"[{DateTime.Now.TimeOfDay:hh\\:mm\\:ss\\.fff}] {message}"); - Plugin.Log(nameof(ConsensusService), level, message); - } - - private void OnChangeViewReceived(ConsensusPayload payload, ChangeView message) - { - if (message.NewViewNumber <= context.ViewNumber) - OnRecoveryRequestReceived(payload); - - if (context.CommitSent) return; - - var expectedView = context.ChangeViewPayloads[payload.ValidatorIndex]?.GetDeserializedMessage().NewViewNumber ?? (byte)0; - if (message.NewViewNumber <= expectedView) - return; - - Log($"{nameof(OnChangeViewReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} nv={message.NewViewNumber}"); - context.ChangeViewPayloads[payload.ValidatorIndex] = payload; - CheckExpectedView(message.NewViewNumber); - } - - private void OnCommitReceived(ConsensusPayload payload, Commit commit) - { - ref ConsensusPayload existingCommitPayload = ref context.CommitPayloads[payload.ValidatorIndex]; - if (existingCommitPayload != null) - { - if (existingCommitPayload.Hash != payload.Hash) - Log($"{nameof(OnCommitReceived)}: different commit from validator! height={payload.BlockIndex} index={payload.ValidatorIndex} view={commit.ViewNumber} existingView={existingCommitPayload.ConsensusMessage.ViewNumber}", LogLevel.Warning); - return; - } - - // Timeout extension: commit has been received with success - // around 4*15s/M=60.0s/5=12.0s ~ 80% block time (for M=5) - ExtendTimerByFactor(4); - - if (commit.ViewNumber == context.ViewNumber) - { - Log($"{nameof(OnCommitReceived)}: height={payload.BlockIndex} view={commit.ViewNumber} index={payload.ValidatorIndex} nc={context.CountCommitted} nf={context.CountFailed}"); - - byte[] hashData = context.EnsureHeader()?.GetHashData(); - if (hashData == null) - { - existingCommitPayload = payload; - } - else if (Crypto.Default.VerifySignature(hashData, commit.Signature, - context.Validators[payload.ValidatorIndex].EncodePoint(false))) - { - existingCommitPayload = payload; - CheckCommits(); - } - return; - } - // Receiving commit from another view - Log($"{nameof(OnCommitReceived)}: record commit for different view={commit.ViewNumber} index={payload.ValidatorIndex} height={payload.BlockIndex}"); - existingCommitPayload = payload; - } - - // this function increases existing timer (never decreases) with a value proportional to `maxDelayInBlockTimes`*`Blockchain.MillisecondsPerBlock` - private void ExtendTimerByFactor(int maxDelayInBlockTimes) - { - TimeSpan nextDelay = expected_delay - (TimeProvider.Current.UtcNow - clock_started) + TimeSpan.FromMilliseconds(maxDelayInBlockTimes * Blockchain.MillisecondsPerBlock / context.M); - if (!context.WatchOnly && !context.ViewChanging && !context.CommitSent && (nextDelay > TimeSpan.Zero)) - ChangeTimer(nextDelay); - } - - private void OnConsensusPayload(ConsensusPayload payload) - { - if (context.BlockSent) return; - if (payload.Version != context.Block.Version) return; - if (payload.PrevHash != context.Block.PrevHash || payload.BlockIndex != context.Block.Index) - { - if (context.Block.Index < payload.BlockIndex) - { - Log($"chain sync: expected={payload.BlockIndex} current={context.Block.Index - 1} nodes={LocalNode.Singleton.ConnectedCount}", LogLevel.Warning); - } - return; - } - if (payload.ValidatorIndex >= context.Validators.Length) return; - ConsensusMessage message; - try - { - message = payload.ConsensusMessage; - } - catch (FormatException) - { - return; - } - catch (IOException) - { - return; - } - context.LastSeenMessage[payload.ValidatorIndex] = (int)payload.BlockIndex; - foreach (IP2PPlugin plugin in Plugin.P2PPlugins) - if (!plugin.OnConsensusMessage(payload)) - return; - switch (message) - { - case ChangeView view: - OnChangeViewReceived(payload, view); - break; - case PrepareRequest request: - OnPrepareRequestReceived(payload, request); - break; - case PrepareResponse response: - OnPrepareResponseReceived(payload, response); - break; - case Commit commit: - OnCommitReceived(payload, commit); - break; - case RecoveryRequest _: - OnRecoveryRequestReceived(payload); - break; - case RecoveryMessage recovery: - OnRecoveryMessageReceived(payload, recovery); - break; - } - } - - private void OnPersistCompleted(Block block) - { - Log($"persist block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); - block_received_time = TimeProvider.Current.UtcNow; - knownHashes.Clear(); - InitializeConsensus(0); - } - - private void OnRecoveryMessageReceived(ConsensusPayload payload, RecoveryMessage message) - { - // isRecovering is always set to false again after OnRecoveryMessageReceived - isRecovering = true; - int validChangeViews = 0, totalChangeViews = 0, validPrepReq = 0, totalPrepReq = 0; - int validPrepResponses = 0, totalPrepResponses = 0, validCommits = 0, totalCommits = 0; - - Log($"{nameof(OnRecoveryMessageReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); - try - { - if (message.ViewNumber > context.ViewNumber) - { - if (context.CommitSent) return; - ConsensusPayload[] changeViewPayloads = message.GetChangeViewPayloads(context, payload); - totalChangeViews = changeViewPayloads.Length; - foreach (ConsensusPayload changeViewPayload in changeViewPayloads) - if (ReverifyAndProcessPayload(changeViewPayload)) validChangeViews++; - } - if (message.ViewNumber == context.ViewNumber && !context.NotAcceptingPayloadsDueToViewChanging && !context.CommitSent) - { - if (!context.RequestSentOrReceived) - { - ConsensusPayload prepareRequestPayload = message.GetPrepareRequestPayload(context, payload); - if (prepareRequestPayload != null) - { - totalPrepReq = 1; - if (ReverifyAndProcessPayload(prepareRequestPayload)) validPrepReq++; - } - else if (context.IsPrimary) - SendPrepareRequest(); - } - ConsensusPayload[] prepareResponsePayloads = message.GetPrepareResponsePayloads(context, payload); - totalPrepResponses = prepareResponsePayloads.Length; - foreach (ConsensusPayload prepareResponsePayload in prepareResponsePayloads) - if (ReverifyAndProcessPayload(prepareResponsePayload)) validPrepResponses++; - } - if (message.ViewNumber <= context.ViewNumber) - { - // Ensure we know about all commits from lower view numbers. - ConsensusPayload[] commitPayloads = message.GetCommitPayloadsFromRecoveryMessage(context, payload); - totalCommits = commitPayloads.Length; - foreach (ConsensusPayload commitPayload in commitPayloads) - if (ReverifyAndProcessPayload(commitPayload)) validCommits++; - } - } - finally - { - Log($"{nameof(OnRecoveryMessageReceived)}: finished (valid/total) " + - $"ChgView: {validChangeViews}/{totalChangeViews} " + - $"PrepReq: {validPrepReq}/{totalPrepReq} " + - $"PrepResp: {validPrepResponses}/{totalPrepResponses} " + - $"Commits: {validCommits}/{totalCommits}"); - isRecovering = false; - } - } - - private void OnRecoveryRequestReceived(ConsensusPayload payload) - { - // We keep track of the payload hashes received in this block, and don't respond with recovery - // in response to the same payload that we already responded to previously. - // ChangeView messages include a Timestamp when the change view is sent, thus if a node restarts - // and issues a change view for the same view, it will have a different hash and will correctly respond - // again; however replay attacks of the ChangeView message from arbitrary nodes will not trigger an - // additional recovery message response. - if (!knownHashes.Add(payload.Hash)) return; - - Log($"On{payload.ConsensusMessage.GetType().Name}Received: height={payload.BlockIndex} index={payload.ValidatorIndex} view={payload.ConsensusMessage.ViewNumber}"); - if (context.WatchOnly) return; - if (!context.CommitSent) - { - bool shouldSendRecovery = false; - int allowedRecoveryNodeCount = context.F; - // Limit recoveries to be sent from an upper limit of `f` nodes - for (int i = 1; i <= allowedRecoveryNodeCount; i++) - { - var chosenIndex = (payload.ValidatorIndex + i) % context.Validators.Length; - if (chosenIndex != context.MyIndex) continue; - shouldSendRecovery = true; - break; - } - - if (!shouldSendRecovery) return; - } - Log($"send recovery: view={context.ViewNumber}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); - } - - private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest message) - { - if (context.RequestSentOrReceived || context.NotAcceptingPayloadsDueToViewChanging) return; - if (payload.ValidatorIndex != context.Block.ConsensusData.PrimaryIndex || message.ViewNumber != context.ViewNumber) return; - Log($"{nameof(OnPrepareRequestReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} tx={message.TransactionHashes.Length}"); - if (message.Timestamp <= context.PrevHeader.Timestamp || message.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestampMS()) - { - Log($"Timestamp incorrect: {message.Timestamp}", LogLevel.Warning); - return; - } - if (message.TransactionHashes.Any(p => context.Snapshot.ContainsTransaction(p))) - { - Log($"Invalid request: transaction already exists", LogLevel.Warning); - return; - } - - // Timeout extension: prepare request has been received with success - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - context.Block.Timestamp = message.Timestamp; - context.Block.ConsensusData.Nonce = message.Nonce; - context.TransactionHashes = message.TransactionHashes; - context.Transactions = new Dictionary(); - for (int i = 0; i < context.PreparationPayloads.Length; i++) - if (context.PreparationPayloads[i] != null) - if (!context.PreparationPayloads[i].GetDeserializedMessage().PreparationHash.Equals(payload.Hash)) - context.PreparationPayloads[i] = null; - context.PreparationPayloads[payload.ValidatorIndex] = payload; - byte[] hashData = context.EnsureHeader().GetHashData(); - for (int i = 0; i < context.CommitPayloads.Length; i++) - if (context.CommitPayloads[i]?.ConsensusMessage.ViewNumber == context.ViewNumber) - if (!Crypto.Default.VerifySignature(hashData, context.CommitPayloads[i].GetDeserializedMessage().Signature, context.Validators[i].EncodePoint(false))) - context.CommitPayloads[i] = null; - Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); - List unverified = new List(); - foreach (UInt256 hash in context.TransactionHashes) - { - if (mempoolVerified.TryGetValue(hash, out Transaction tx)) - { - if (!AddTransaction(tx, false)) - return; - } - else - { - if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) - unverified.Add(tx); - } - } - foreach (Transaction tx in unverified) - if (!AddTransaction(tx, true)) - return; - if (context.Transactions.Count < context.TransactionHashes.Length) - { - UInt256[] hashes = context.TransactionHashes.Where(i => !context.Transactions.ContainsKey(i)).ToArray(); - taskManager.Tell(new TaskManager.RestartTasks - { - Payload = InvPayload.Create(InventoryType.TX, hashes) - }); - } - } - - private void OnPrepareResponseReceived(ConsensusPayload payload, PrepareResponse message) - { - if (message.ViewNumber != context.ViewNumber) return; - if (context.PreparationPayloads[payload.ValidatorIndex] != null || context.NotAcceptingPayloadsDueToViewChanging) return; - if (context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex] != null && !message.PreparationHash.Equals(context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex].Hash)) - return; - - // Timeout extension: prepare response has been received with success - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - Log($"{nameof(OnPrepareResponseReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); - context.PreparationPayloads[payload.ValidatorIndex] = payload; - if (context.WatchOnly || context.CommitSent) return; - if (context.RequestSentOrReceived) - CheckPreparations(); - } - - protected override void OnReceive(object message) - { - if (message is Start options) - { - if (started) return; - OnStart(options); - } - else - { - if (!started) return; - switch (message) - { - case SetViewNumber setView: - InitializeConsensus(setView.ViewNumber); - break; - case Timer timer: - OnTimer(timer); - break; - case ConsensusPayload payload: - OnConsensusPayload(payload); - break; - case Transaction transaction: - OnTransaction(transaction); - break; - case Blockchain.PersistCompleted completed: - OnPersistCompleted(completed.Block); - break; - } - } - } - - private void RequestRecovery() - { - if (context.Block.Index == Blockchain.Singleton.HeaderHeight + 1) - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryRequest() }); - } - - private void OnStart(Start options) - { - Log("OnStart"); - started = true; - if (!options.IgnoreRecoveryLogs && context.Load()) - { - if (context.Transactions != null) - { - Sender.Ask(new Blockchain.FillMemoryPool - { - Transactions = context.Transactions.Values - }).Wait(); - } - if (context.CommitSent) - { - CheckPreparations(); - return; - } - } - InitializeConsensus(0); - // Issue a ChangeView with NewViewNumber of 0 to request recovery messages on start-up. - if (!context.WatchOnly) - RequestRecovery(); - } - - private void OnTimer(Timer timer) - { - if (context.WatchOnly || context.BlockSent) return; - if (timer.Height != context.Block.Index || timer.ViewNumber != context.ViewNumber) return; - Log($"timeout: height={timer.Height} view={timer.ViewNumber}"); - if (context.IsPrimary && !context.RequestSentOrReceived) - { - SendPrepareRequest(); - } - else if ((context.IsPrimary && context.RequestSentOrReceived) || context.IsBackup) - { - if (context.CommitSent) - { - // Re-send commit periodically by sending recover message in case of a network issue. - Log($"send recovery to resend commit"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << 1)); - } - else - { - var reason = ChangeViewReason.Timeout; - - if (context.Block != null && context.TransactionHashes?.Count() > context.Transactions?.Count) - { - reason = ChangeViewReason.TxNotFound; - } - - RequestChangeView(reason); - } - } - } - - private void OnTransaction(Transaction transaction) - { - if (!context.IsBackup || context.NotAcceptingPayloadsDueToViewChanging || !context.RequestSentOrReceived || context.ResponseSent || context.BlockSent) - return; - if (context.Transactions.ContainsKey(transaction.Hash)) return; - if (!context.TransactionHashes.Contains(transaction.Hash)) return; - AddTransaction(transaction, true); - } - - protected override void PostStop() - { - Log("OnStop"); - started = false; - Context.System.EventStream.Unsubscribe(Self); - context.Dispose(); - base.PostStop(); - } - - public static Props Props(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) - { - return Akka.Actor.Props.Create(() => new ConsensusService(localNode, taskManager, store, wallet)).WithMailbox("consensus-service-mailbox"); - } - - private void RequestChangeView(ChangeViewReason reason) - { - if (context.WatchOnly) return; - // Request for next view is always one view more than the current context.ViewNumber - // Nodes will not contribute for changing to a view higher than (context.ViewNumber+1), unless they are recovered - // The latter may happen by nodes in higher views with, at least, `M` proofs - byte expectedView = context.ViewNumber; - expectedView++; - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (expectedView + 1))); - if ((context.CountCommitted + context.CountFailed) > context.F) - { - Log($"Skip requesting change view to nv={expectedView} because nc={context.CountCommitted} nf={context.CountFailed}"); - RequestRecovery(); - return; - } - Log($"request change view: height={context.Block.Index} view={context.ViewNumber} nv={expectedView} nc={context.CountCommitted} nf={context.CountFailed}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(reason) }); - CheckExpectedView(expectedView); - } - - private bool ReverifyAndProcessPayload(ConsensusPayload payload) - { - if (!payload.Verify(context.Snapshot)) return false; - OnConsensusPayload(payload); - return true; - } - - private void SendPrepareRequest() - { - Log($"send prepare request: height={context.Block.Index} view={context.ViewNumber}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareRequest() }); - - if (context.Validators.Length == 1) - CheckPreparations(); - - if (context.TransactionHashes.Length > 0) - { - foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) - localNode.Tell(Message.Create(MessageCommand.Inv, payload)); - } - ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); - } - } - - internal class ConsensusServiceMailbox : PriorityMailbox - { - public ConsensusServiceMailbox(Akka.Actor.Settings settings, Config config) - : base(settings, config) - { - } - - internal protected override bool IsHighPriority(object message) - { - switch (message) - { - case ConsensusPayload _: - case ConsensusService.SetViewNumber _: - case ConsensusService.Timer _: - case Blockchain.PersistCompleted _: - return true; - default: - return false; - } - } - } -} + } + + // Timeout extension due to prepare response sent + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + Log($"send prepare response"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse() }); + CheckPreparations(); + } + return true; + } + + private void ChangeTimer(TimeSpan delay) + { + clock_started = TimeProvider.Current.UtcNow; + expected_delay = delay; + timer_token.CancelIfNotNull(); + timer_token = Context.System.Scheduler.ScheduleTellOnceCancelable(delay, Self, new Timer + { + Height = context.Block.Index, + ViewNumber = context.ViewNumber + }, ActorRefs.NoSender); + } + + private void CheckCommits() + { + if (context.CommitPayloads.Count(p => p?.ConsensusMessage.ViewNumber == context.ViewNumber) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + { + Block block = context.CreateBlock(); + Log($"relay block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); + localNode.Tell(new LocalNode.Relay { Inventory = block }); + } + } + + private void CheckExpectedView(byte viewNumber) + { + if (context.ViewNumber >= viewNumber) return; + // if there are `M` change view payloads with NewViewNumber greater than viewNumber, then, it is safe to move + if (context.ChangeViewPayloads.Count(p => p != null && p.GetDeserializedMessage().NewViewNumber >= viewNumber) >= context.M) + { + if (!context.WatchOnly) + { + ChangeView message = context.ChangeViewPayloads[context.MyIndex]?.GetDeserializedMessage(); + // Communicate the network about my agreement to move to `viewNumber` + // if my last change view payload, `message`, has NewViewNumber lower than current view to change + if (message is null || message.NewViewNumber < viewNumber) + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(ChangeViewReason.ChangeAgreement) }); + } + InitializeConsensus(viewNumber); + } + } + + private void CheckPreparations() + { + if (context.PreparationPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + { + ConsensusPayload payload = context.MakeCommit(); + Log($"send commit"); + context.Save(); + localNode.Tell(new LocalNode.SendDirectly { Inventory = payload }); + // Set timer, so we will resend the commit in case of a networking issue + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock)); + CheckCommits(); + } + } + + private void InitializeConsensus(byte viewNumber) + { + context.Reset(viewNumber); + if (viewNumber > 0) + Log($"changeview: view={viewNumber} primary={context.Validators[context.GetPrimaryIndex((byte)(viewNumber - 1u))]}", LogLevel.Warning); + Log($"initialize: height={context.Block.Index} view={viewNumber} index={context.MyIndex} role={(context.IsPrimary ? "Primary" : context.WatchOnly ? "WatchOnly" : "Backup")}"); + if (context.WatchOnly) return; + if (context.IsPrimary) + { + if (isRecovering) + { + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); + } + else + { + TimeSpan span = TimeProvider.Current.UtcNow - block_received_time; + if (span >= Blockchain.TimePerBlock) + ChangeTimer(TimeSpan.Zero); + else + ChangeTimer(Blockchain.TimePerBlock - span); + } + } + else + { + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); + } + } + + private void Log(string message, LogLevel level = LogLevel.Info) + { + Console.WriteLine($"[{DateTime.Now.TimeOfDay:hh\\:mm\\:ss\\.fff}] {message}"); + Plugin.Log(nameof(ConsensusService), level, message); + } + + private void OnChangeViewReceived(ConsensusPayload payload, ChangeView message) + { + if (message.NewViewNumber <= context.ViewNumber) + OnRecoveryRequestReceived(payload); + + if (context.CommitSent) return; + + var expectedView = context.ChangeViewPayloads[payload.ValidatorIndex]?.GetDeserializedMessage().NewViewNumber ?? (byte)0; + if (message.NewViewNumber <= expectedView) + return; + + Log($"{nameof(OnChangeViewReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} nv={message.NewViewNumber}"); + context.ChangeViewPayloads[payload.ValidatorIndex] = payload; + CheckExpectedView(message.NewViewNumber); + } + + private void OnCommitReceived(ConsensusPayload payload, Commit commit) + { + ref ConsensusPayload existingCommitPayload = ref context.CommitPayloads[payload.ValidatorIndex]; + if (existingCommitPayload != null) + { + if (existingCommitPayload.Hash != payload.Hash) + Log($"{nameof(OnCommitReceived)}: different commit from validator! height={payload.BlockIndex} index={payload.ValidatorIndex} view={commit.ViewNumber} existingView={existingCommitPayload.ConsensusMessage.ViewNumber}", LogLevel.Warning); + return; + } + + // Timeout extension: commit has been received with success + // around 4*15s/M=60.0s/5=12.0s ~ 80% block time (for M=5) + ExtendTimerByFactor(4); + + if (commit.ViewNumber == context.ViewNumber) + { + Log($"{nameof(OnCommitReceived)}: height={payload.BlockIndex} view={commit.ViewNumber} index={payload.ValidatorIndex} nc={context.CountCommitted} nf={context.CountFailed}"); + + byte[] hashData = context.EnsureHeader()?.GetHashData(); + if (hashData == null) + { + existingCommitPayload = payload; + } + else if (Crypto.Default.VerifySignature(hashData, commit.Signature, + context.Validators[payload.ValidatorIndex].EncodePoint(false))) + { + existingCommitPayload = payload; + CheckCommits(); + } + return; + } + // Receiving commit from another view + Log($"{nameof(OnCommitReceived)}: record commit for different view={commit.ViewNumber} index={payload.ValidatorIndex} height={payload.BlockIndex}"); + existingCommitPayload = payload; + } + + // this function increases existing timer (never decreases) with a value proportional to `maxDelayInBlockTimes`*`Blockchain.MillisecondsPerBlock` + private void ExtendTimerByFactor(int maxDelayInBlockTimes) + { + TimeSpan nextDelay = expected_delay - (TimeProvider.Current.UtcNow - clock_started) + TimeSpan.FromMilliseconds(maxDelayInBlockTimes * Blockchain.MillisecondsPerBlock / context.M); + if (!context.WatchOnly && !context.ViewChanging && !context.CommitSent && (nextDelay > TimeSpan.Zero)) + ChangeTimer(nextDelay); + } + + private void OnConsensusPayload(ConsensusPayload payload) + { + if (context.BlockSent) return; + if (payload.Version != context.Block.Version) return; + if (payload.PrevHash != context.Block.PrevHash || payload.BlockIndex != context.Block.Index) + { + if (context.Block.Index < payload.BlockIndex) + { + Log($"chain sync: expected={payload.BlockIndex} current={context.Block.Index - 1} nodes={LocalNode.Singleton.ConnectedCount}", LogLevel.Warning); + } + return; + } + if (payload.ValidatorIndex >= context.Validators.Length) return; + ConsensusMessage message; + try + { + message = payload.ConsensusMessage; + } + catch (FormatException) + { + return; + } + catch (IOException) + { + return; + } + context.LastSeenMessage[payload.ValidatorIndex] = (int)payload.BlockIndex; + foreach (IP2PPlugin plugin in Plugin.P2PPlugins) + if (!plugin.OnConsensusMessage(payload)) + return; + switch (message) + { + case ChangeView view: + OnChangeViewReceived(payload, view); + break; + case PrepareRequest request: + OnPrepareRequestReceived(payload, request); + break; + case PrepareResponse response: + OnPrepareResponseReceived(payload, response); + break; + case Commit commit: + OnCommitReceived(payload, commit); + break; + case RecoveryRequest _: + OnRecoveryRequestReceived(payload); + break; + case RecoveryMessage recovery: + OnRecoveryMessageReceived(payload, recovery); + break; + } + } + + private void OnPersistCompleted(Block block) + { + Log($"persist block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); + block_received_time = TimeProvider.Current.UtcNow; + knownHashes.Clear(); + InitializeConsensus(0); + } + + private void OnRecoveryMessageReceived(ConsensusPayload payload, RecoveryMessage message) + { + // isRecovering is always set to false again after OnRecoveryMessageReceived + isRecovering = true; + int validChangeViews = 0, totalChangeViews = 0, validPrepReq = 0, totalPrepReq = 0; + int validPrepResponses = 0, totalPrepResponses = 0, validCommits = 0, totalCommits = 0; + + Log($"{nameof(OnRecoveryMessageReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); + try + { + if (message.ViewNumber > context.ViewNumber) + { + if (context.CommitSent) return; + ConsensusPayload[] changeViewPayloads = message.GetChangeViewPayloads(context, payload); + totalChangeViews = changeViewPayloads.Length; + foreach (ConsensusPayload changeViewPayload in changeViewPayloads) + if (ReverifyAndProcessPayload(changeViewPayload)) validChangeViews++; + } + if (message.ViewNumber == context.ViewNumber && !context.NotAcceptingPayloadsDueToViewChanging && !context.CommitSent) + { + if (!context.RequestSentOrReceived) + { + ConsensusPayload prepareRequestPayload = message.GetPrepareRequestPayload(context, payload); + if (prepareRequestPayload != null) + { + totalPrepReq = 1; + if (ReverifyAndProcessPayload(prepareRequestPayload)) validPrepReq++; + } + else if (context.IsPrimary) + SendPrepareRequest(); + } + ConsensusPayload[] prepareResponsePayloads = message.GetPrepareResponsePayloads(context, payload); + totalPrepResponses = prepareResponsePayloads.Length; + foreach (ConsensusPayload prepareResponsePayload in prepareResponsePayloads) + if (ReverifyAndProcessPayload(prepareResponsePayload)) validPrepResponses++; + } + if (message.ViewNumber <= context.ViewNumber) + { + // Ensure we know about all commits from lower view numbers. + ConsensusPayload[] commitPayloads = message.GetCommitPayloadsFromRecoveryMessage(context, payload); + totalCommits = commitPayloads.Length; + foreach (ConsensusPayload commitPayload in commitPayloads) + if (ReverifyAndProcessPayload(commitPayload)) validCommits++; + } + } + finally + { + Log($"{nameof(OnRecoveryMessageReceived)}: finished (valid/total) " + + $"ChgView: {validChangeViews}/{totalChangeViews} " + + $"PrepReq: {validPrepReq}/{totalPrepReq} " + + $"PrepResp: {validPrepResponses}/{totalPrepResponses} " + + $"Commits: {validCommits}/{totalCommits}"); + isRecovering = false; + } + } + + private void OnRecoveryRequestReceived(ConsensusPayload payload) + { + // We keep track of the payload hashes received in this block, and don't respond with recovery + // in response to the same payload that we already responded to previously. + // ChangeView messages include a Timestamp when the change view is sent, thus if a node restarts + // and issues a change view for the same view, it will have a different hash and will correctly respond + // again; however replay attacks of the ChangeView message from arbitrary nodes will not trigger an + // additional recovery message response. + if (!knownHashes.Add(payload.Hash)) return; + + Log($"On{payload.ConsensusMessage.GetType().Name}Received: height={payload.BlockIndex} index={payload.ValidatorIndex} view={payload.ConsensusMessage.ViewNumber}"); + if (context.WatchOnly) return; + if (!context.CommitSent) + { + bool shouldSendRecovery = false; + int allowedRecoveryNodeCount = context.F; + // Limit recoveries to be sent from an upper limit of `f` nodes + for (int i = 1; i <= allowedRecoveryNodeCount; i++) + { + var chosenIndex = (payload.ValidatorIndex + i) % context.Validators.Length; + if (chosenIndex != context.MyIndex) continue; + shouldSendRecovery = true; + break; + } + + if (!shouldSendRecovery) return; + } + Log($"send recovery: view={context.ViewNumber}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); + } + + private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest message) + { + if (context.RequestSentOrReceived || context.NotAcceptingPayloadsDueToViewChanging) return; + if (payload.ValidatorIndex != context.Block.ConsensusData.PrimaryIndex || message.ViewNumber != context.ViewNumber) return; + Log($"{nameof(OnPrepareRequestReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} tx={message.TransactionHashes.Length}"); + if (message.Timestamp <= context.PrevHeader.Timestamp || message.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestampMS()) + { + Log($"Timestamp incorrect: {message.Timestamp}", LogLevel.Warning); + return; + } + if (message.TransactionHashes.Any(p => context.Snapshot.ContainsTransaction(p))) + { + Log($"Invalid request: transaction already exists", LogLevel.Warning); + return; + } + + // Timeout extension: prepare request has been received with success + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + context.Block.Timestamp = message.Timestamp; + context.Block.ConsensusData.Nonce = message.Nonce; + context.TransactionHashes = message.TransactionHashes; + context.Transactions = new Dictionary(); + for (int i = 0; i < context.PreparationPayloads.Length; i++) + if (context.PreparationPayloads[i] != null) + if (!context.PreparationPayloads[i].GetDeserializedMessage().PreparationHash.Equals(payload.Hash)) + context.PreparationPayloads[i] = null; + context.PreparationPayloads[payload.ValidatorIndex] = payload; + byte[] hashData = context.EnsureHeader().GetHashData(); + for (int i = 0; i < context.CommitPayloads.Length; i++) + if (context.CommitPayloads[i]?.ConsensusMessage.ViewNumber == context.ViewNumber) + if (!Crypto.Default.VerifySignature(hashData, context.CommitPayloads[i].GetDeserializedMessage().Signature, context.Validators[i].EncodePoint(false))) + context.CommitPayloads[i] = null; + Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); + List unverified = new List(); + foreach (UInt256 hash in context.TransactionHashes) + { + if (mempoolVerified.TryGetValue(hash, out Transaction tx)) + { + if (!AddTransaction(tx, false)) + return; + } + else + { + if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) + unverified.Add(tx); + } + } + foreach (Transaction tx in unverified) + if (!AddTransaction(tx, true)) + return; + if (context.Transactions.Count < context.TransactionHashes.Length) + { + UInt256[] hashes = context.TransactionHashes.Where(i => !context.Transactions.ContainsKey(i)).ToArray(); + taskManager.Tell(new TaskManager.RestartTasks + { + Payload = InvPayload.Create(InventoryType.TX, hashes) + }); + } + } + + private void OnPrepareResponseReceived(ConsensusPayload payload, PrepareResponse message) + { + if (message.ViewNumber != context.ViewNumber) return; + if (context.PreparationPayloads[payload.ValidatorIndex] != null || context.NotAcceptingPayloadsDueToViewChanging) return; + if (context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex] != null && !message.PreparationHash.Equals(context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex].Hash)) + return; + + // Timeout extension: prepare response has been received with success + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + Log($"{nameof(OnPrepareResponseReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); + context.PreparationPayloads[payload.ValidatorIndex] = payload; + if (context.WatchOnly || context.CommitSent) return; + if (context.RequestSentOrReceived) + CheckPreparations(); + } + + protected override void OnReceive(object message) + { + if (message is Start options) + { + if (started) return; + OnStart(options); + } + else + { + if (!started) return; + switch (message) + { + case SetViewNumber setView: + InitializeConsensus(setView.ViewNumber); + break; + case Timer timer: + OnTimer(timer); + break; + case ConsensusPayload payload: + OnConsensusPayload(payload); + break; + case Transaction transaction: + OnTransaction(transaction); + break; + case Blockchain.PersistCompleted completed: + OnPersistCompleted(completed.Block); + break; + } + } + } + + private void RequestRecovery() + { + if (context.Block.Index == Blockchain.Singleton.HeaderHeight + 1) + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryRequest() }); + } + + private void OnStart(Start options) + { + Log("OnStart"); + started = true; + if (!options.IgnoreRecoveryLogs && context.Load()) + { + if (context.Transactions != null) + { + Sender.Ask(new Blockchain.FillMemoryPool + { + Transactions = context.Transactions.Values + }).Wait(); + } + if (context.CommitSent) + { + CheckPreparations(); + return; + } + } + InitializeConsensus(0); + // Issue a ChangeView with NewViewNumber of 0 to request recovery messages on start-up. + if (!context.WatchOnly) + RequestRecovery(); + } + + private void OnTimer(Timer timer) + { + if (context.WatchOnly || context.BlockSent) return; + if (timer.Height != context.Block.Index || timer.ViewNumber != context.ViewNumber) return; + Log($"timeout: height={timer.Height} view={timer.ViewNumber}"); + if (context.IsPrimary && !context.RequestSentOrReceived) + { + SendPrepareRequest(); + } + else if ((context.IsPrimary && context.RequestSentOrReceived) || context.IsBackup) + { + if (context.CommitSent) + { + // Re-send commit periodically by sending recover message in case of a network issue. + Log($"send recovery to resend commit"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << 1)); + } + else + { + var reason = ChangeViewReason.Timeout; + + if (context.Block != null && context.TransactionHashes?.Count() > context.Transactions?.Count) + { + reason = ChangeViewReason.TxNotFound; + } + + RequestChangeView(reason); + } + } + } + + private void OnTransaction(Transaction transaction) + { + if (!context.IsBackup || context.NotAcceptingPayloadsDueToViewChanging || !context.RequestSentOrReceived || context.ResponseSent || context.BlockSent) + return; + if (context.Transactions.ContainsKey(transaction.Hash)) return; + if (!context.TransactionHashes.Contains(transaction.Hash)) return; + AddTransaction(transaction, true); + } + + protected override void PostStop() + { + Log("OnStop"); + started = false; + Context.System.EventStream.Unsubscribe(Self); + context.Dispose(); + base.PostStop(); + } + + public static Props Props(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) + { + return Akka.Actor.Props.Create(() => new ConsensusService(localNode, taskManager, store, wallet)).WithMailbox("consensus-service-mailbox"); + } + + private void RequestChangeView(ChangeViewReason reason) + { + if (context.WatchOnly) return; + // Request for next view is always one view more than the current context.ViewNumber + // Nodes will not contribute for changing to a view higher than (context.ViewNumber+1), unless they are recovered + // The latter may happen by nodes in higher views with, at least, `M` proofs + byte expectedView = context.ViewNumber; + expectedView++; + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (expectedView + 1))); + if ((context.CountCommitted + context.CountFailed) > context.F) + { + Log($"Skip requesting change view to nv={expectedView} because nc={context.CountCommitted} nf={context.CountFailed}"); + RequestRecovery(); + return; + } + Log($"request change view: height={context.Block.Index} view={context.ViewNumber} nv={expectedView} nc={context.CountCommitted} nf={context.CountFailed}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(reason) }); + CheckExpectedView(expectedView); + } + + private bool ReverifyAndProcessPayload(ConsensusPayload payload) + { + if (!payload.Verify(context.Snapshot)) return false; + OnConsensusPayload(payload); + return true; + } + + private void SendPrepareRequest() + { + Log($"send prepare request: height={context.Block.Index} view={context.ViewNumber}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareRequest() }); + + if (context.Validators.Length == 1) + CheckPreparations(); + + if (context.TransactionHashes.Length > 0) + { + foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) + localNode.Tell(Message.Create(MessageCommand.Inv, payload)); + } + ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); + } + } + + internal class ConsensusServiceMailbox : PriorityMailbox + { + public ConsensusServiceMailbox(Akka.Actor.Settings settings, Config config) + : base(settings, config) + { + } + + internal protected override bool IsHighPriority(object message) + { + switch (message) + { + case ConsensusPayload _: + case ConsensusService.SetViewNumber _: + case ConsensusService.Timer _: + case Blockchain.PersistCompleted _: + return true; + default: + return false; + } + } + } +} From a9282c07659e91e300cc7cdf8c5ad0ff5610355b Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 12 Aug 2019 11:10:47 -0300 Subject: [PATCH 40/41] Revert "Remove extra line" This reverts commit e1348813a488e3be736f177e20df79e1efa01c6c. --- neo/Consensus/ConsensusService.cs | 1307 +++++++++++++++-------------- 1 file changed, 654 insertions(+), 653 deletions(-) diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index eb30880022..cb6e6e1dc4 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -1,660 +1,661 @@ -using Akka.Actor; -using Akka.Configuration; -using Neo.Cryptography; -using Neo.IO; -using Neo.IO.Actors; -using Neo.Ledger; -using Neo.Network.P2P; -using Neo.Network.P2P.Payloads; -using Neo.Persistence; -using Neo.Plugins; -using Neo.SmartContract.Native; -using Neo.Wallets; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; - -namespace Neo.Consensus -{ - public sealed class ConsensusService : UntypedActor - { - public class Start { public bool IgnoreRecoveryLogs; } - public class SetViewNumber { public byte ViewNumber; } - internal class Timer { public uint Height; public byte ViewNumber; } - - private readonly ConsensusContext context; - private readonly IActorRef localNode; - private readonly IActorRef taskManager; - private ICancelable timer_token; - private DateTime block_received_time; - private bool started = false; - - /// - /// This will record the information from last scheduled timer - /// - private DateTime clock_started = TimeProvider.Current.UtcNow; - private TimeSpan expected_delay = TimeSpan.Zero; - - /// - /// This will be cleared every block (so it will not grow out of control, but is used to prevent repeatedly - /// responding to the same message. - /// - private readonly HashSet knownHashes = new HashSet(); - /// - /// This variable is only true during OnRecoveryMessageReceived - /// - private bool isRecovering = false; - - public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) - : this(localNode, taskManager, new ConsensusContext(wallet, store)) - { - } - - internal ConsensusService(IActorRef localNode, IActorRef taskManager, ConsensusContext context) - { - this.localNode = localNode; - this.taskManager = taskManager; - this.context = context; - Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); - } - - private bool AddTransaction(Transaction tx, bool verify) - { - if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values)) - { - Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxInvalid); - return false; - } - if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot)) - { - Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); - RequestChangeView(ChangeViewReason.TxRejectedByPolicy); - return false; - } - context.Transactions[tx.Hash] = tx; - if (context.TransactionHashes.Length == context.Transactions.Count) - { - // if we are the primary for this view, but acting as a backup because we recovered our own - // previously sent prepare request, then we don't want to send a prepare response. +using Akka.Actor; +using Akka.Configuration; +using Neo.Cryptography; +using Neo.IO; +using Neo.IO.Actors; +using Neo.Ledger; +using Neo.Network.P2P; +using Neo.Network.P2P.Payloads; +using Neo.Persistence; +using Neo.Plugins; +using Neo.SmartContract.Native; +using Neo.Wallets; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Neo.Consensus +{ + public sealed class ConsensusService : UntypedActor + { + public class Start { public bool IgnoreRecoveryLogs; } + public class SetViewNumber { public byte ViewNumber; } + internal class Timer { public uint Height; public byte ViewNumber; } + + private readonly ConsensusContext context; + private readonly IActorRef localNode; + private readonly IActorRef taskManager; + private ICancelable timer_token; + private DateTime block_received_time; + private bool started = false; + + /// + /// This will record the information from last scheduled timer + /// + private DateTime clock_started = TimeProvider.Current.UtcNow; + private TimeSpan expected_delay = TimeSpan.Zero; + + /// + /// This will be cleared every block (so it will not grow out of control, but is used to prevent repeatedly + /// responding to the same message. + /// + private readonly HashSet knownHashes = new HashSet(); + /// + /// This variable is only true during OnRecoveryMessageReceived + /// + private bool isRecovering = false; + + public ConsensusService(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) + : this(localNode, taskManager, new ConsensusContext(wallet, store)) + { + } + + internal ConsensusService(IActorRef localNode, IActorRef taskManager, ConsensusContext context) + { + this.localNode = localNode; + this.taskManager = taskManager; + this.context = context; + Context.System.EventStream.Subscribe(Self, typeof(Blockchain.PersistCompleted)); + } + + private bool AddTransaction(Transaction tx, bool verify) + { + if (verify && !tx.Verify(context.Snapshot, context.Transactions.Values)) + { + Log($"Invalid transaction: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxInvalid); + return false; + } + if (!NativeContract.Policy.CheckPolicy(tx, context.Snapshot)) + { + Log($"reject tx: {tx.Hash}{Environment.NewLine}{tx.ToArray().ToHexString()}", LogLevel.Warning); + RequestChangeView(ChangeViewReason.TxRejectedByPolicy); + return false; + } + context.Transactions[tx.Hash] = tx; + if (context.TransactionHashes.Length == context.Transactions.Count) + { + // if we are the primary for this view, but acting as a backup because we recovered our own + // previously sent prepare request, then we don't want to send a prepare response. if (context.IsPrimary || context.WatchOnly) return true; - // Check maximum block size via Native.Contract policy + // Check policy + if (context.GetExpectedBlockSize() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) { Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); return false; - } - - // Timeout extension due to prepare response sent - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - Log($"send prepare response"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse() }); - CheckPreparations(); - } - return true; - } - - private void ChangeTimer(TimeSpan delay) - { - clock_started = TimeProvider.Current.UtcNow; - expected_delay = delay; - timer_token.CancelIfNotNull(); - timer_token = Context.System.Scheduler.ScheduleTellOnceCancelable(delay, Self, new Timer - { - Height = context.Block.Index, - ViewNumber = context.ViewNumber - }, ActorRefs.NoSender); - } - - private void CheckCommits() - { - if (context.CommitPayloads.Count(p => p?.ConsensusMessage.ViewNumber == context.ViewNumber) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) - { - Block block = context.CreateBlock(); - Log($"relay block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); - localNode.Tell(new LocalNode.Relay { Inventory = block }); - } - } - - private void CheckExpectedView(byte viewNumber) - { - if (context.ViewNumber >= viewNumber) return; - // if there are `M` change view payloads with NewViewNumber greater than viewNumber, then, it is safe to move - if (context.ChangeViewPayloads.Count(p => p != null && p.GetDeserializedMessage().NewViewNumber >= viewNumber) >= context.M) - { - if (!context.WatchOnly) - { - ChangeView message = context.ChangeViewPayloads[context.MyIndex]?.GetDeserializedMessage(); - // Communicate the network about my agreement to move to `viewNumber` - // if my last change view payload, `message`, has NewViewNumber lower than current view to change - if (message is null || message.NewViewNumber < viewNumber) - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(ChangeViewReason.ChangeAgreement) }); - } - InitializeConsensus(viewNumber); - } - } - - private void CheckPreparations() - { - if (context.PreparationPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) - { - ConsensusPayload payload = context.MakeCommit(); - Log($"send commit"); - context.Save(); - localNode.Tell(new LocalNode.SendDirectly { Inventory = payload }); - // Set timer, so we will resend the commit in case of a networking issue - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock)); - CheckCommits(); - } - } - - private void InitializeConsensus(byte viewNumber) - { - context.Reset(viewNumber); - if (viewNumber > 0) - Log($"changeview: view={viewNumber} primary={context.Validators[context.GetPrimaryIndex((byte)(viewNumber - 1u))]}", LogLevel.Warning); - Log($"initialize: height={context.Block.Index} view={viewNumber} index={context.MyIndex} role={(context.IsPrimary ? "Primary" : context.WatchOnly ? "WatchOnly" : "Backup")}"); - if (context.WatchOnly) return; - if (context.IsPrimary) - { - if (isRecovering) - { - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); - } - else - { - TimeSpan span = TimeProvider.Current.UtcNow - block_received_time; - if (span >= Blockchain.TimePerBlock) - ChangeTimer(TimeSpan.Zero); - else - ChangeTimer(Blockchain.TimePerBlock - span); - } - } - else - { - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); - } - } - - private void Log(string message, LogLevel level = LogLevel.Info) - { - Console.WriteLine($"[{DateTime.Now.TimeOfDay:hh\\:mm\\:ss\\.fff}] {message}"); - Plugin.Log(nameof(ConsensusService), level, message); - } - - private void OnChangeViewReceived(ConsensusPayload payload, ChangeView message) - { - if (message.NewViewNumber <= context.ViewNumber) - OnRecoveryRequestReceived(payload); - - if (context.CommitSent) return; - - var expectedView = context.ChangeViewPayloads[payload.ValidatorIndex]?.GetDeserializedMessage().NewViewNumber ?? (byte)0; - if (message.NewViewNumber <= expectedView) - return; - - Log($"{nameof(OnChangeViewReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} nv={message.NewViewNumber}"); - context.ChangeViewPayloads[payload.ValidatorIndex] = payload; - CheckExpectedView(message.NewViewNumber); - } - - private void OnCommitReceived(ConsensusPayload payload, Commit commit) - { - ref ConsensusPayload existingCommitPayload = ref context.CommitPayloads[payload.ValidatorIndex]; - if (existingCommitPayload != null) - { - if (existingCommitPayload.Hash != payload.Hash) - Log($"{nameof(OnCommitReceived)}: different commit from validator! height={payload.BlockIndex} index={payload.ValidatorIndex} view={commit.ViewNumber} existingView={existingCommitPayload.ConsensusMessage.ViewNumber}", LogLevel.Warning); - return; - } - - // Timeout extension: commit has been received with success - // around 4*15s/M=60.0s/5=12.0s ~ 80% block time (for M=5) - ExtendTimerByFactor(4); - - if (commit.ViewNumber == context.ViewNumber) - { - Log($"{nameof(OnCommitReceived)}: height={payload.BlockIndex} view={commit.ViewNumber} index={payload.ValidatorIndex} nc={context.CountCommitted} nf={context.CountFailed}"); - - byte[] hashData = context.EnsureHeader()?.GetHashData(); - if (hashData == null) - { - existingCommitPayload = payload; - } - else if (Crypto.Default.VerifySignature(hashData, commit.Signature, - context.Validators[payload.ValidatorIndex].EncodePoint(false))) - { - existingCommitPayload = payload; - CheckCommits(); - } - return; - } - // Receiving commit from another view - Log($"{nameof(OnCommitReceived)}: record commit for different view={commit.ViewNumber} index={payload.ValidatorIndex} height={payload.BlockIndex}"); - existingCommitPayload = payload; - } - - // this function increases existing timer (never decreases) with a value proportional to `maxDelayInBlockTimes`*`Blockchain.MillisecondsPerBlock` - private void ExtendTimerByFactor(int maxDelayInBlockTimes) - { - TimeSpan nextDelay = expected_delay - (TimeProvider.Current.UtcNow - clock_started) + TimeSpan.FromMilliseconds(maxDelayInBlockTimes * Blockchain.MillisecondsPerBlock / context.M); - if (!context.WatchOnly && !context.ViewChanging && !context.CommitSent && (nextDelay > TimeSpan.Zero)) - ChangeTimer(nextDelay); - } - - private void OnConsensusPayload(ConsensusPayload payload) - { - if (context.BlockSent) return; - if (payload.Version != context.Block.Version) return; - if (payload.PrevHash != context.Block.PrevHash || payload.BlockIndex != context.Block.Index) - { - if (context.Block.Index < payload.BlockIndex) - { - Log($"chain sync: expected={payload.BlockIndex} current={context.Block.Index - 1} nodes={LocalNode.Singleton.ConnectedCount}", LogLevel.Warning); - } - return; - } - if (payload.ValidatorIndex >= context.Validators.Length) return; - ConsensusMessage message; - try - { - message = payload.ConsensusMessage; - } - catch (FormatException) - { - return; - } - catch (IOException) - { - return; - } - context.LastSeenMessage[payload.ValidatorIndex] = (int)payload.BlockIndex; - foreach (IP2PPlugin plugin in Plugin.P2PPlugins) - if (!plugin.OnConsensusMessage(payload)) - return; - switch (message) - { - case ChangeView view: - OnChangeViewReceived(payload, view); - break; - case PrepareRequest request: - OnPrepareRequestReceived(payload, request); - break; - case PrepareResponse response: - OnPrepareResponseReceived(payload, response); - break; - case Commit commit: - OnCommitReceived(payload, commit); - break; - case RecoveryRequest _: - OnRecoveryRequestReceived(payload); - break; - case RecoveryMessage recovery: - OnRecoveryMessageReceived(payload, recovery); - break; - } - } - - private void OnPersistCompleted(Block block) - { - Log($"persist block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); - block_received_time = TimeProvider.Current.UtcNow; - knownHashes.Clear(); - InitializeConsensus(0); - } - - private void OnRecoveryMessageReceived(ConsensusPayload payload, RecoveryMessage message) - { - // isRecovering is always set to false again after OnRecoveryMessageReceived - isRecovering = true; - int validChangeViews = 0, totalChangeViews = 0, validPrepReq = 0, totalPrepReq = 0; - int validPrepResponses = 0, totalPrepResponses = 0, validCommits = 0, totalCommits = 0; - - Log($"{nameof(OnRecoveryMessageReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); - try - { - if (message.ViewNumber > context.ViewNumber) - { - if (context.CommitSent) return; - ConsensusPayload[] changeViewPayloads = message.GetChangeViewPayloads(context, payload); - totalChangeViews = changeViewPayloads.Length; - foreach (ConsensusPayload changeViewPayload in changeViewPayloads) - if (ReverifyAndProcessPayload(changeViewPayload)) validChangeViews++; - } - if (message.ViewNumber == context.ViewNumber && !context.NotAcceptingPayloadsDueToViewChanging && !context.CommitSent) - { - if (!context.RequestSentOrReceived) - { - ConsensusPayload prepareRequestPayload = message.GetPrepareRequestPayload(context, payload); - if (prepareRequestPayload != null) - { - totalPrepReq = 1; - if (ReverifyAndProcessPayload(prepareRequestPayload)) validPrepReq++; - } - else if (context.IsPrimary) - SendPrepareRequest(); - } - ConsensusPayload[] prepareResponsePayloads = message.GetPrepareResponsePayloads(context, payload); - totalPrepResponses = prepareResponsePayloads.Length; - foreach (ConsensusPayload prepareResponsePayload in prepareResponsePayloads) - if (ReverifyAndProcessPayload(prepareResponsePayload)) validPrepResponses++; - } - if (message.ViewNumber <= context.ViewNumber) - { - // Ensure we know about all commits from lower view numbers. - ConsensusPayload[] commitPayloads = message.GetCommitPayloadsFromRecoveryMessage(context, payload); - totalCommits = commitPayloads.Length; - foreach (ConsensusPayload commitPayload in commitPayloads) - if (ReverifyAndProcessPayload(commitPayload)) validCommits++; - } - } - finally - { - Log($"{nameof(OnRecoveryMessageReceived)}: finished (valid/total) " + - $"ChgView: {validChangeViews}/{totalChangeViews} " + - $"PrepReq: {validPrepReq}/{totalPrepReq} " + - $"PrepResp: {validPrepResponses}/{totalPrepResponses} " + - $"Commits: {validCommits}/{totalCommits}"); - isRecovering = false; - } - } - - private void OnRecoveryRequestReceived(ConsensusPayload payload) - { - // We keep track of the payload hashes received in this block, and don't respond with recovery - // in response to the same payload that we already responded to previously. - // ChangeView messages include a Timestamp when the change view is sent, thus if a node restarts - // and issues a change view for the same view, it will have a different hash and will correctly respond - // again; however replay attacks of the ChangeView message from arbitrary nodes will not trigger an - // additional recovery message response. - if (!knownHashes.Add(payload.Hash)) return; - - Log($"On{payload.ConsensusMessage.GetType().Name}Received: height={payload.BlockIndex} index={payload.ValidatorIndex} view={payload.ConsensusMessage.ViewNumber}"); - if (context.WatchOnly) return; - if (!context.CommitSent) - { - bool shouldSendRecovery = false; - int allowedRecoveryNodeCount = context.F; - // Limit recoveries to be sent from an upper limit of `f` nodes - for (int i = 1; i <= allowedRecoveryNodeCount; i++) - { - var chosenIndex = (payload.ValidatorIndex + i) % context.Validators.Length; - if (chosenIndex != context.MyIndex) continue; - shouldSendRecovery = true; - break; - } - - if (!shouldSendRecovery) return; - } - Log($"send recovery: view={context.ViewNumber}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); - } - - private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest message) - { - if (context.RequestSentOrReceived || context.NotAcceptingPayloadsDueToViewChanging) return; - if (payload.ValidatorIndex != context.Block.ConsensusData.PrimaryIndex || message.ViewNumber != context.ViewNumber) return; - Log($"{nameof(OnPrepareRequestReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} tx={message.TransactionHashes.Length}"); - if (message.Timestamp <= context.PrevHeader.Timestamp || message.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestampMS()) - { - Log($"Timestamp incorrect: {message.Timestamp}", LogLevel.Warning); - return; - } - if (message.TransactionHashes.Any(p => context.Snapshot.ContainsTransaction(p))) - { - Log($"Invalid request: transaction already exists", LogLevel.Warning); - return; - } - - // Timeout extension: prepare request has been received with success - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - context.Block.Timestamp = message.Timestamp; - context.Block.ConsensusData.Nonce = message.Nonce; - context.TransactionHashes = message.TransactionHashes; - context.Transactions = new Dictionary(); - for (int i = 0; i < context.PreparationPayloads.Length; i++) - if (context.PreparationPayloads[i] != null) - if (!context.PreparationPayloads[i].GetDeserializedMessage().PreparationHash.Equals(payload.Hash)) - context.PreparationPayloads[i] = null; - context.PreparationPayloads[payload.ValidatorIndex] = payload; - byte[] hashData = context.EnsureHeader().GetHashData(); - for (int i = 0; i < context.CommitPayloads.Length; i++) - if (context.CommitPayloads[i]?.ConsensusMessage.ViewNumber == context.ViewNumber) - if (!Crypto.Default.VerifySignature(hashData, context.CommitPayloads[i].GetDeserializedMessage().Signature, context.Validators[i].EncodePoint(false))) - context.CommitPayloads[i] = null; - Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); - List unverified = new List(); - foreach (UInt256 hash in context.TransactionHashes) - { - if (mempoolVerified.TryGetValue(hash, out Transaction tx)) - { - if (!AddTransaction(tx, false)) - return; - } - else - { - if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) - unverified.Add(tx); - } - } - foreach (Transaction tx in unverified) - if (!AddTransaction(tx, true)) - return; - if (context.Transactions.Count < context.TransactionHashes.Length) - { - UInt256[] hashes = context.TransactionHashes.Where(i => !context.Transactions.ContainsKey(i)).ToArray(); - taskManager.Tell(new TaskManager.RestartTasks - { - Payload = InvPayload.Create(InventoryType.TX, hashes) - }); - } - } - - private void OnPrepareResponseReceived(ConsensusPayload payload, PrepareResponse message) - { - if (message.ViewNumber != context.ViewNumber) return; - if (context.PreparationPayloads[payload.ValidatorIndex] != null || context.NotAcceptingPayloadsDueToViewChanging) return; - if (context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex] != null && !message.PreparationHash.Equals(context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex].Hash)) - return; - - // Timeout extension: prepare response has been received with success - // around 2*15/M=30.0/5 ~ 40% block time (for M=5) - ExtendTimerByFactor(2); - - Log($"{nameof(OnPrepareResponseReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); - context.PreparationPayloads[payload.ValidatorIndex] = payload; - if (context.WatchOnly || context.CommitSent) return; - if (context.RequestSentOrReceived) - CheckPreparations(); - } - - protected override void OnReceive(object message) - { - if (message is Start options) - { - if (started) return; - OnStart(options); - } - else - { - if (!started) return; - switch (message) - { - case SetViewNumber setView: - InitializeConsensus(setView.ViewNumber); - break; - case Timer timer: - OnTimer(timer); - break; - case ConsensusPayload payload: - OnConsensusPayload(payload); - break; - case Transaction transaction: - OnTransaction(transaction); - break; - case Blockchain.PersistCompleted completed: - OnPersistCompleted(completed.Block); - break; - } - } - } - - private void RequestRecovery() - { - if (context.Block.Index == Blockchain.Singleton.HeaderHeight + 1) - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryRequest() }); - } - - private void OnStart(Start options) - { - Log("OnStart"); - started = true; - if (!options.IgnoreRecoveryLogs && context.Load()) - { - if (context.Transactions != null) - { - Sender.Ask(new Blockchain.FillMemoryPool - { - Transactions = context.Transactions.Values - }).Wait(); - } - if (context.CommitSent) - { - CheckPreparations(); - return; - } - } - InitializeConsensus(0); - // Issue a ChangeView with NewViewNumber of 0 to request recovery messages on start-up. - if (!context.WatchOnly) - RequestRecovery(); - } - - private void OnTimer(Timer timer) - { - if (context.WatchOnly || context.BlockSent) return; - if (timer.Height != context.Block.Index || timer.ViewNumber != context.ViewNumber) return; - Log($"timeout: height={timer.Height} view={timer.ViewNumber}"); - if (context.IsPrimary && !context.RequestSentOrReceived) - { - SendPrepareRequest(); - } - else if ((context.IsPrimary && context.RequestSentOrReceived) || context.IsBackup) - { - if (context.CommitSent) - { - // Re-send commit periodically by sending recover message in case of a network issue. - Log($"send recovery to resend commit"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << 1)); - } - else - { - var reason = ChangeViewReason.Timeout; - - if (context.Block != null && context.TransactionHashes?.Count() > context.Transactions?.Count) - { - reason = ChangeViewReason.TxNotFound; - } - - RequestChangeView(reason); - } - } - } - - private void OnTransaction(Transaction transaction) - { - if (!context.IsBackup || context.NotAcceptingPayloadsDueToViewChanging || !context.RequestSentOrReceived || context.ResponseSent || context.BlockSent) - return; - if (context.Transactions.ContainsKey(transaction.Hash)) return; - if (!context.TransactionHashes.Contains(transaction.Hash)) return; - AddTransaction(transaction, true); - } - - protected override void PostStop() - { - Log("OnStop"); - started = false; - Context.System.EventStream.Unsubscribe(Self); - context.Dispose(); - base.PostStop(); - } - - public static Props Props(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) - { - return Akka.Actor.Props.Create(() => new ConsensusService(localNode, taskManager, store, wallet)).WithMailbox("consensus-service-mailbox"); - } - - private void RequestChangeView(ChangeViewReason reason) - { - if (context.WatchOnly) return; - // Request for next view is always one view more than the current context.ViewNumber - // Nodes will not contribute for changing to a view higher than (context.ViewNumber+1), unless they are recovered - // The latter may happen by nodes in higher views with, at least, `M` proofs - byte expectedView = context.ViewNumber; - expectedView++; - ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (expectedView + 1))); - if ((context.CountCommitted + context.CountFailed) > context.F) - { - Log($"Skip requesting change view to nv={expectedView} because nc={context.CountCommitted} nf={context.CountFailed}"); - RequestRecovery(); - return; - } - Log($"request change view: height={context.Block.Index} view={context.ViewNumber} nv={expectedView} nc={context.CountCommitted} nf={context.CountFailed}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(reason) }); - CheckExpectedView(expectedView); - } - - private bool ReverifyAndProcessPayload(ConsensusPayload payload) - { - if (!payload.Verify(context.Snapshot)) return false; - OnConsensusPayload(payload); - return true; - } - - private void SendPrepareRequest() - { - Log($"send prepare request: height={context.Block.Index} view={context.ViewNumber}"); - localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareRequest() }); - - if (context.Validators.Length == 1) - CheckPreparations(); - - if (context.TransactionHashes.Length > 0) - { - foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) - localNode.Tell(Message.Create(MessageCommand.Inv, payload)); - } - ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); - } - } - - internal class ConsensusServiceMailbox : PriorityMailbox - { - public ConsensusServiceMailbox(Akka.Actor.Settings settings, Config config) - : base(settings, config) - { - } - - internal protected override bool IsHighPriority(object message) - { - switch (message) - { - case ConsensusPayload _: - case ConsensusService.SetViewNumber _: - case ConsensusService.Timer _: - case Blockchain.PersistCompleted _: - return true; - default: - return false; - } - } - } -} + } + + // Timeout extension due to prepare response sent + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + Log($"send prepare response"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareResponse() }); + CheckPreparations(); + } + return true; + } + + private void ChangeTimer(TimeSpan delay) + { + clock_started = TimeProvider.Current.UtcNow; + expected_delay = delay; + timer_token.CancelIfNotNull(); + timer_token = Context.System.Scheduler.ScheduleTellOnceCancelable(delay, Self, new Timer + { + Height = context.Block.Index, + ViewNumber = context.ViewNumber + }, ActorRefs.NoSender); + } + + private void CheckCommits() + { + if (context.CommitPayloads.Count(p => p?.ConsensusMessage.ViewNumber == context.ViewNumber) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + { + Block block = context.CreateBlock(); + Log($"relay block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); + localNode.Tell(new LocalNode.Relay { Inventory = block }); + } + } + + private void CheckExpectedView(byte viewNumber) + { + if (context.ViewNumber >= viewNumber) return; + // if there are `M` change view payloads with NewViewNumber greater than viewNumber, then, it is safe to move + if (context.ChangeViewPayloads.Count(p => p != null && p.GetDeserializedMessage().NewViewNumber >= viewNumber) >= context.M) + { + if (!context.WatchOnly) + { + ChangeView message = context.ChangeViewPayloads[context.MyIndex]?.GetDeserializedMessage(); + // Communicate the network about my agreement to move to `viewNumber` + // if my last change view payload, `message`, has NewViewNumber lower than current view to change + if (message is null || message.NewViewNumber < viewNumber) + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(ChangeViewReason.ChangeAgreement) }); + } + InitializeConsensus(viewNumber); + } + } + + private void CheckPreparations() + { + if (context.PreparationPayloads.Count(p => p != null) >= context.M && context.TransactionHashes.All(p => context.Transactions.ContainsKey(p))) + { + ConsensusPayload payload = context.MakeCommit(); + Log($"send commit"); + context.Save(); + localNode.Tell(new LocalNode.SendDirectly { Inventory = payload }); + // Set timer, so we will resend the commit in case of a networking issue + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock)); + CheckCommits(); + } + } + + private void InitializeConsensus(byte viewNumber) + { + context.Reset(viewNumber); + if (viewNumber > 0) + Log($"changeview: view={viewNumber} primary={context.Validators[context.GetPrimaryIndex((byte)(viewNumber - 1u))]}", LogLevel.Warning); + Log($"initialize: height={context.Block.Index} view={viewNumber} index={context.MyIndex} role={(context.IsPrimary ? "Primary" : context.WatchOnly ? "WatchOnly" : "Backup")}"); + if (context.WatchOnly) return; + if (context.IsPrimary) + { + if (isRecovering) + { + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); + } + else + { + TimeSpan span = TimeProvider.Current.UtcNow - block_received_time; + if (span >= Blockchain.TimePerBlock) + ChangeTimer(TimeSpan.Zero); + else + ChangeTimer(Blockchain.TimePerBlock - span); + } + } + else + { + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (viewNumber + 1))); + } + } + + private void Log(string message, LogLevel level = LogLevel.Info) + { + Console.WriteLine($"[{DateTime.Now.TimeOfDay:hh\\:mm\\:ss\\.fff}] {message}"); + Plugin.Log(nameof(ConsensusService), level, message); + } + + private void OnChangeViewReceived(ConsensusPayload payload, ChangeView message) + { + if (message.NewViewNumber <= context.ViewNumber) + OnRecoveryRequestReceived(payload); + + if (context.CommitSent) return; + + var expectedView = context.ChangeViewPayloads[payload.ValidatorIndex]?.GetDeserializedMessage().NewViewNumber ?? (byte)0; + if (message.NewViewNumber <= expectedView) + return; + + Log($"{nameof(OnChangeViewReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} nv={message.NewViewNumber}"); + context.ChangeViewPayloads[payload.ValidatorIndex] = payload; + CheckExpectedView(message.NewViewNumber); + } + + private void OnCommitReceived(ConsensusPayload payload, Commit commit) + { + ref ConsensusPayload existingCommitPayload = ref context.CommitPayloads[payload.ValidatorIndex]; + if (existingCommitPayload != null) + { + if (existingCommitPayload.Hash != payload.Hash) + Log($"{nameof(OnCommitReceived)}: different commit from validator! height={payload.BlockIndex} index={payload.ValidatorIndex} view={commit.ViewNumber} existingView={existingCommitPayload.ConsensusMessage.ViewNumber}", LogLevel.Warning); + return; + } + + // Timeout extension: commit has been received with success + // around 4*15s/M=60.0s/5=12.0s ~ 80% block time (for M=5) + ExtendTimerByFactor(4); + + if (commit.ViewNumber == context.ViewNumber) + { + Log($"{nameof(OnCommitReceived)}: height={payload.BlockIndex} view={commit.ViewNumber} index={payload.ValidatorIndex} nc={context.CountCommitted} nf={context.CountFailed}"); + + byte[] hashData = context.EnsureHeader()?.GetHashData(); + if (hashData == null) + { + existingCommitPayload = payload; + } + else if (Crypto.Default.VerifySignature(hashData, commit.Signature, + context.Validators[payload.ValidatorIndex].EncodePoint(false))) + { + existingCommitPayload = payload; + CheckCommits(); + } + return; + } + // Receiving commit from another view + Log($"{nameof(OnCommitReceived)}: record commit for different view={commit.ViewNumber} index={payload.ValidatorIndex} height={payload.BlockIndex}"); + existingCommitPayload = payload; + } + + // this function increases existing timer (never decreases) with a value proportional to `maxDelayInBlockTimes`*`Blockchain.MillisecondsPerBlock` + private void ExtendTimerByFactor(int maxDelayInBlockTimes) + { + TimeSpan nextDelay = expected_delay - (TimeProvider.Current.UtcNow - clock_started) + TimeSpan.FromMilliseconds(maxDelayInBlockTimes * Blockchain.MillisecondsPerBlock / context.M); + if (!context.WatchOnly && !context.ViewChanging && !context.CommitSent && (nextDelay > TimeSpan.Zero)) + ChangeTimer(nextDelay); + } + + private void OnConsensusPayload(ConsensusPayload payload) + { + if (context.BlockSent) return; + if (payload.Version != context.Block.Version) return; + if (payload.PrevHash != context.Block.PrevHash || payload.BlockIndex != context.Block.Index) + { + if (context.Block.Index < payload.BlockIndex) + { + Log($"chain sync: expected={payload.BlockIndex} current={context.Block.Index - 1} nodes={LocalNode.Singleton.ConnectedCount}", LogLevel.Warning); + } + return; + } + if (payload.ValidatorIndex >= context.Validators.Length) return; + ConsensusMessage message; + try + { + message = payload.ConsensusMessage; + } + catch (FormatException) + { + return; + } + catch (IOException) + { + return; + } + context.LastSeenMessage[payload.ValidatorIndex] = (int)payload.BlockIndex; + foreach (IP2PPlugin plugin in Plugin.P2PPlugins) + if (!plugin.OnConsensusMessage(payload)) + return; + switch (message) + { + case ChangeView view: + OnChangeViewReceived(payload, view); + break; + case PrepareRequest request: + OnPrepareRequestReceived(payload, request); + break; + case PrepareResponse response: + OnPrepareResponseReceived(payload, response); + break; + case Commit commit: + OnCommitReceived(payload, commit); + break; + case RecoveryRequest _: + OnRecoveryRequestReceived(payload); + break; + case RecoveryMessage recovery: + OnRecoveryMessageReceived(payload, recovery); + break; + } + } + + private void OnPersistCompleted(Block block) + { + Log($"persist block: height={block.Index} hash={block.Hash} tx={block.Transactions.Length}"); + block_received_time = TimeProvider.Current.UtcNow; + knownHashes.Clear(); + InitializeConsensus(0); + } + + private void OnRecoveryMessageReceived(ConsensusPayload payload, RecoveryMessage message) + { + // isRecovering is always set to false again after OnRecoveryMessageReceived + isRecovering = true; + int validChangeViews = 0, totalChangeViews = 0, validPrepReq = 0, totalPrepReq = 0; + int validPrepResponses = 0, totalPrepResponses = 0, validCommits = 0, totalCommits = 0; + + Log($"{nameof(OnRecoveryMessageReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); + try + { + if (message.ViewNumber > context.ViewNumber) + { + if (context.CommitSent) return; + ConsensusPayload[] changeViewPayloads = message.GetChangeViewPayloads(context, payload); + totalChangeViews = changeViewPayloads.Length; + foreach (ConsensusPayload changeViewPayload in changeViewPayloads) + if (ReverifyAndProcessPayload(changeViewPayload)) validChangeViews++; + } + if (message.ViewNumber == context.ViewNumber && !context.NotAcceptingPayloadsDueToViewChanging && !context.CommitSent) + { + if (!context.RequestSentOrReceived) + { + ConsensusPayload prepareRequestPayload = message.GetPrepareRequestPayload(context, payload); + if (prepareRequestPayload != null) + { + totalPrepReq = 1; + if (ReverifyAndProcessPayload(prepareRequestPayload)) validPrepReq++; + } + else if (context.IsPrimary) + SendPrepareRequest(); + } + ConsensusPayload[] prepareResponsePayloads = message.GetPrepareResponsePayloads(context, payload); + totalPrepResponses = prepareResponsePayloads.Length; + foreach (ConsensusPayload prepareResponsePayload in prepareResponsePayloads) + if (ReverifyAndProcessPayload(prepareResponsePayload)) validPrepResponses++; + } + if (message.ViewNumber <= context.ViewNumber) + { + // Ensure we know about all commits from lower view numbers. + ConsensusPayload[] commitPayloads = message.GetCommitPayloadsFromRecoveryMessage(context, payload); + totalCommits = commitPayloads.Length; + foreach (ConsensusPayload commitPayload in commitPayloads) + if (ReverifyAndProcessPayload(commitPayload)) validCommits++; + } + } + finally + { + Log($"{nameof(OnRecoveryMessageReceived)}: finished (valid/total) " + + $"ChgView: {validChangeViews}/{totalChangeViews} " + + $"PrepReq: {validPrepReq}/{totalPrepReq} " + + $"PrepResp: {validPrepResponses}/{totalPrepResponses} " + + $"Commits: {validCommits}/{totalCommits}"); + isRecovering = false; + } + } + + private void OnRecoveryRequestReceived(ConsensusPayload payload) + { + // We keep track of the payload hashes received in this block, and don't respond with recovery + // in response to the same payload that we already responded to previously. + // ChangeView messages include a Timestamp when the change view is sent, thus if a node restarts + // and issues a change view for the same view, it will have a different hash and will correctly respond + // again; however replay attacks of the ChangeView message from arbitrary nodes will not trigger an + // additional recovery message response. + if (!knownHashes.Add(payload.Hash)) return; + + Log($"On{payload.ConsensusMessage.GetType().Name}Received: height={payload.BlockIndex} index={payload.ValidatorIndex} view={payload.ConsensusMessage.ViewNumber}"); + if (context.WatchOnly) return; + if (!context.CommitSent) + { + bool shouldSendRecovery = false; + int allowedRecoveryNodeCount = context.F; + // Limit recoveries to be sent from an upper limit of `f` nodes + for (int i = 1; i <= allowedRecoveryNodeCount; i++) + { + var chosenIndex = (payload.ValidatorIndex + i) % context.Validators.Length; + if (chosenIndex != context.MyIndex) continue; + shouldSendRecovery = true; + break; + } + + if (!shouldSendRecovery) return; + } + Log($"send recovery: view={context.ViewNumber}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); + } + + private void OnPrepareRequestReceived(ConsensusPayload payload, PrepareRequest message) + { + if (context.RequestSentOrReceived || context.NotAcceptingPayloadsDueToViewChanging) return; + if (payload.ValidatorIndex != context.Block.ConsensusData.PrimaryIndex || message.ViewNumber != context.ViewNumber) return; + Log($"{nameof(OnPrepareRequestReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex} tx={message.TransactionHashes.Length}"); + if (message.Timestamp <= context.PrevHeader.Timestamp || message.Timestamp > TimeProvider.Current.UtcNow.AddMinutes(10).ToTimestampMS()) + { + Log($"Timestamp incorrect: {message.Timestamp}", LogLevel.Warning); + return; + } + if (message.TransactionHashes.Any(p => context.Snapshot.ContainsTransaction(p))) + { + Log($"Invalid request: transaction already exists", LogLevel.Warning); + return; + } + + // Timeout extension: prepare request has been received with success + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + context.Block.Timestamp = message.Timestamp; + context.Block.ConsensusData.Nonce = message.Nonce; + context.TransactionHashes = message.TransactionHashes; + context.Transactions = new Dictionary(); + for (int i = 0; i < context.PreparationPayloads.Length; i++) + if (context.PreparationPayloads[i] != null) + if (!context.PreparationPayloads[i].GetDeserializedMessage().PreparationHash.Equals(payload.Hash)) + context.PreparationPayloads[i] = null; + context.PreparationPayloads[payload.ValidatorIndex] = payload; + byte[] hashData = context.EnsureHeader().GetHashData(); + for (int i = 0; i < context.CommitPayloads.Length; i++) + if (context.CommitPayloads[i]?.ConsensusMessage.ViewNumber == context.ViewNumber) + if (!Crypto.Default.VerifySignature(hashData, context.CommitPayloads[i].GetDeserializedMessage().Signature, context.Validators[i].EncodePoint(false))) + context.CommitPayloads[i] = null; + Dictionary mempoolVerified = Blockchain.Singleton.MemPool.GetVerifiedTransactions().ToDictionary(p => p.Hash); + List unverified = new List(); + foreach (UInt256 hash in context.TransactionHashes) + { + if (mempoolVerified.TryGetValue(hash, out Transaction tx)) + { + if (!AddTransaction(tx, false)) + return; + } + else + { + if (Blockchain.Singleton.MemPool.TryGetValue(hash, out tx)) + unverified.Add(tx); + } + } + foreach (Transaction tx in unverified) + if (!AddTransaction(tx, true)) + return; + if (context.Transactions.Count < context.TransactionHashes.Length) + { + UInt256[] hashes = context.TransactionHashes.Where(i => !context.Transactions.ContainsKey(i)).ToArray(); + taskManager.Tell(new TaskManager.RestartTasks + { + Payload = InvPayload.Create(InventoryType.TX, hashes) + }); + } + } + + private void OnPrepareResponseReceived(ConsensusPayload payload, PrepareResponse message) + { + if (message.ViewNumber != context.ViewNumber) return; + if (context.PreparationPayloads[payload.ValidatorIndex] != null || context.NotAcceptingPayloadsDueToViewChanging) return; + if (context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex] != null && !message.PreparationHash.Equals(context.PreparationPayloads[context.Block.ConsensusData.PrimaryIndex].Hash)) + return; + + // Timeout extension: prepare response has been received with success + // around 2*15/M=30.0/5 ~ 40% block time (for M=5) + ExtendTimerByFactor(2); + + Log($"{nameof(OnPrepareResponseReceived)}: height={payload.BlockIndex} view={message.ViewNumber} index={payload.ValidatorIndex}"); + context.PreparationPayloads[payload.ValidatorIndex] = payload; + if (context.WatchOnly || context.CommitSent) return; + if (context.RequestSentOrReceived) + CheckPreparations(); + } + + protected override void OnReceive(object message) + { + if (message is Start options) + { + if (started) return; + OnStart(options); + } + else + { + if (!started) return; + switch (message) + { + case SetViewNumber setView: + InitializeConsensus(setView.ViewNumber); + break; + case Timer timer: + OnTimer(timer); + break; + case ConsensusPayload payload: + OnConsensusPayload(payload); + break; + case Transaction transaction: + OnTransaction(transaction); + break; + case Blockchain.PersistCompleted completed: + OnPersistCompleted(completed.Block); + break; + } + } + } + + private void RequestRecovery() + { + if (context.Block.Index == Blockchain.Singleton.HeaderHeight + 1) + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryRequest() }); + } + + private void OnStart(Start options) + { + Log("OnStart"); + started = true; + if (!options.IgnoreRecoveryLogs && context.Load()) + { + if (context.Transactions != null) + { + Sender.Ask(new Blockchain.FillMemoryPool + { + Transactions = context.Transactions.Values + }).Wait(); + } + if (context.CommitSent) + { + CheckPreparations(); + return; + } + } + InitializeConsensus(0); + // Issue a ChangeView with NewViewNumber of 0 to request recovery messages on start-up. + if (!context.WatchOnly) + RequestRecovery(); + } + + private void OnTimer(Timer timer) + { + if (context.WatchOnly || context.BlockSent) return; + if (timer.Height != context.Block.Index || timer.ViewNumber != context.ViewNumber) return; + Log($"timeout: height={timer.Height} view={timer.ViewNumber}"); + if (context.IsPrimary && !context.RequestSentOrReceived) + { + SendPrepareRequest(); + } + else if ((context.IsPrimary && context.RequestSentOrReceived) || context.IsBackup) + { + if (context.CommitSent) + { + // Re-send commit periodically by sending recover message in case of a network issue. + Log($"send recovery to resend commit"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeRecoveryMessage() }); + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << 1)); + } + else + { + var reason = ChangeViewReason.Timeout; + + if (context.Block != null && context.TransactionHashes?.Count() > context.Transactions?.Count) + { + reason = ChangeViewReason.TxNotFound; + } + + RequestChangeView(reason); + } + } + } + + private void OnTransaction(Transaction transaction) + { + if (!context.IsBackup || context.NotAcceptingPayloadsDueToViewChanging || !context.RequestSentOrReceived || context.ResponseSent || context.BlockSent) + return; + if (context.Transactions.ContainsKey(transaction.Hash)) return; + if (!context.TransactionHashes.Contains(transaction.Hash)) return; + AddTransaction(transaction, true); + } + + protected override void PostStop() + { + Log("OnStop"); + started = false; + Context.System.EventStream.Unsubscribe(Self); + context.Dispose(); + base.PostStop(); + } + + public static Props Props(IActorRef localNode, IActorRef taskManager, Store store, Wallet wallet) + { + return Akka.Actor.Props.Create(() => new ConsensusService(localNode, taskManager, store, wallet)).WithMailbox("consensus-service-mailbox"); + } + + private void RequestChangeView(ChangeViewReason reason) + { + if (context.WatchOnly) return; + // Request for next view is always one view more than the current context.ViewNumber + // Nodes will not contribute for changing to a view higher than (context.ViewNumber+1), unless they are recovered + // The latter may happen by nodes in higher views with, at least, `M` proofs + byte expectedView = context.ViewNumber; + expectedView++; + ChangeTimer(TimeSpan.FromMilliseconds(Blockchain.MillisecondsPerBlock << (expectedView + 1))); + if ((context.CountCommitted + context.CountFailed) > context.F) + { + Log($"Skip requesting change view to nv={expectedView} because nc={context.CountCommitted} nf={context.CountFailed}"); + RequestRecovery(); + return; + } + Log($"request change view: height={context.Block.Index} view={context.ViewNumber} nv={expectedView} nc={context.CountCommitted} nf={context.CountFailed}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakeChangeView(reason) }); + CheckExpectedView(expectedView); + } + + private bool ReverifyAndProcessPayload(ConsensusPayload payload) + { + if (!payload.Verify(context.Snapshot)) return false; + OnConsensusPayload(payload); + return true; + } + + private void SendPrepareRequest() + { + Log($"send prepare request: height={context.Block.Index} view={context.ViewNumber}"); + localNode.Tell(new LocalNode.SendDirectly { Inventory = context.MakePrepareRequest() }); + + if (context.Validators.Length == 1) + CheckPreparations(); + + if (context.TransactionHashes.Length > 0) + { + foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) + localNode.Tell(Message.Create(MessageCommand.Inv, payload)); + } + ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); + } + } + + internal class ConsensusServiceMailbox : PriorityMailbox + { + public ConsensusServiceMailbox(Akka.Actor.Settings settings, Config config) + : base(settings, config) + { + } + + internal protected override bool IsHighPriority(object message) + { + switch (message) + { + case ConsensusPayload _: + case ConsensusService.SetViewNumber _: + case ConsensusService.Timer _: + case Blockchain.PersistCompleted _: + return true; + default: + return false; + } + } + } +} From c7a063ef0e01e21c0ea098a6fd93667b6336e827 Mon Sep 17 00:00:00 2001 From: Vitor Date: Mon, 12 Aug 2019 11:11:45 -0300 Subject: [PATCH 41/41] Remove extra line --- neo/Consensus/ConsensusService.cs | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/neo/Consensus/ConsensusService.cs b/neo/Consensus/ConsensusService.cs index cb6e6e1dc4..32293c4299 100644 --- a/neo/Consensus/ConsensusService.cs +++ b/neo/Consensus/ConsensusService.cs @@ -78,15 +78,14 @@ private bool AddTransaction(Transaction tx, bool verify) { // if we are the primary for this view, but acting as a backup because we recovered our own // previously sent prepare request, then we don't want to send a prepare response. - if (context.IsPrimary || context.WatchOnly) return true; - - // Check policy - - if (context.GetExpectedBlockSize() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) - { - Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); - RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); - return false; + if (context.IsPrimary || context.WatchOnly) return true; + + // Check maximum block size via Native Contract policy + if (context.GetExpectedBlockSize() > NativeContract.Policy.GetMaxBlockSize(context.Snapshot)) + { + Log($"rejected block: {context.Block.Index}{Environment.NewLine} The size exceed the policy", LogLevel.Warning); + RequestChangeView(ChangeViewReason.BlockRejectedByPolicy); + return false; } // Timeout extension due to prepare response sent