From 45fb11e8846f7de81c2bfa3d0c75a06fd6e8156d Mon Sep 17 00:00:00 2001 From: Shargon Date: Thu, 3 Sep 2020 18:22:53 +0200 Subject: [PATCH 1/6] Optional p2p compression --- src/neo/Network/P2P/Message.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/Message.cs b/src/neo/Network/P2P/Message.cs index 2f395ce01f..ea96b48e2c 100644 --- a/src/neo/Network/P2P/Message.cs +++ b/src/neo/Network/P2P/Message.cs @@ -26,7 +26,7 @@ public class Message : ISerializable public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + _payload_compressed.GetVarSize(); - public static Message Create(MessageCommand command, ISerializable payload = null) + public static Message Create(MessageCommand command, ISerializable payload = null, bool tryCompression = false) { Message message = new Message { @@ -37,7 +37,7 @@ public static Message Create(MessageCommand command, ISerializable payload = nul }; // Try compression - if (message._payload_compressed.Length > CompressionMinSize) + if (tryCompression && message._payload_compressed.Length > CompressionMinSize) { var compressed = message._payload_compressed.CompressLz4(); if (compressed.Length < message._payload_compressed.Length - CompressionThreshold) From 64180bd4eb3edd7ce210c14d45cf585e230dca51 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 4 Sep 2020 10:39:54 +0200 Subject: [PATCH 2/6] Add optional compression --- src/neo/Consensus/ConsensusService.cs | 2 +- src/neo/Network/P2P/LocalNode.cs | 2 +- src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs | 8 ++++---- src/neo/Network/P2P/RemoteNode.cs | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/neo/Consensus/ConsensusService.cs b/src/neo/Consensus/ConsensusService.cs index 8054147eb5..bc9fcdf4fc 100644 --- a/src/neo/Consensus/ConsensusService.cs +++ b/src/neo/Consensus/ConsensusService.cs @@ -658,7 +658,7 @@ private void SendPrepareRequest() if (context.TransactionHashes.Length > 0) { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) - localNode.Tell(Message.Create(MessageCommand.Inv, payload)); + localNode.Tell(Message.Create(MessageCommand.Inv, payload, true)); } ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); } diff --git a/src/neo/Network/P2P/LocalNode.cs b/src/neo/Network/P2P/LocalNode.cs index c8bc937001..370ef30955 100644 --- a/src/neo/Network/P2P/LocalNode.cs +++ b/src/neo/Network/P2P/LocalNode.cs @@ -75,7 +75,7 @@ public LocalNode(NeoSystem system) /// Optional payload to be Serialized along the message. private void BroadcastMessage(MessageCommand command, ISerializable payload = null) { - BroadcastMessage(Message.Create(command, payload)); + BroadcastMessage(Message.Create(command, payload, true)); } /// diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index 0cddbd49d7..f541e14019 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -198,7 +198,7 @@ private void OnGetBlockByIndexMessageReceived(GetBlockByIndexPayload payload) if (bloom_filter == null) { - EnqueueMessage(Message.Create(MessageCommand.Block, block)); + EnqueueMessage(Message.Create(MessageCommand.Block, block, true)); } else { @@ -224,7 +224,7 @@ private void OnGetDataMessageReceived(InvPayload payload) case InventoryType.TX: Transaction tx = Blockchain.Singleton.GetTransaction(hash); if (tx != null) - EnqueueMessage(Message.Create(MessageCommand.Transaction, tx)); + EnqueueMessage(Message.Create(MessageCommand.Transaction, tx, true)); else notFound.Add(hash); break; @@ -234,7 +234,7 @@ private void OnGetDataMessageReceived(InvPayload payload) { if (bloom_filter == null) { - EnqueueMessage(Message.Create(MessageCommand.Block, block)); + EnqueueMessage(Message.Create(MessageCommand.Block, block, true)); } else { @@ -249,7 +249,7 @@ private void OnGetDataMessageReceived(InvPayload payload) break; default: if (Blockchain.Singleton.RelayCache.TryGet(hash, out IInventory inventory)) - EnqueueMessage(Message.Create((MessageCommand)payload.Type, inventory)); + EnqueueMessage(Message.Create((MessageCommand)payload.Type, inventory, true)); break; } } diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index b1457a8522..713cd466b3 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -58,7 +58,7 @@ private void CheckMessageQueue() private void EnqueueMessage(MessageCommand command, ISerializable payload = null) { - EnqueueMessage(Message.Create(command, payload)); + EnqueueMessage(Message.Create(command, payload, true)); } /// From 31972dedfed5d1407259e23d522e2621eb3211e6 Mon Sep 17 00:00:00 2001 From: Shargon Date: Fri, 4 Sep 2020 10:42:07 +0200 Subject: [PATCH 3/6] Fix UT --- src/neo/Network/P2P/LocalNode.cs | 2 +- tests/neo.UnitTests/Network/P2P/UT_Message.cs | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/src/neo/Network/P2P/LocalNode.cs b/src/neo/Network/P2P/LocalNode.cs index 370ef30955..c8bc937001 100644 --- a/src/neo/Network/P2P/LocalNode.cs +++ b/src/neo/Network/P2P/LocalNode.cs @@ -75,7 +75,7 @@ public LocalNode(NeoSystem system) /// Optional payload to be Serialized along the message. private void BroadcastMessage(MessageCommand command, ISerializable payload = null) { - BroadcastMessage(Message.Create(command, payload, true)); + BroadcastMessage(Message.Create(command, payload)); } /// diff --git a/tests/neo.UnitTests/Network/P2P/UT_Message.cs b/tests/neo.UnitTests/Network/P2P/UT_Message.cs index 36c14831f0..78b0750ef2 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_Message.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_Message.cs @@ -140,9 +140,14 @@ public void Compression() } }; - var msg = Message.Create(MessageCommand.Version, payload); + var msg = Message.Create(MessageCommand.Version, payload, false); var buffer = msg.ToArray(); + buffer.Length.Should().BeGreaterThan(80); + + msg = Message.Create(MessageCommand.Version, payload, true); + buffer = msg.ToArray(); + buffer.Length.Should().BeLessThan(80); var copy = buffer.AsSerializable(); From fc848b7f465ba9a79272b4aa8a4eb6f0708dc00f Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 6 Sep 2020 12:24:34 +0200 Subject: [PATCH 4/6] Remove Inv compression --- src/neo/Consensus/ConsensusService.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/neo/Consensus/ConsensusService.cs b/src/neo/Consensus/ConsensusService.cs index bc9fcdf4fc..8054147eb5 100644 --- a/src/neo/Consensus/ConsensusService.cs +++ b/src/neo/Consensus/ConsensusService.cs @@ -658,7 +658,7 @@ private void SendPrepareRequest() if (context.TransactionHashes.Length > 0) { foreach (InvPayload payload in InvPayload.CreateGroup(InventoryType.TX, context.TransactionHashes)) - localNode.Tell(Message.Create(MessageCommand.Inv, payload, true)); + localNode.Tell(Message.Create(MessageCommand.Inv, payload)); } ChangeTimer(TimeSpan.FromMilliseconds((Blockchain.MillisecondsPerBlock << (context.ViewNumber + 1)) - (context.ViewNumber == 0 ? Blockchain.MillisecondsPerBlock : 0))); } From f2d89654cc28d57dfbefe39dea583d0aea715369 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sat, 12 Sep 2020 16:08:51 +0200 Subject: [PATCH 5/6] Try according to the type --- src/neo/Network/P2P/Message.cs | 10 +++++- .../Network/P2P/RemoteNode.ProtocolHandler.cs | 8 ++--- src/neo/Network/P2P/RemoteNode.cs | 2 +- tests/neo.UnitTests/Network/P2P/UT_Message.cs | 36 +++++++------------ 4 files changed, 27 insertions(+), 29 deletions(-) diff --git a/src/neo/Network/P2P/Message.cs b/src/neo/Network/P2P/Message.cs index ea96b48e2c..d79cdf43dd 100644 --- a/src/neo/Network/P2P/Message.cs +++ b/src/neo/Network/P2P/Message.cs @@ -26,7 +26,7 @@ public class Message : ISerializable public int Size => sizeof(MessageFlags) + sizeof(MessageCommand) + _payload_compressed.GetVarSize(); - public static Message Create(MessageCommand command, ISerializable payload = null, bool tryCompression = false) + public static Message Create(MessageCommand command, ISerializable payload = null) { Message message = new Message { @@ -36,6 +36,14 @@ public static Message Create(MessageCommand command, ISerializable payload = nul _payload_compressed = payload?.ToArray() ?? Array.Empty() }; + bool tryCompression = + command == MessageCommand.Block || + command == MessageCommand.Consensus || + command == MessageCommand.Transaction || + command == MessageCommand.Headers || + command == MessageCommand.FilterLoad || + command == MessageCommand.FilterAdd; + // Try compression if (tryCompression && message._payload_compressed.Length > CompressionMinSize) { diff --git a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs index f541e14019..0cddbd49d7 100644 --- a/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs +++ b/src/neo/Network/P2P/RemoteNode.ProtocolHandler.cs @@ -198,7 +198,7 @@ private void OnGetBlockByIndexMessageReceived(GetBlockByIndexPayload payload) if (bloom_filter == null) { - EnqueueMessage(Message.Create(MessageCommand.Block, block, true)); + EnqueueMessage(Message.Create(MessageCommand.Block, block)); } else { @@ -224,7 +224,7 @@ private void OnGetDataMessageReceived(InvPayload payload) case InventoryType.TX: Transaction tx = Blockchain.Singleton.GetTransaction(hash); if (tx != null) - EnqueueMessage(Message.Create(MessageCommand.Transaction, tx, true)); + EnqueueMessage(Message.Create(MessageCommand.Transaction, tx)); else notFound.Add(hash); break; @@ -234,7 +234,7 @@ private void OnGetDataMessageReceived(InvPayload payload) { if (bloom_filter == null) { - EnqueueMessage(Message.Create(MessageCommand.Block, block, true)); + EnqueueMessage(Message.Create(MessageCommand.Block, block)); } else { @@ -249,7 +249,7 @@ private void OnGetDataMessageReceived(InvPayload payload) break; default: if (Blockchain.Singleton.RelayCache.TryGet(hash, out IInventory inventory)) - EnqueueMessage(Message.Create((MessageCommand)payload.Type, inventory, true)); + EnqueueMessage(Message.Create((MessageCommand)payload.Type, inventory)); break; } } diff --git a/src/neo/Network/P2P/RemoteNode.cs b/src/neo/Network/P2P/RemoteNode.cs index 713cd466b3..b1457a8522 100644 --- a/src/neo/Network/P2P/RemoteNode.cs +++ b/src/neo/Network/P2P/RemoteNode.cs @@ -58,7 +58,7 @@ private void CheckMessageQueue() private void EnqueueMessage(MessageCommand command, ISerializable payload = null) { - EnqueueMessage(Message.Create(command, payload, true)); + EnqueueMessage(Message.Create(command, payload)); } /// diff --git a/tests/neo.UnitTests/Network/P2P/UT_Message.cs b/tests/neo.UnitTests/Network/P2P/UT_Message.cs index 78b0750ef2..55d65449e2 100644 --- a/tests/neo.UnitTests/Network/P2P/UT_Message.cs +++ b/tests/neo.UnitTests/Network/P2P/UT_Message.cs @@ -127,44 +127,34 @@ public void MultipleSizes() [TestMethod] public void Compression() { - var payload = new VersionPayload() + var payload = new Transaction() { - UserAgent = "".PadLeft(1024, '0'), Nonce = 1, - Magic = 2, - Timestamp = 5, - Version = 6, - Capabilities = new NodeCapability[] - { - new ServerCapability(NodeCapabilityType.TcpServer, 25) - } + Version = 0, + Attributes = new TransactionAttribute[0], + Script = new byte[75], + Signers = new Signer[] { new Signer() { Account = UInt160.Zero } }, + Witnesses = new Witness[0], }; - var msg = Message.Create(MessageCommand.Version, payload, false); + var msg = Message.Create(MessageCommand.Transaction, payload); var buffer = msg.ToArray(); - buffer.Length.Should().BeGreaterThan(80); + buffer.Length.Should().Be(128); - msg = Message.Create(MessageCommand.Version, payload, true); + payload.Script = new byte[payload.Script.Length + 10]; + msg = Message.Create(MessageCommand.Transaction, payload); buffer = msg.ToArray(); - buffer.Length.Should().BeLessThan(80); + buffer.Length.Should().Be(33); var copy = buffer.AsSerializable(); - var payloadCopy = (VersionPayload)copy.Payload; + var payloadCopy = (Transaction)copy.Payload; copy.Command.Should().Be(msg.Command); copy.Flags.Should().HaveFlag(MessageFlags.Compressed); - payloadCopy.UserAgent.Should().Be(payload.UserAgent); - payloadCopy.Nonce.Should().Be(payload.Nonce); - payloadCopy.Magic.Should().Be(payload.Magic); - payloadCopy.Timestamp.Should().Be(payload.Timestamp); - payloadCopy.Version.Should().Be(payload.Version); - - payloadCopy.Capabilities.Length.Should().Be(1); - ((ServerCapability)payloadCopy.Capabilities[0]).Type.Should().Be(NodeCapabilityType.TcpServer); - ((ServerCapability)payloadCopy.Capabilities[0]).Port.Should().Be(25); + payloadCopy.ToArray().ToHexString().Should().Be(payload.ToArray().ToHexString()); } } } From d8b068422c8bb8d7589414a72e285150bb0c3828 Mon Sep 17 00:00:00 2001 From: Shargon Date: Sun, 13 Sep 2020 11:36:47 +0200 Subject: [PATCH 6/6] Add Addr and MerkleBlock --- src/neo/Network/P2P/Message.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/neo/Network/P2P/Message.cs b/src/neo/Network/P2P/Message.cs index d79cdf43dd..397ee4d033 100644 --- a/src/neo/Network/P2P/Message.cs +++ b/src/neo/Network/P2P/Message.cs @@ -41,6 +41,8 @@ public static Message Create(MessageCommand command, ISerializable payload = nul command == MessageCommand.Consensus || command == MessageCommand.Transaction || command == MessageCommand.Headers || + command == MessageCommand.Addr || + command == MessageCommand.MerkleBlock || command == MessageCommand.FilterLoad || command == MessageCommand.FilterAdd;