workaround DotNetty v0.4.6 issues on .NET Core Linux by disabling buffer pooling via HOCON

src/core/Akka.Remote.Tests/Transport/Bug3370DotNettyLinuxBufferPoolSpec.cs

@@ -0,0 +1,90 @@
+//-----------------------------------------------------------------------
+// <copyright file="Bug3370DotNettyLinuxBufferPoolSpec.cs" company="Akka.NET Project">
+//     Copyright (C) 2009-2018 Lightbend Inc. <http://www.lightbend.com>
+//     Copyright (C) 2013-2018 .NET Foundation <https://github.com/akkadotnet/akka.net>
+// </copyright>
+//-----------------------------------------------------------------------
+
+using System.Threading.Tasks;
+using Akka.Actor;
+using Akka.Actor.Dsl;
+using Akka.Configuration;
+using Akka.Remote.Transport.DotNetty;
+using Akka.TestKit;
+using DotNetty.Buffers;
+using FluentAssertions;
+using Xunit;
+
+namespace Akka.Remote.Tests.Transport
+{
+    /// <summary>
+    /// Spec designed to verify that https://github.com/akkadotnet/akka.net/issues/3370
+    /// </summary>
+    public class Bug3370DotNettyLinuxBufferPoolSpec : AkkaSpec
+    {
+        private static readonly Config Config = ConfigurationFactory.ParseString(@"
+            akka {
+                loglevel = DEBUG
+                actor.provider = ""Akka.Remote.RemoteActorRefProvider,Akka.Remote""
+                remote {
+                    dot-netty.tcp {
+                        port = 0
+                        hostname = ""127.0.0.1""
+                        log-transport = true
+                        enable-pooling = false
+                    }
+                }
+            }");
+
+        public Bug3370DotNettyLinuxBufferPoolSpec() : base(Config)
+        {
+
+        }
+
+        [Fact]
+        public async Task DotNettyTcpTransport_should_start_without_pooling()
+        {
+            var t1 = new TcpTransport(Sys, Sys.Settings.Config.GetConfig("akka.remote.dot-netty.tcp"));
+            try
+            {
+                // bind
+                await t1.Listen();
+
+                // verify that ServerChannel is active and open
+                var sc = t1.ServerChannel;
+                sc.Should().NotBeNull();
+                sc.Active.Should().BeTrue();
+                sc.Open.Should().BeTrue();
+                sc.Allocator.Should().NotBeOfType<PooledByteBufferAllocator>(); // verify we are not using pooling
+                sc.Allocator.Should().BeOfType<UnpooledByteBufferAllocator>(); 
+            }
+            finally
+            {
+                await t1.Shutdown();
+            }
+        }
+
+        [Fact]
+        public void DotNettyTcpTransport_should_communicate_without_pooling()
+        {
+            var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config);
+
+            InitializeLogger(sys2);
+            try
+            {
+                var echo = sys2.ActorOf(act => { act.ReceiveAny((o, context) => context.Sender.Tell(o)); }, "echo");
+
+                var address1 = RARP.For(Sys).Provider.DefaultAddress;
+                var address2 = RARP.For(sys2).Provider.DefaultAddress;
+                var echoPath = new RootActorPath(address2) / "user" / "echo";
+                var probe = CreateTestProbe();
+                Sys.ActorSelection(echoPath).Tell("hello", probe.Ref);
+                probe.ExpectMsg("hello");
+            }
+            finally
+            { 
+                Shutdown(sys2);
+            }
+        }
+    }
+}

src/core/Akka.Remote/Configuration/Remote.conf

@@ -416,6 +416,11 @@ akka {
       # i.e. how long a connect may take until it is timed out
       connection-timeout = 15 s
 
+      # Toggles buffer pooling on and off inside DotNetty.
+      # Only intended to be a work-around for users who are still running on DotNetty v0.4.6-v0.4.7
+      # for the following bug: https://github.com/akkadotnet/akka.net/issues/3370
+      enable-pooling = true
+
       # If set to "<id.of.dispatcher>" then the specified dispatcher
       # will be used to accept inbound connections, and perform IO. If "" then
       # dedicated threads will be used.

src/core/Akka.Remote/Transport/DotNetty/DotNettyTransport.cs

@@ -276,6 +276,7 @@ protected Bootstrap ClientFactory(Address remoteAddress)
                 .Option(ChannelOption.TcpNodelay, Settings.TcpNoDelay)
                 .Option(ChannelOption.ConnectTimeout, Settings.ConnectTimeout)
                 .Option(ChannelOption.AutoRead, false)
+                .Option(ChannelOption.Allocator, Settings.EnableBufferPooling ? (IByteBufferAllocator)new PooledByteBufferAllocator() : new UnpooledByteBufferAllocator())
                 .ChannelFactory(() => Settings.EnforceIpFamily
                     ? new TcpSocketChannel(addressFamily)
                     : new TcpSocketChannel())
@@ -384,6 +385,7 @@ private ServerBootstrap ServerFactory()
                 .Option(ChannelOption.TcpNodelay, Settings.TcpNoDelay)
                 .Option(ChannelOption.AutoRead, false)
                 .Option(ChannelOption.SoBacklog, Settings.Backlog)
+                .Option(ChannelOption.Allocator, Settings.EnableBufferPooling ? (IByteBufferAllocator)new PooledByteBufferAllocator() : new UnpooledByteBufferAllocator())
                 .ChannelFactory(() => Settings.EnforceIpFamily
                     ? new TcpServerSocketChannel(addressFamily)
                     : new TcpServerSocketChannel())

src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs

@@ -17,6 +17,11 @@
 
 namespace Akka.Remote.Transport.DotNetty
 {
+    /// <summary>
+    /// INTERNAL API.
+    /// 
+    /// Defines the settings for the <see cref="DotNettyTransport"/>.
+    /// </summary>
     internal sealed class DotNettyTransportSettings
     {
         public static DotNettyTransportSettings Create(ActorSystem system)
@@ -67,7 +72,8 @@ public static DotNettyTransportSettings Create(Config config)
                 writeBufferLowWaterMark: ToNullableInt(config.GetByteSize("write-buffer-low-water-mark")),
                 backwardsCompatibilityModeEnabled: config.GetBoolean("enable-backwards-compatibility", false),
                 logTransport: config.HasPath("log-transport") && config.GetBoolean("log-transport"),
-                byteOrder: order);
+                byteOrder: order,
+                enableBufferPooling: config.GetBoolean("enable-pooling", true));
         }
 
         private static int? ToNullableInt(long? value) => value.HasValue && value.Value > 0 ? (int?)value.Value : null;
@@ -195,10 +201,18 @@ private static int ComputeWorkerPoolSize(Config config)
         /// </summary>
         public readonly ByteOrder ByteOrder;
 
+        /// <summary>
+        /// Used mostly as a work-around for https://github.com/akkadotnet/akka.net/issues/3370
+        /// on .NET Core on Linux. Should always be left to <c>true</c> unless running DotNetty v0.4.6
+        /// on Linux, which can accidentally release buffers early and corrupt frames. Turn this setting
+        /// to <c>false</c> to disable pooling and work-around this issue at the cost of some performance.
+        /// </summary>
+        public readonly bool EnableBufferPooling;
+
         public DotNettyTransportSettings(TransportMode transportMode, bool enableSsl, TimeSpan connectTimeout, string hostname, string publicHostname,
             int port, int? publicPort, int serverSocketWorkerPoolSize, int clientSocketWorkerPoolSize, int maxFrameSize, SslSettings ssl,
             bool dnsUseIpv6, bool tcpReuseAddr, bool tcpKeepAlive, bool tcpNoDelay, int backlog, bool enforceIpFamily,
-            int? receiveBufferSize, int? sendBufferSize, int? writeBufferHighWaterMark, int? writeBufferLowWaterMark, bool backwardsCompatibilityModeEnabled, bool logTransport, ByteOrder byteOrder)
+            int? receiveBufferSize, int? sendBufferSize, int? writeBufferHighWaterMark, int? writeBufferLowWaterMark, bool backwardsCompatibilityModeEnabled, bool logTransport, ByteOrder byteOrder, bool enableBufferPooling)
         {
             if (maxFrameSize < 32000) throw new ArgumentException("maximum-frame-size must be at least 32000 bytes", nameof(maxFrameSize));
 
@@ -226,6 +240,7 @@ public DotNettyTransportSettings(TransportMode transportMode, bool enableSsl, Ti
             BackwardsCompatibilityModeEnabled = backwardsCompatibilityModeEnabled;
             LogTransport = logTransport;
             ByteOrder = byteOrder;
+            EnableBufferPooling = enableBufferPooling;
         }
     }
     internal enum TransportMode