From 7accfdab3b204a1d926cf5ddf4d472ffedede2d8 Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 16 Apr 2018 19:17:01 -0500 Subject: [PATCH] close #3370 - workaround DotNetty v0.4.6 issues on .NET Core Linux by disabling buffer pooling via HOCON --- .../Bug3370DotNettyLinuxBufferPoolSpec.cs | 90 +++++++++++++++++++ .../Akka.Remote/Configuration/Remote.conf | 5 ++ .../Transport/DotNetty/DotNettyTransport.cs | 2 + .../DotNetty/DotNettyTransportSettings.cs | 19 +++- 4 files changed, 114 insertions(+), 2 deletions(-) create mode 100644 src/core/Akka.Remote.Tests/Transport/Bug3370DotNettyLinuxBufferPoolSpec.cs diff --git a/src/core/Akka.Remote.Tests/Transport/Bug3370DotNettyLinuxBufferPoolSpec.cs b/src/core/Akka.Remote.Tests/Transport/Bug3370DotNettyLinuxBufferPoolSpec.cs new file mode 100644 index 00000000000..deca0e85b09 --- /dev/null +++ b/src/core/Akka.Remote.Tests/Transport/Bug3370DotNettyLinuxBufferPoolSpec.cs @@ -0,0 +1,90 @@ +//----------------------------------------------------------------------- +// +// Copyright (C) 2009-2018 Lightbend Inc. +// Copyright (C) 2013-2018 .NET Foundation +// +//----------------------------------------------------------------------- + +using System.Threading.Tasks; +using Akka.Actor; +using Akka.Actor.Dsl; +using Akka.Configuration; +using Akka.Remote.Transport.DotNetty; +using Akka.TestKit; +using DotNetty.Buffers; +using FluentAssertions; +using Xunit; + +namespace Akka.Remote.Tests.Transport +{ + /// + /// Spec designed to verify that https://github.com/akkadotnet/akka.net/issues/3370 + /// + public class Bug3370DotNettyLinuxBufferPoolSpec : AkkaSpec + { + private static readonly Config Config = ConfigurationFactory.ParseString(@" + akka { + loglevel = DEBUG + actor.provider = ""Akka.Remote.RemoteActorRefProvider,Akka.Remote"" + remote { + dot-netty.tcp { + port = 0 + hostname = ""127.0.0.1"" + log-transport = true + enable-pooling = false + } + } + }"); + + public Bug3370DotNettyLinuxBufferPoolSpec() : base(Config) + { + + } + + [Fact] + public async Task DotNettyTcpTransport_should_start_without_pooling() + { + var t1 = new TcpTransport(Sys, Sys.Settings.Config.GetConfig("akka.remote.dot-netty.tcp")); + try + { + // bind + await t1.Listen(); + + // verify that ServerChannel is active and open + var sc = t1.ServerChannel; + sc.Should().NotBeNull(); + sc.Active.Should().BeTrue(); + sc.Open.Should().BeTrue(); + sc.Allocator.Should().NotBeOfType(); // verify we are not using pooling + sc.Allocator.Should().BeOfType(); + } + finally + { + await t1.Shutdown(); + } + } + + [Fact] + public void DotNettyTcpTransport_should_communicate_without_pooling() + { + var sys2 = ActorSystem.Create(Sys.Name, Sys.Settings.Config); + + InitializeLogger(sys2); + try + { + var echo = sys2.ActorOf(act => { act.ReceiveAny((o, context) => context.Sender.Tell(o)); }, "echo"); + + var address1 = RARP.For(Sys).Provider.DefaultAddress; + var address2 = RARP.For(sys2).Provider.DefaultAddress; + var echoPath = new RootActorPath(address2) / "user" / "echo"; + var probe = CreateTestProbe(); + Sys.ActorSelection(echoPath).Tell("hello", probe.Ref); + probe.ExpectMsg("hello"); + } + finally + { + Shutdown(sys2); + } + } + } +} diff --git a/src/core/Akka.Remote/Configuration/Remote.conf b/src/core/Akka.Remote/Configuration/Remote.conf index 9e3324134b9..07402fa24c9 100644 --- a/src/core/Akka.Remote/Configuration/Remote.conf +++ b/src/core/Akka.Remote/Configuration/Remote.conf @@ -416,6 +416,11 @@ akka { # i.e. how long a connect may take until it is timed out connection-timeout = 15 s + # Toggles buffer pooling on and off inside DotNetty. + # Only intended to be a work-around for users who are still running on DotNetty v0.4.6-v0.4.7 + # for the following bug: https://github.com/akkadotnet/akka.net/issues/3370 + enable-pooling = true + # If set to "" then the specified dispatcher # will be used to accept inbound connections, and perform IO. If "" then # dedicated threads will be used. diff --git a/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransport.cs b/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransport.cs index d3f53f103ea..1cdc39d944b 100644 --- a/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransport.cs +++ b/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransport.cs @@ -276,6 +276,7 @@ protected Bootstrap ClientFactory(Address remoteAddress) .Option(ChannelOption.TcpNodelay, Settings.TcpNoDelay) .Option(ChannelOption.ConnectTimeout, Settings.ConnectTimeout) .Option(ChannelOption.AutoRead, false) + .Option(ChannelOption.Allocator, Settings.EnableBufferPooling ? (IByteBufferAllocator)new PooledByteBufferAllocator() : new UnpooledByteBufferAllocator()) .ChannelFactory(() => Settings.EnforceIpFamily ? new TcpSocketChannel(addressFamily) : new TcpSocketChannel()) @@ -384,6 +385,7 @@ private ServerBootstrap ServerFactory() .Option(ChannelOption.TcpNodelay, Settings.TcpNoDelay) .Option(ChannelOption.AutoRead, false) .Option(ChannelOption.SoBacklog, Settings.Backlog) + .Option(ChannelOption.Allocator, Settings.EnableBufferPooling ? (IByteBufferAllocator)new PooledByteBufferAllocator() : new UnpooledByteBufferAllocator()) .ChannelFactory(() => Settings.EnforceIpFamily ? new TcpServerSocketChannel(addressFamily) : new TcpServerSocketChannel()) diff --git a/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs b/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs index 8450776140d..ebac1435d56 100644 --- a/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs +++ b/src/core/Akka.Remote/Transport/DotNetty/DotNettyTransportSettings.cs @@ -17,6 +17,11 @@ namespace Akka.Remote.Transport.DotNetty { + /// + /// INTERNAL API. + /// + /// Defines the settings for the . + /// internal sealed class DotNettyTransportSettings { public static DotNettyTransportSettings Create(ActorSystem system) @@ -67,7 +72,8 @@ public static DotNettyTransportSettings Create(Config config) writeBufferLowWaterMark: ToNullableInt(config.GetByteSize("write-buffer-low-water-mark")), backwardsCompatibilityModeEnabled: config.GetBoolean("enable-backwards-compatibility", false), logTransport: config.HasPath("log-transport") && config.GetBoolean("log-transport"), - byteOrder: order); + byteOrder: order, + enableBufferPooling: config.GetBoolean("enable-pooling", true)); } private static int? ToNullableInt(long? value) => value.HasValue && value.Value > 0 ? (int?)value.Value : null; @@ -195,10 +201,18 @@ private static int ComputeWorkerPoolSize(Config config) /// public readonly ByteOrder ByteOrder; + /// + /// Used mostly as a work-around for https://github.com/akkadotnet/akka.net/issues/3370 + /// on .NET Core on Linux. Should always be left to true unless running DotNetty v0.4.6 + /// on Linux, which can accidentally release buffers early and corrupt frames. Turn this setting + /// to false to disable pooling and work-around this issue at the cost of some performance. + /// + public readonly bool EnableBufferPooling; + public DotNettyTransportSettings(TransportMode transportMode, bool enableSsl, TimeSpan connectTimeout, string hostname, string publicHostname, int port, int? publicPort, int serverSocketWorkerPoolSize, int clientSocketWorkerPoolSize, int maxFrameSize, SslSettings ssl, bool dnsUseIpv6, bool tcpReuseAddr, bool tcpKeepAlive, bool tcpNoDelay, int backlog, bool enforceIpFamily, - int? receiveBufferSize, int? sendBufferSize, int? writeBufferHighWaterMark, int? writeBufferLowWaterMark, bool backwardsCompatibilityModeEnabled, bool logTransport, ByteOrder byteOrder) + int? receiveBufferSize, int? sendBufferSize, int? writeBufferHighWaterMark, int? writeBufferLowWaterMark, bool backwardsCompatibilityModeEnabled, bool logTransport, ByteOrder byteOrder, bool enableBufferPooling) { if (maxFrameSize < 32000) throw new ArgumentException("maximum-frame-size must be at least 32000 bytes", nameof(maxFrameSize)); @@ -226,6 +240,7 @@ public DotNettyTransportSettings(TransportMode transportMode, bool enableSsl, Ti BackwardsCompatibilityModeEnabled = backwardsCompatibilityModeEnabled; LogTransport = logTransport; ByteOrder = byteOrder; + EnableBufferPooling = enableBufferPooling; } } internal enum TransportMode