Added PhiAccrualFailureDetector warning logging for slow heartbeats

src/core/Akka.API.Tests/CoreAPISpec.ApproveRemote.approved.txt

@@ -147,6 +147,7 @@ namespace Akka.Remote
         public PhiAccrualFailureDetector(double threshold, int maxSampleSize, System.TimeSpan minStdDeviation, System.TimeSpan acceptableHeartbeatPause, System.TimeSpan firstHeartbeatEstimate, Akka.Remote.Clock clock = null) { }
         public PhiAccrualFailureDetector(Akka.Configuration.Config config, Akka.Event.EventStream ev) { }
         protected PhiAccrualFailureDetector(Akka.Remote.Clock clock) { }
+        public string Address { get; set; }
         public override bool IsAvailable { get; }
         public override bool IsMonitoring { get; }
         public override void HeartBeat() { }

src/core/Akka.Remote/DefaultFailureDetectorRegistry.cs

@@ -89,6 +89,14 @@ public void Heartbeat(T resource)
                     else
                     {
                         var newDetector = _factory();
+
+                        switch (newDetector)
+                        {
+                            case PhiAccrualFailureDetector phi:
+                                phi.Address = resource.ToString();
+                                break;
+                        }
+
                         newDetector.HeartBeat();
                         oldTable.Add(resource, newDetector);
                         ResourceToFailureDetector = oldTable;

src/core/Akka.Remote/PhiAccrualFailureDetector.cs

@@ -90,12 +90,13 @@ public PhiAccrualFailureDetector(Config config, EventStream ev)
             _acceptableHeartbeatPause = config.GetTimeSpan("acceptable-heartbeat-pause", null);
             _firstHeartbeatEstimate = config.GetTimeSpan("heartbeat-interval", null);
             state = new State(FirstHeartBeat, null);
+            EventStream = ev ?? Option<EventStream>.None;
         }
 
         /// <summary>
-        /// TBD
+        /// Protected constructor to be used for sub-classing only.
         /// </summary>
-        /// <param name="clock">TBD</param>
+        /// <param name="clock">The clock used fo marking time.</param>
         protected PhiAccrualFailureDetector(Clock clock)
         {
             _clock = clock ?? DefaultClock;
@@ -116,6 +117,13 @@ private HeartbeatHistory FirstHeartBeat
             }
         }
 
+        private Option<EventStream> EventStream { get; }
+
+        /// <summary>
+        /// Address introduced as a mutable property in order to avoid shuffling around API signatures
+        /// </summary>
+        public string Address { get; set; } = "N/A";
+
         /// <summary>
         /// Uses volatile memory and immutability for lockless concurrency.
         /// </summary>
@@ -187,7 +195,15 @@ public override void HeartBeat()
                 //this is a known connection
                 var interval = timestamp - oldState.TimeStamp.Value;
                 //don't use the first heartbeat after failure for the history, since a long pause will skew the stats
-                if (IsTimeStampAvailable(timestamp)) newHistory = (oldState.History + interval);
+                if (IsTimeStampAvailable(timestamp))
+                {
+                    if (interval >= (AcceptableHeartbeatPauseMillis / 3 * 2) && EventStream.HasValue)
+                    {
+                        EventStream.Value.Publish(new Warning(ToString(), GetType(),
+                            $"heartbeat interval is growing too large for address {Address}: {interval} millis"));
+                    }
+                    newHistory = (oldState.History + interval);
+                }
                 else newHistory = oldState.History;
             }