Expose activation working set as a metric

src/Orleans.Core/Diagnostics/Metrics/CatalogInstruments.cs

@@ -24,8 +24,15 @@ internal static class CatalogInstruments
     internal static readonly Counter<int> ActivationsDestroyed = Instruments.Meter.CreateCounter<int>(InstrumentNames.CATALOG_ACTIVATION_DESTROYED);
 
     internal static ObservableGauge<int> ActivationCount;
+
     internal static void RegisterActivationCountObserve(Func<int> observeValue)
     {
         ActivationCount = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_COUNT, observeValue);
     }
+
+    internal static ObservableGauge<int> ActivationWorkingSet;
+    internal static void RegisterActivationWorkingSetObserve(Func<int> observeValue)
+    {
+        ActivationWorkingSet = Instruments.Meter.CreateObservableGauge(InstrumentNames.CATALOG_ACTIVATION_WORKING_SET, observeValue);
+    }
 }

src/Orleans.Core/Diagnostics/Metrics/InstrumentNames.cs

@@ -40,6 +40,7 @@ internal static class InstrumentNames
 
     // Catalog
     public const string CATALOG_ACTIVATION_COUNT = "orleans-catalog-activations";
+    public const string CATALOG_ACTIVATION_WORKING_SET = "orleans-catalog-activation-working-set";
     public const string CATALOG_ACTIVATION_CREATED = "orleans-catalog-activation-created";
     public const string CATALOG_ACTIVATION_DESTROYED = "orleans-catalog-activation-destroyed";
     public const string CATALOG_ACTIVATION_FAILED_TO_ACTIVATE = "orleans-catalog-activation-failed-to-activate";

src/Orleans.Core/Diagnostics/Metrics/MessagingInstruments.cs

@@ -22,7 +22,7 @@ internal static class MessagingInstruments
         internal static readonly Counter<int> ReroutedMessagesCounter = Instruments.Meter.CreateCounter<int>(InstrumentNames.MESSAGING_REROUTED);
         internal static readonly Counter<int> ExpiredMessagesCounter = Instruments.Meter.CreateCounter<int>(InstrumentNames.MESSAGING_EXPIRED);
 
-        internal static readonly Counter<int> ConnectedClient = Instruments.Meter.CreateCounter<int>(InstrumentNames.GATEWAY_CONNECTED_CLIENTS);
+        internal static readonly UpDownCounter<int> ConnectedClient = Instruments.Meter.CreateUpDownCounter<int>(InstrumentNames.GATEWAY_CONNECTED_CLIENTS);
         internal static readonly Counter<int> PingSendCounter = Instruments.Meter.CreateCounter<int>(InstrumentNames.MESSAGING_PINGS_SENT);
         internal static readonly Counter<int> PingReceivedCounter = Instruments.Meter.CreateCounter<int>(InstrumentNames.MESSAGING_PINGS_RECEIVED);
         internal static readonly Counter<int> PingReplyReceivedCounter = Instruments.Meter.CreateCounter<int>(InstrumentNames.MESSAGING_PINGS_REPLYRECEIVED);

src/Orleans.Runtime/Catalog/ActivationData.cs

@@ -180,7 +180,7 @@ private bool IsStuckProcessingMessage
 
         public TimeSpan CollectionAgeLimit => _shared.CollectionAgeLimit;
 
-        public TTarget GetTarget<TTarget>() where TTarget : class => (TTarget)GrainInstance; 
+        public TTarget GetTarget<TTarget>() where TTarget : class => (TTarget)GrainInstance;
 
         TComponent ITargetHolder.GetComponent<TComponent>()
         {
@@ -1202,7 +1202,7 @@ async Task<bool> CallActivateAsync(Dictionary<string, object> requestContextData
                 {
                     CatalogInstruments.ActivationFailedToActivate.Add(1);
 
-                    // Capture the exeption so that it can be propagated to rejection messages
+                    // Capture the exception so that it can be propagated to rejection messages
                     var sourceException = (exception as OrleansLifecycleCanceledException)?.InnerException ?? exception;
                     _shared.Logger.LogError((int)ErrorCode.Catalog_ErrorCallingActivate, sourceException, "Error activating grain {Grain}", this);
 

src/Orleans.Runtime/Catalog/ActivationDirectory.cs

@@ -1,27 +1,46 @@
 using System.Collections;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
+using System.Threading;
 
-namespace Orleans.Runtime
+namespace Orleans.Runtime;
+
+internal sealed class ActivationDirectory : IEnumerable<KeyValuePair<GrainId, IGrainContext>>
 {
-    internal sealed class ActivationDirectory : IEnumerable<KeyValuePair<GrainId, IGrainContext>>
+    private int _activationsCount;
+
+    private readonly ConcurrentDictionary<GrainId, IGrainContext> _activations = new();
+
+    public ActivationDirectory()
     {
-        private readonly ConcurrentDictionary<GrainId, IGrainContext> _activations = new();
+        CatalogInstruments.RegisterActivationCountObserve(() => Count);
+    }
 
-        public int Count => _activations.Count;
+    public int Count => _activationsCount;
 
-        public IGrainContext FindTarget(GrainId key)
+    public IGrainContext FindTarget(GrainId key)
+    {
+        _activations.TryGetValue(key, out var result);
+        return result;
+    }
+
+    public void RecordNewTarget(IGrainContext target)
+    {
+        if (_activations.TryAdd(target.GrainId, target))
         {
-            _activations.TryGetValue(key, out var result);
-            return result;
+            Interlocked.Increment(ref _activationsCount);
         }
+    }
 
-        public void RecordNewTarget(IGrainContext target) => _activations.TryAdd(target.GrainId, target);
-
-        public void RemoveTarget(IGrainContext target) => _activations.TryRemove(KeyValuePair.Create(target.GrainId, target));
+    public void RemoveTarget(IGrainContext target)
+    {
+        if (_activations.TryRemove(KeyValuePair.Create(target.GrainId, target)))
+        {
+            Interlocked.Decrement(ref _activationsCount);
+        }
+    }
 
-        public IEnumerator<KeyValuePair<GrainId, IGrainContext>> GetEnumerator() => _activations.GetEnumerator();
+    public IEnumerator<KeyValuePair<GrainId, IGrainContext>> GetEnumerator() => _activations.GetEnumerator();
 
-        IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
-    }
+    IEnumerator IEnumerable.GetEnumerator() => GetEnumerator();
 }

src/Orleans.Runtime/Catalog/ActivationWorkingSet.cs

@@ -35,6 +35,7 @@ public ActivationWorkingSet(
             _logger = logger;
             _scanPeriodTimer = asyncTimerFactory.Create(TimeSpan.FromMilliseconds(100), nameof(ActivationWorkingSet) + "." + nameof(MonitorWorkingSet));
             _observers = observers.ToList();
+            CatalogInstruments.RegisterActivationWorkingSetObserve(() => Count);
         }
 
         public int Count => _activeCount;

src/Orleans.Runtime/Catalog/Catalog.cs

@@ -63,7 +63,6 @@ public Catalog(
 
             GC.GetTotalMemory(true); // need to call once w/true to ensure false returns OK value
 
-            CatalogInstruments.RegisterActivationCountObserve(() => activations.Count);
             MessagingProcessingInstruments.RegisterActivationDataAllObserve(() =>
             {
                 long counter = 0;