Less allocations for node processing

src/Nethermind/Nethermind.Core/Collections/ArrayPoolList.cs

@@ -121,6 +121,49 @@ void ICollection.CopyTo(Array array, int index)
     }
 
     public int Count { get; private set; } = 0;
+    public void ReduceCount(int count)
+    {
+        GuardDispose();
+        var oldCount = Count;
+        if (count == oldCount) return;
+
+        if (count > oldCount)
+        {
+            ThrowOnlyReduce(count);
+        }
+
+        Count = count;
+        if (count < _capacity / 2)
+        {
+            // Reduced to less than half of the capacity, resize the array.
+            T[] newArray = _arrayPool.Rent(count);
+            _array.AsSpan(0, count).CopyTo(newArray);
+            T[] oldArray = Interlocked.Exchange(ref _array, newArray);
+            _capacity = newArray.Length;
+            _arrayPool.Return(oldArray);
+        }
+        else if (RuntimeHelpers.IsReferenceOrContainsReferences<T>())
+        {
+            // Release any references to the objects in the array that are no longer in use.
+            Array.Clear(_array, count, oldCount);
+        }
+
+        void ThrowOnlyReduce(int count)
+        {
+            throw new ArgumentException($"Count can only be reduced. {count} is larger than {Count}", nameof(count));
+        }
+    }
+
+    public void Sort(Comparison<T> comparison)
+    {
+        ArgumentNullException.ThrowIfNull(comparison);
+        GuardDispose();
+
+        if (Count > 1)
+        {
+            _array.AsSpan(0, Count).Sort(comparison);
+        }
+    }
 
     public int Capacity => _capacity;
 

src/Nethermind/Nethermind.Network.Discovery/Lifecycle/NodeLifecycleManager.cs

@@ -212,10 +212,16 @@ public void ProcessFindNodeMsg(FindNodeMsg msg)
         NodeStats.AddNodeStatsEvent(NodeStatsEventType.DiscoveryFindNodeIn);
         RefreshNodeContactTime();
 
-        Node[] nodes = _nodeTable
-            .GetClosestNodes(msg.SearchedNodeId)
-            .Take(12) // Otherwise the payload may become too big, which is out of spec.
-            .ToArray();
+        // 12 otherwise the payload may become too big, which is out of spec.
+        var closestNodes = _nodeTable.GetClosestNodes(msg.SearchedNodeId, bucketSize: 12);
+        Node[] nodes = new Node[closestNodes.Count];
+        int count = 0;
+        foreach (Node node in closestNodes)
+        {
+            nodes[count] = node;
+            count++;
+        }
+
         SendNeighbors(nodes);
     }
 

src/Nethermind/Nethermind.Network.Discovery/NodesLocator.cs

@@ -33,9 +33,9 @@ public void Initialize(Node masterNode)
         _masterNode = masterNode;
     }
 
-    public async Task LocateNodesAsync(CancellationToken cancellationToken)
+    public Task LocateNodesAsync(CancellationToken cancellationToken)
     {
-        await LocateNodesAsync(null, cancellationToken);
+        return LocateNodesAsync(null, cancellationToken);
     }
 
     public async Task LocateNodesAsync(byte[]? searchedNodeId, CancellationToken cancellationToken)
@@ -59,16 +59,37 @@ public async Task LocateNodesAsync(byte[]? searchedNodeId, CancellationToken can
             int attemptsCount = 0;
             while (true)
             {
-                //if searched node is not specified master node is used
-                IEnumerable<Node> closestNodes = searchedNodeId is not null ? _nodeTable.GetClosestNodes(searchedNodeId) : _nodeTable.GetClosestNodes();
-
                 candidatesCount = 0;
-                foreach (Node closestNode in closestNodes.Where(node => !alreadyTriedNodes.Contains(node.IdHash)))
+                if (searchedNodeId is not null)
+                {
+                    foreach (Node closestNode in _nodeTable.GetClosestNodes(searchedNodeId))
+                    {
+                        if (alreadyTriedNodes.Contains(closestNode.IdHash))
+                        {
+                            continue;
+                        }
+
+                        tryCandidates[candidatesCount++] = closestNode;
+                        if (candidatesCount > tryCandidates.Length - 1)
+                        {
+                            break;
+                        }
+                    }
+                }
+                else
                 {
-                    tryCandidates[candidatesCount++] = closestNode;
-                    if (candidatesCount > tryCandidates.Length - 1)
+                    foreach (Node closestNode in _nodeTable.GetClosestNodes())
                     {
-                        break;
+                        if (alreadyTriedNodes.Contains(closestNode.IdHash))
+                        {
+                            continue;
+                        }
+
+                        tryCandidates[candidatesCount++] = closestNode;
+                        if (candidatesCount > tryCandidates.Length - 1)
+                        {
+                            break;
+                        }
                     }
                 }
 
@@ -134,7 +155,14 @@ public async Task LocateNodesAsync(byte[]? searchedNodeId, CancellationToken can
                 }
             }
         }
-        int nodesCountAfterDiscovery = _nodeTable.Buckets.Sum(x => x.BondedItemsCount);
+
+        int nodesCountAfterDiscovery = 0;
+        var buckets = _nodeTable.Buckets;
+        for (int i = 0; i < buckets.Length; i++)
+        {
+            nodesCountAfterDiscovery += buckets[i].BondedItemsCount;
+        }
+
         if (_logger.IsDebug) _logger.Debug($"Finished discovery cycle, tried contacting {alreadyTriedNodes.Count} nodes. All nodes count before the process: {nodesCountBeforeDiscovery}, after the process: {nodesCountAfterDiscovery}");
 
         if (_logger.IsTrace)
@@ -172,7 +200,7 @@ private int NodesCountBeforeDiscovery
 
     private void LogNodeTable()
     {
-        IEnumerable<NodeBucket> nonEmptyBuckets = _nodeTable.Buckets.Where(x => x.BondedItems.Any());
+        IEnumerable<NodeBucket> nonEmptyBuckets = _nodeTable.Buckets.Where(x => x.AnyBondedItems());
         StringBuilder sb = new();
 
         int length = 0;

src/Nethermind/Nethermind.Network.Discovery/RoutingTable/INodeTable.cs

@@ -3,6 +3,7 @@
 
 using Nethermind.Core.Crypto;
 using Nethermind.Stats.Model;
+using static Nethermind.Network.Discovery.RoutingTable.NodeTable;
 
 namespace Nethermind.Network.Discovery.RoutingTable;
 
@@ -18,10 +19,11 @@ public interface INodeTable
     /// <summary>
     /// GetClosestNodes to MasterNode
     /// </summary>
-    IEnumerable<Node> GetClosestNodes();
+    ClosestNodesEnumerator GetClosestNodes();
 
     /// <summary>
     /// GetClosestNodes to provided Node
     /// </summary>
-    IEnumerable<Node> GetClosestNodes(byte[] nodeId);
+    ClosestNodesFromNodeEnumerator GetClosestNodes(byte[] nodeId);
+    ClosestNodesFromNodeEnumerator GetClosestNodes(byte[] nodeId, int bucketSize);
 }

src/Nethermind/Nethermind.Network.Discovery/RoutingTable/NodeBucket.cs

@@ -1,6 +1,7 @@
 // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited
 // SPDX-License-Identifier: LGPL-3.0-only
 
+using System.Collections;
 using System.Diagnostics;
 using Nethermind.Stats.Model;
 
@@ -28,26 +29,71 @@ public NodeBucket(int distance, int bucketSize, float dropFullBucketProbability
 
     public int BucketSize { get; }
 
-    public IEnumerable<NodeBucketItem> BondedItems
+    public bool AnyBondedItems()
     {
-        get
+        foreach (NodeBucketItem _ in BondedItems)
         {
-            lock (_nodeBucketLock)
+            return true;
+        }
+
+        return false;
+    }
+
+    public BondedItemsEnumerator BondedItems
+        => new(this);
+
+    public struct BondedItemsEnumerator : IEnumerator<NodeBucketItem>, IEnumerable<NodeBucketItem>
+    {
+        private NodeBucket _nodeBucket;
+        private LinkedListNode<NodeBucketItem>? _currentNode;
+        private DateTime _referenceTime;
+
+        public BondedItemsEnumerator(NodeBucket nodeBucket)
+        {
+            _nodeBucket = nodeBucket;
+            _referenceTime = DateTime.UtcNow;
+            Monitor.Enter(_nodeBucket._nodeBucketLock);
+            _currentNode = nodeBucket._items.Last;
+            Current = null!;
+        }
+
+        public NodeBucketItem Current { get; private set; }
+
+        object IEnumerator.Current => Current;
+
+        public bool MoveNext()
+        {
+            while (_currentNode is not null)
             {
-                LinkedListNode<NodeBucketItem>? node = _items.Last;
-                DateTime utcNow = DateTime.UtcNow;
-                while (node is not null)
+                Current = _currentNode.Value;
+                _currentNode = _currentNode.Previous;
+                if (Current.IsBonded(_referenceTime))
                 {
-                    if (!node.Value.IsBonded(utcNow))
-                    {
-                        break;
-                    }
-
-                    yield return node.Value;
-                    node = node.Previous;
+                    return true;
                 }
             }
+
+            Current = null!;
+            return false;
+        }
+
+        void IEnumerator.Reset() => throw new NotSupportedException();
+
+        public void Dispose()
+        {
+            if (_nodeBucket is not null)
+            {
+                Monitor.Exit(_nodeBucket._nodeBucketLock);
+            }
+            _nodeBucket = null!;
         }
+        public BondedItemsEnumerator GetEnumerator() => this;
+
+        IEnumerator<NodeBucketItem> IEnumerable<NodeBucketItem>.GetEnumerator()
+            => GetEnumerator();
+
+        IEnumerator IEnumerable.GetEnumerator()
+            => GetEnumerator();
     }
 
     public int BondedItemsCount