Feature/node stats 6.3 (#3359)

* fix #3329 Add Adaptive selection statistics to NodeStats

* add ingest statistics

src/Nest/Cluster/NodesStats/AdaptiveSelectionStats.cs

@@ -0,0 +1,52 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public class AdaptiveSelectionStats
+	{
+		/// <summary>
+		/// The number of outstanding search requests from the node these stats are for to the keyed node.
+		/// </summary>
+		[JsonProperty("outgoing_searches")]
+		public long OutgoingSearches { get; internal set; }
+
+		/// <summary>
+		/// The exponentially weighted moving average queue size of search requests on the keyed node.
+		/// </summary>
+		[JsonProperty("avg_queue_size")]
+		public long AverageQueueSize { get; internal set; }
+
+		/// <summary>
+		/// The exponentially weighted moving average service time of search requests on the keyed node.
+		/// </summary>
+		/// <remarks>only set when requesting human readable response</remarks>
+		[JsonProperty("avg_service_time")]
+		public string AverageServiceTime { get; internal set; }
+
+		/// <summary>
+		/// The exponentially weighted moving average service time of search requests on the keyed node.
+		/// </summary>
+		[JsonProperty("avg_service_time_ns")]
+		public long AverageServiceTimeInNanoseconds { get; internal set; }
+
+		/// <summary>
+		/// The exponentially weighted moving average response time of search requests on the keyed node.
+		/// </summary>
+		/// <remarks>only set when requesting human readable response</remarks>
+		[JsonProperty("avg_response_time")]
+		public long AverageResponseTime { get; internal set; }
+
+		/// <summary>
+		/// The exponentially weighted moving average response time of search requests on the keyed node.
+		/// </summary>
+		[JsonProperty("avg_response_time_ns")]
+		public long AverageResponseTimeInNanoseconds { get; internal set; }
+
+		/// <summary>
+		/// The rank of this node; used for shard selection when routing search requests.
+		/// </summary>
+		[JsonProperty("rank")]
+		public string Rank { get; internal set; }
+	}
+}

src/Nest/Cluster/NodesStats/NodeStats.cs

@@ -60,8 +60,17 @@ public class NodeStats
 
 		[JsonProperty("http")]
 		public HttpStats Http { get; internal set; }
+
+		[JsonProperty("ingest")]
+		public NodeIngestStats Ingest { get; internal set; }
+
+		[JsonProperty("adaptive_selection")]
+		[JsonConverter(typeof(VerbatimDictionaryKeysJsonConverter<string, AdaptiveSelectionStats>))]
+		public IReadOnlyDictionary<string, AdaptiveSelectionStats> AdaptiveSelection { get; internal set; }
+			= EmptyReadOnly<string, AdaptiveSelectionStats>.Dictionary;
 	}
 
+
 	[JsonObject]
 	public class ScriptStats
 	{

src/Nest/Cluster/NodesStats/Statistics/IngestStats.cs

@@ -0,0 +1,32 @@
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	[JsonObject]
+	public class IngestStats
+	{
+		/// <summary>
+		/// The total number of document ingested during the lifetime of this node
+		/// </summary>
+		[JsonProperty("count")]
+		public long Count { get; set; }
+
+		/// <summary>
+		/// The total time spent on ingest preprocessing documents during the lifetime of this node
+		/// </summary>
+		[JsonProperty("time_in_millis")]
+		public long TimeInMilliseconds { get; set; }
+
+		/// <summary>
+		/// The total number of documents currently being ingested.
+		/// </summary>
+		[JsonProperty("current")]
+		public long Current { get; set; }
+
+		/// <summary>
+		/// The total number ingest preprocessing operations failed during the lifetime of this node
+		/// </summary>
+		[JsonProperty("failed")]
+		public long Failed { get; set; }
+	}
+}

src/Nest/Cluster/NodesStats/Statistics/NodeIngestStats.cs

@@ -0,0 +1,18 @@
+using System.Collections.Generic;
+using Newtonsoft.Json;
+
+namespace Nest
+{
+	public class NodeIngestStats
+	{
+		/// <summary> Overal global ingest statistics </summary>
+		[JsonProperty("total")]
+		public IngestStats Total { get; set; }
+
+		/// <summary> Per pipeline ingest statistics </summary>
+		[JsonProperty("pipelines")]
+		[JsonConverter(typeof(VerbatimDictionaryKeysJsonConverter<string, IngestStats>))]
+		public IReadOnlyDictionary<string, IngestStats> Pipelines { get; internal set; }
+			= EmptyReadOnly<string, IngestStats>.Dictionary;
+	}
+}

src/Tests/Tests.Core/ManagedElasticsearch/NodeSeeders/DefaultSeeder.cs

@@ -37,6 +37,7 @@ public void SeedNode()
 		{
 			if (!TestClient.Configuration.ForceReseed && this.AlreadySeeded()) return;
 			// Ensure a clean slate by deleting everything regardless of whether they may already exist
+			this.ClusterSettings();
 			this.DeleteIndicesAndTemplates();
 			// and now recreate everything
 			this.CreateIndicesAndSeedIndexData();
@@ -47,6 +48,17 @@ public void SeedNode()
 		// If raw_fields exists assume this cluster is already seeded.
 		private bool AlreadySeeded() => this.Client.IndexTemplateExists(TestsIndexTemplateName).Exists;
 
+		public void ClusterSettings()
+		{
+			var putSettingsResponse = this.Client.ClusterPutSettings(s=>s
+				.Transient(t=>t
+					.Add("cluster.routing.use_adaptive_replica_selection", true)
+				)
+			);
+
+			putSettingsResponse.ShouldBeValid();
+		}
+
 		public void DeleteIndicesAndTemplates()
 		{
 			if (this.Client.IndexTemplateExists(TestsIndexTemplateName).Exists)

src/Tests/Tests/Cluster/NodesStats/NodesStatsApiTests.cs

@@ -5,6 +5,7 @@
 using Nest;
 using Tests.Core.Extensions;
 using Tests.Core.ManagedElasticsearch.Clusters;
+using Tests.Domain;
 using Tests.Framework;
 using Tests.Framework.Integration;
 using Tests.Framework.ManagedElasticsearch.Clusters;
@@ -28,6 +29,22 @@ protected override LazyResponses ClientUsage() => Calls(
 		protected override HttpMethod HttpMethod => HttpMethod.GET;
 		protected override string UrlPath => "/_nodes/stats";
 
+		protected override void IntegrationSetup(IElasticClient client, CallUniqueValues values)
+		{
+			// performing a bunch of search actions to make sure some stats have been gathered
+			// even if this api is tested in isolation
+			for (var i = 0; i < 5; i++)
+			{
+				var searchResult = client.MultiSearch(m => m
+					.Search<Project>(s => s.MatchAll().Size(0))
+					.Search<Project>(s => s.MatchAll().Size(0))
+					.Search<Project>(s => s.MatchAll().Size(0))
+					.Search<Project>(s => s.MatchAll().Size(0))
+				);
+				searchResult.ShouldBeValid();
+			}
+		}
+
 		protected override void ExpectResponse(INodesStatsResponse response)
 		{
 			response.ClusterName.Should().NotBeNullOrWhiteSpace();
@@ -47,6 +64,25 @@ protected override void ExpectResponse(INodesStatsResponse response)
 			Assert(node.FileSystem);
 			Assert(node.ThreadPool);
 			Assert(node.Jvm);
+			Assert(node.AdaptiveSelection);
+			Assert(node.Ingest);
+		}
+
+		protected void Assert(NodeIngestStats nodeIngestStats)
+		{
+			nodeIngestStats.Should().NotBeNull();
+			nodeIngestStats.Total.Should().NotBeNull();
+			nodeIngestStats.Pipelines.Should().NotBeNull().And.NotBeEmpty();
+		}
+
+		protected void Assert(IReadOnlyDictionary<string, AdaptiveSelectionStats> adaptiveSelectionStats)
+		{
+			adaptiveSelectionStats.Should().NotBeNull(); //.And.NotBeEmpty();
+			// TODO: for the single node case, this is empty. Should it be?
+			//	var nodeSelectionStats = adaptiveSelectionStats.First().Value;
+			//	nodeSelectionStats.Rank.Should().NotBeNullOrWhiteSpace();
+			//	nodeSelectionStats.AverageResponseTimeInNanoseconds.Should().BeGreaterThan(0);
+			//	nodeSelectionStats.AverageServiceTimeInNanoseconds.Should().BeGreaterThan(0);
 		}
 
 		protected void Assert(NodeStats node)