Skip to content

Commit e1ef3cd

Browse files
russcamrusscam
authored
Add moving_percentiles aggregation (#4887)
Relates: elastic/elasticsearch#55441
1 parent aefc43d commit e1ef3cd

File tree

8 files changed

+305
-0
lines changed

8 files changed

+305
-0
lines changed

‎docs/aggregations.asciidoc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,8 @@ There are many different types of pipeline aggregation, each computing different
277277

278278
* <<moving-function-aggregation-usage,Moving Function Aggregation Usage>>
279279

280+
* <<moving-percentiles-aggregation-usage,Moving Percentiles Aggregation Usage>>
281+
280282
* <<normalize-aggregation-usage,Normalize Aggregation Usage>>
281283

282284
* <<percentiles-bucket-aggregation-usage,Percentiles Bucket Aggregation Usage>>
@@ -323,6 +325,8 @@ include::aggregations/pipeline/moving-average/moving-average-simple-aggregation-
323325

324326
include::aggregations/pipeline/moving-function/moving-function-aggregation-usage.asciidoc[]
325327

328+
include::aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc[]
329+
326330
include::aggregations/pipeline/normalize/normalize-aggregation-usage.asciidoc[]
327331

328332
include::aggregations/pipeline/percentiles-bucket/percentiles-bucket-aggregation-usage.asciidoc[]

‎docs/aggregations/pipeline/moving-percentiles/moving-percentiles-aggregation-usage.asciidoc‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,110 @@
1+
:ref_current: https://www.elastic.co/guide/en/elasticsearch/reference/master
2+
3+
:github: https://github.com/elastic/elasticsearch-net
4+
5+
:nuget: https://www.nuget.org/packages
6+
7+
////
8+
IMPORTANT NOTE
9+
==============
10+
This file has been generated from https://github.com/elastic/elasticsearch-net/tree/master/src/Tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs.
11+
If you wish to submit a PR for any spelling mistakes, typos or grammatical errors for this file,
12+
please modify the original csharp file found at the link and submit the PR with that change. Thanks!
13+
////
14+
15+
[[moving-percentiles-aggregation-usage]]
16+
=== Moving Percentiles Aggregation Usage
17+
18+
Given an ordered series of percentiles, the Moving Percentile aggregation will slide a window across those
19+
percentiles and allow the user to compute the cumulative percentile.
20+
21+
This is conceptually very similar to the Moving Function pipeline aggregation, except it works on the percentiles sketches instead of the actual buckets values.
22+
23+
NOTE: Available in Elasticsearch 7.9.0+ with at least basic license level
24+
25+
==== Fluent DSL example
26+
27+
[source,csharp]
28+
----
29+
a => a
30+
.DateHistogram("projects_started_per_month", dh => dh
31+
.Field(p => p.StartedOn)
32+
.CalendarInterval(DateInterval.Month)
33+
.MinimumDocumentCount(0)
34+
.Aggregations(aa => aa
35+
.Percentiles("percentiles", sm => sm
36+
.Field(p => p.NumberOfCommits)
37+
)
38+
.MovingPercentiles("moving_percentiles", mv => mv
39+
.BucketsPath("percentiles")
40+
.Window(10)
41+
)
42+
)
43+
)
44+
----
45+
46+
==== Object Initializer syntax example
47+
48+
[source,csharp]
49+
----
50+
new DateHistogramAggregation("projects_started_per_month")
51+
{
52+
Field = "startedOn",
53+
CalendarInterval = DateInterval.Month,
54+
MinimumDocumentCount = 0,
55+
Aggregations =
56+
new PercentilesAggregation("percentiles", "numberOfCommits")
57+
&& new MovingPercentilesAggregation("moving_percentiles", "percentiles")
58+
{
59+
Window = 10
60+
}
61+
}
62+
----
63+
64+
[source,javascript]
65+
.Example json output
66+
----
67+
{
68+
"projects_started_per_month": {
69+
"date_histogram": {
70+
"field": "startedOn",
71+
"calendar_interval": "month",
72+
"min_doc_count": 0
73+
},
74+
"aggs": {
75+
"percentiles": {
76+
"percentiles": {
77+
"field": "numberOfCommits"
78+
}
79+
},
80+
"moving_percentiles": {
81+
"moving_percentiles": {
82+
"buckets_path": "percentiles",
83+
"window": 10
84+
}
85+
}
86+
}
87+
}
88+
}
89+
----
90+
91+
==== Handling Responses
92+
93+
[source,csharp]
94+
----
95+
response.ShouldBeValid();
96+
97+
var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month");
98+
projectsPerMonth.Should().NotBeNull();
99+
projectsPerMonth.Buckets.Should().NotBeNull();
100+
projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0);
101+
102+
// percentiles not calculated for the first bucket
103+
foreach (var item in projectsPerMonth.Buckets.Skip(1))
104+
{
105+
var movingPercentiles = item.MovingPercentiles("moving_percentiles");
106+
movingPercentiles.Should().NotBeNull();
107+
movingPercentiles.Items.Should().NotBeNull();
108+
}
109+
----
110+

‎src/Nest/Aggregations/AggregateDictionary.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -98,6 +98,8 @@ public ScriptedMetricAggregate ScriptedMetric(string key)
9898

9999
public PercentilesAggregate PercentilesBucket(string key) => TryGet<PercentilesAggregate>(key);
100100

101+
public PercentilesAggregate MovingPercentiles(string key) => TryGet<PercentilesAggregate>(key);
102+
101103
public PercentilesAggregate PercentileRanks(string key) => TryGet<PercentilesAggregate>(key);
102104

103105
public TopHitsAggregate TopHits(string key) => TryGet<TopHitsAggregate>(key);

‎src/Nest/Aggregations/AggregationContainer.cs‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -199,6 +199,10 @@ public interface IAggregationContainer
199199
[DataMember(Name = "moving_fn")]
200200
IMovingFunctionAggregation MovingFunction { get; set; }
201201

202+
/// <inheritdoc cref="IMovingPercentilesAggregation"/>
203+
[DataMember(Name = "moving_percentiles")]
204+
IMovingPercentilesAggregation MovingPercentiles { get; set; }
205+
202206
[DataMember(Name = "nested")]
203207
INestedAggregation Nested { get; set; }
204208

@@ -358,6 +362,9 @@ public class AggregationContainer : IAggregationContainer
358362

359363
public IMovingFunctionAggregation MovingFunction { get; set; }
360364

365+
/// <inheritdoc cref="IMovingPercentilesAggregation"/>
366+
public IMovingPercentilesAggregation MovingPercentiles { get; set; }
367+
361368
public INestedAggregation Nested { get; set; }
362369

363370
/// <inheritdoc cref="INormalizeAggregation"/>
@@ -518,6 +525,8 @@ public class AggregationContainerDescriptor<T> : DescriptorBase<AggregationConta
518525

519526
IMovingFunctionAggregation IAggregationContainer.MovingFunction { get; set; }
520527

528+
IMovingPercentilesAggregation IAggregationContainer.MovingPercentiles { get; set; }
529+
521530
INestedAggregation IAggregationContainer.Nested { get; set; }
522531

523532
INormalizeAggregation IAggregationContainer.Normalize { get; set; }
@@ -809,6 +818,11 @@ Func<MovingFunctionAggregationDescriptor, IMovingFunctionAggregation> selector
809818
) =>
810819
_SetInnerAggregation(name, selector, (a, d) => a.MovingFunction = d);
811820

821+
public AggregationContainerDescriptor<T> MovingPercentiles(string name,
822+
Func<MovingPercentilesAggregationDescriptor, IMovingPercentilesAggregation> selector
823+
) =>
824+
_SetInnerAggregation(name, selector, (a, d) => a.MovingPercentiles = d);
825+
812826
public AggregationContainerDescriptor<T> CumulativeSum(string name,
813827
Func<CumulativeSumAggregationDescriptor, ICumulativeSumAggregation> selector
814828
) =>

‎src/Nest/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregation.cs‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System.Runtime.Serialization;
6+
using Elasticsearch.Net.Utf8Json;
7+
8+
namespace Nest
9+
{
10+
/// <summary>
11+
/// the Moving Percentile aggregation will slide a window across those percentiles and allow the user to compute the cumulative percentile.
12+
/// </summary>
13+
[InterfaceDataContract]
14+
[ReadAs(typeof(MovingPercentilesAggregation))]
15+
public interface IMovingPercentilesAggregation : IPipelineAggregation
16+
{
17+
/// <summary>
18+
/// The size of window to "slide" across the histogram.
19+
/// </summary>
20+
[DataMember(Name ="window")]
21+
int? Window { get; set; }
22+
23+
/// <summary>
24+
/// Shift of window position.
25+
/// </summary>
26+
[DataMember(Name ="shift")]
27+
int? Shift { get; set; }
28+
}
29+
30+
/// <inheritdoc cref="IMovingPercentilesAggregation"/>
31+
public class MovingPercentilesAggregation
32+
: PipelineAggregationBase, IMovingPercentilesAggregation
33+
{
34+
internal MovingPercentilesAggregation() { }
35+
36+
public MovingPercentilesAggregation(string name, SingleBucketsPath bucketsPath)
37+
: base(name, bucketsPath) { }
38+
39+
/// <inheritdoc cref="IMovingPercentilesAggregation.Window"/>
40+
public int? Window { get; set; }
41+
42+
/// <inheritdoc cref="IMovingPercentilesAggregation.Shift"/>
43+
public int? Shift { get; set; }
44+
45+
internal override void WrapInContainer(AggregationContainer c) => c.MovingPercentiles = this;
46+
}
47+
48+
public class MovingPercentilesAggregationDescriptor
49+
: PipelineAggregationDescriptorBase<MovingPercentilesAggregationDescriptor, IMovingPercentilesAggregation, SingleBucketsPath>
50+
, IMovingPercentilesAggregation
51+
{
52+
int? IMovingPercentilesAggregation.Window { get; set; }
53+
int? IMovingPercentilesAggregation.Shift { get; set; }
54+
55+
/// <inheritdoc cref="IMovingPercentilesAggregation.Window"/>
56+
public MovingPercentilesAggregationDescriptor Window(int? windowSize) => Assign(windowSize, (a, v) => a.Window = v);
57+
58+
/// <inheritdoc cref="IMovingPercentilesAggregation.Shift"/>
59+
public MovingPercentilesAggregationDescriptor Shift(int? shift) => Assign(shift, (a, v) => a.Shift = v);
60+
}
61+
}

‎src/Nest/Aggregations/Visitor/AggregationVisitor.cs‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,8 @@ public interface IAggregationVisitor
114114

115115
void Visit(IMovingAverageAggregation aggregation);
116116

117+
void Visit(IMovingPercentilesAggregation aggregation);
118+
117119
void Visit(ICumulativeSumAggregation aggregation);
118120

119121
void Visit(ICumulativeCardinalityAggregation aggregation);
@@ -219,6 +221,8 @@ public virtual void Visit(ISerialDifferencingAggregation aggregation) { }
219221

220222
public virtual void Visit(IMovingAverageAggregation aggregation) { }
221223

224+
public virtual void Visit(IMovingPercentilesAggregation aggregation) { }
225+
222226
public virtual void Visit(IMinBucketAggregation aggregation) { }
223227

224228
public virtual void Visit(IDerivativeAggregation aggregation) { }

‎src/Nest/Aggregations/Visitor/AggregationWalker.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -117,6 +117,7 @@ public void Walk(IAggregationContainer aggregation, IAggregationVisitor visitor)
117117
Accept(v, d.Aggregations);
118118
});
119119
AcceptAggregation(aggregation.MovingAverage, visitor, (v, d) => v.Visit(d));
120+
AcceptAggregation(aggregation.MovingPercentiles, visitor, (v, d) => v.Visit(d));
120121
AcceptAggregation(aggregation.Nested, visitor, (v, d) =>
121122
{
122123
v.Visit(d);

‎tests/Tests/Aggregations/Pipeline/MovingPercentiles/MovingPercentilesAggregationUsageTests.cs‎

Lines changed: 109 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,109 @@
1+
// Licensed to Elasticsearch B.V under one or more agreements.
2+
// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
3+
// See the LICENSE file in the project root for more information
4+
5+
using System;
6+
using System.Linq;
7+
using Elastic.Elasticsearch.Xunit.XunitPlumbing;
8+
using FluentAssertions;
9+
using Nest;
10+
using Tests.Core.Extensions;
11+
using Tests.Core.ManagedElasticsearch.Clusters;
12+
using Tests.Domain;
13+
using Tests.Framework.EndpointTests.TestState;
14+
15+
namespace Tests.Aggregations.Pipeline.MovingPercentiles
16+
{
17+
/**
18+
* Given an ordered series of percentiles, the Moving Percentile aggregation will slide a window across those
19+
* percentiles and allow the user to compute the cumulative percentile.
20+
*
21+
* This is conceptually very similar to the Moving Function pipeline aggregation, except it works on the percentiles sketches instead of the actual buckets values.
22+
*
23+
* NOTE: Available in Elasticsearch 7.9.0+ with at least basic license level
24+
*/
25+
[SkipVersion("<7.9.0", "introduced in 7.9.0+")]
26+
public class MovingPercentilesAggregationUsageTests : AggregationUsageTestBase
27+
{
28+
public MovingPercentilesAggregationUsageTests(ReadOnlyCluster cluster, EndpointUsage usage) : base(cluster, usage) { }
29+
30+
protected override object AggregationJson => new
31+
{
32+
projects_started_per_month = new
33+
{
34+
date_histogram = new
35+
{
36+
field = "startedOn",
37+
calendar_interval = "month",
38+
min_doc_count = 0
39+
},
40+
aggs = new
41+
{
42+
percentiles = new
43+
{
44+
percentiles = new
45+
{
46+
field = "numberOfCommits"
47+
}
48+
},
49+
moving_percentiles = new
50+
{
51+
moving_percentiles = new
52+
{
53+
buckets_path = "percentiles",
54+
window = 10,
55+
}
56+
}
57+
}
58+
}
59+
};
60+
61+
protected override Func<AggregationContainerDescriptor<Project>, IAggregationContainer> FluentAggs => a => a
62+
.DateHistogram("projects_started_per_month", dh => dh
63+
.Field(p => p.StartedOn)
64+
.CalendarInterval(DateInterval.Month)
65+
.MinimumDocumentCount(0)
66+
.Aggregations(aa => aa
67+
.Percentiles("percentiles", sm => sm
68+
.Field(p => p.NumberOfCommits)
69+
)
70+
.MovingPercentiles("moving_percentiles", mv => mv
71+
.BucketsPath("percentiles")
72+
.Window(10)
73+
)
74+
)
75+
);
76+
77+
protected override AggregationDictionary InitializerAggs =>
78+
new DateHistogramAggregation("projects_started_per_month")
79+
{
80+
Field = "startedOn",
81+
CalendarInterval = DateInterval.Month,
82+
MinimumDocumentCount = 0,
83+
Aggregations =
84+
new PercentilesAggregation("percentiles", "numberOfCommits")
85+
&& new MovingPercentilesAggregation("moving_percentiles", "percentiles")
86+
{
87+
Window = 10
88+
}
89+
};
90+
91+
protected override void ExpectResponse(ISearchResponse<Project> response)
92+
{
93+
response.ShouldBeValid();
94+
95+
var projectsPerMonth = response.Aggregations.DateHistogram("projects_started_per_month");
96+
projectsPerMonth.Should().NotBeNull();
97+
projectsPerMonth.Buckets.Should().NotBeNull();
98+
projectsPerMonth.Buckets.Count.Should().BeGreaterThan(0);
99+
100+
// percentiles not calculated for the first bucket
101+
foreach (var item in projectsPerMonth.Buckets.Skip(1))
102+
{
103+
var movingPercentiles = item.MovingPercentiles("moving_percentiles");
104+
movingPercentiles.Should().NotBeNull();
105+
movingPercentiles.Items.Should().NotBeNull();
106+
}
107+
}
108+
}
109+
}

0 commit comments

Comments
 (0)