diff --git a/docs/reference/aggregations/pipeline.asciidoc b/docs/reference/aggregations/pipeline.asciidoc index 37c1c357007b0..2d3f07a33c11f 100644 --- a/docs/reference/aggregations/pipeline.asciidoc +++ b/docs/reference/aggregations/pipeline.asciidoc @@ -46,7 +46,7 @@ For example, the path `"my_bucket>my_stats.avg"` will path to the `avg` value in contained in the `"my_bucket"` bucket aggregation. Paths are relative from the position of the pipeline aggregation; they are not absolute paths, and the path cannot go back "up" the -aggregation tree. For example, this moving average is embedded inside a date_histogram and refers to a "sibling" +aggregation tree. For example, this derivative is embedded inside a date_histogram and refers to a "sibling" metric `"the_sum"`: [source,js] @@ -63,8 +63,8 @@ POST /_search "the_sum":{ "sum":{ "field": "lemmings" } <1> }, - "the_movavg":{ - "moving_avg":{ "buckets_path": "the_sum" } <2> + "the_deriv":{ + "derivative":{ "buckets_path": "the_sum" } <2> } } } @@ -72,7 +72,6 @@ POST /_search } -------------------------------------------------- // CONSOLE -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] <1> The metric is called `"the_sum"` <2> The `buckets_path` refers to the metric via a relative path `"the_sum"` @@ -115,7 +114,8 @@ POST /_search === Special Paths Instead of pathing to a metric, `buckets_path` can use a special `"_count"` path. This instructs -the pipeline aggregation to use the document count as its input. For example, a moving average can be calculated on the document count of each bucket, instead of a specific metric: +the pipeline aggregation to use the document count as its input. For example, a derivative can be calculated +on the document count of each bucket, instead of a specific metric: [source,js] -------------------------------------------------- @@ -128,8 +128,8 @@ POST /_search "interval":"day" }, "aggs": { - "the_movavg": { - "moving_avg": { "buckets_path": "_count" } <1> + "the_deriv": { + "derivative": { "buckets_path": "_count" } <1> } } } @@ -137,8 +137,7 @@ POST /_search } -------------------------------------------------- // CONSOLE -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] -<1> By using `_count` instead of a metric name, we can calculate the moving average of document counts in the histogram +<1> By using `_count` instead of a metric name, we can calculate the derivative of document counts in the histogram The `buckets_path` can also use `"_bucket_count"` and path to a multi-bucket aggregation to use the number of buckets returned by that aggregation in the pipeline aggregation instead of a metric. for example a `bucket_selector` can be diff --git a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc index 06641391ced32..48bcbbf17d51f 100644 --- a/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc +++ b/docs/reference/aggregations/pipeline/movavg-aggregation.asciidoc @@ -1,661 +1,5 @@ [[search-aggregations-pipeline-movavg-aggregation]] === Moving Average Aggregation -deprecated[6.4.0, The Moving Average aggregation has been deprecated in favor of the more general -<>. The new Moving Function aggregation provides -all the same functionality as the Moving Average aggregation, but also provides more flexibility.] - -Given an ordered series of data, the Moving Average aggregation will slide a window across the data and emit the average -value of that window. For example, given the data `[1, 2, 3, 4, 5, 6, 7, 8, 9, 10]`, we can calculate a simple moving -average with windows size of `5` as follows: - -- (1 + 2 + 3 + 4 + 5) / 5 = 3 -- (2 + 3 + 4 + 5 + 6) / 5 = 4 -- (3 + 4 + 5 + 6 + 7) / 5 = 5 -- etc - -Moving averages are a simple method to smooth sequential data. Moving averages are typically applied to time-based data, -such as stock prices or server metrics. The smoothing can be used to eliminate high frequency fluctuations or random noise, -which allows the lower frequency trends to be more easily visualized, such as seasonality. - -==== Syntax - -A `moving_avg` aggregation looks like this in isolation: - -[source,js] --------------------------------------------------- -{ - "moving_avg": { - "buckets_path": "the_sum", - "model": "holt", - "window": 5, - "gap_policy": "insert_zeros", - "settings": { - "alpha": 0.8 - } - } -} --------------------------------------------------- -// NOTCONSOLE - -.`moving_avg` Parameters -|=== -|Parameter Name |Description |Required |Default Value -|`buckets_path` |Path to the metric of interest (see <> for more details |Required | -|`model` |The moving average weighting model that we wish to use |Optional |`simple` -|`gap_policy` |Determines what should happen when a gap in the data is encountered. |Optional |`insert_zeros` -|`window` |The size of window to "slide" across the histogram. |Optional |`5` -|`minimize` |If the model should be algorithmically minimized. See <> for more - details |Optional |`false` for most models -|`settings` |Model-specific settings, contents which differ depending on the model specified. |Optional | -|=== - -`moving_avg` aggregations must be embedded inside of a `histogram` or `date_histogram` aggregation. They can be -embedded like any other metric aggregation: - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ <1> - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } <2> - }, - "the_movavg":{ - "moving_avg":{ "buckets_path": "the_sum" } <3> - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -<1> A `date_histogram` named "my_date_histo" is constructed on the "timestamp" field, with one-day intervals -<2> A `sum` metric is used to calculate the sum of a field. This could be any metric (sum, min, max, etc) -<3> Finally, we specify a `moving_avg` aggregation which uses "the_sum" metric as its input. - -Moving averages are built by first specifying a `histogram` or `date_histogram` over a field. You can then optionally -add normal metrics, such as a `sum`, inside of that histogram. Finally, the `moving_avg` is embedded inside the histogram. -The `buckets_path` parameter is then used to "point" at one of the sibling metrics inside of the histogram (see -<> for a description of the syntax for `buckets_path`. - -An example response from the above aggregation may look like: - -[source,js] --------------------------------------------------- -{ - "took": 11, - "timed_out": false, - "_shards": ..., - "hits": ..., - "aggregations": { - "my_date_histo": { - "buckets": [ - { - "key_as_string": "2015/01/01 00:00:00", - "key": 1420070400000, - "doc_count": 3, - "the_sum": { - "value": 550.0 - } - }, - { - "key_as_string": "2015/02/01 00:00:00", - "key": 1422748800000, - "doc_count": 2, - "the_sum": { - "value": 60.0 - }, - "the_movavg": { - "value": 550.0 - } - }, - { - "key_as_string": "2015/03/01 00:00:00", - "key": 1425168000000, - "doc_count": 2, - "the_sum": { - "value": 375.0 - }, - "the_movavg": { - "value": 305.0 - } - } - ] - } - } -} --------------------------------------------------- -// TESTRESPONSE[s/"took": 11/"took": $body.took/] -// TESTRESPONSE[s/"_shards": \.\.\./"_shards": $body._shards/] -// TESTRESPONSE[s/"hits": \.\.\./"hits": $body.hits/] - - -==== Models - -The `moving_avg` aggregation includes four different moving average "models". The main difference is how the values in the -window are weighted. As data-points become "older" in the window, they may be weighted differently. This will -affect the final average for that window. - -Models are specified using the `model` parameter. Some models may have optional configurations which are specified inside -the `settings` parameter. - -===== Simple - -The `simple` model calculates the sum of all values in the window, then divides by the size of the window. It is effectively -a simple arithmetic mean of the window. The simple model does not perform any time-dependent weighting, which means -the values from a `simple` moving average tend to "lag" behind the real data. - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg":{ - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "simple" - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -A `simple` model has no special settings to configure - -The window size can change the behavior of the moving average. For example, a small window (`"window": 10`) will closely -track the data and only smooth out small scale fluctuations: - -[[movavg_10window]] -.Moving average with window of size 10 -image::images/pipeline_movavg/movavg_10window.png[] - -In contrast, a `simple` moving average with larger window (`"window": 100`) will smooth out all higher-frequency fluctuations, -leaving only low-frequency, long term trends. It also tends to "lag" behind the actual data by a substantial amount: - -[[movavg_100window]] -.Moving average with window of size 100 -image::images/pipeline_movavg/movavg_100window.png[] - - -==== Linear - -The `linear` model assigns a linear weighting to points in the series, such that "older" datapoints (e.g. those at -the beginning of the window) contribute a linearly less amount to the total average. The linear weighting helps reduce -the "lag" behind the data's mean, since older points have less influence. - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "linear" - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -A `linear` model has no special settings to configure - -Like the `simple` model, window size can change the behavior of the moving average. For example, a small window (`"window": 10`) -will closely track the data and only smooth out small scale fluctuations: - -[[linear_10window]] -.Linear moving average with window of size 10 -image::images/pipeline_movavg/linear_10window.png[] - -In contrast, a `linear` moving average with larger window (`"window": 100`) will smooth out all higher-frequency fluctuations, -leaving only low-frequency, long term trends. It also tends to "lag" behind the actual data by a substantial amount, -although typically less than the `simple` model: - -[[linear_100window]] -.Linear moving average with window of size 100 -image::images/pipeline_movavg/linear_100window.png[] - -==== EWMA (Exponentially Weighted) - -The `ewma` model (aka "single-exponential") is similar to the `linear` model, except older data-points become exponentially less important, -rather than linearly less important. The speed at which the importance decays can be controlled with an `alpha` -setting. Small values make the weight decay slowly, which provides greater smoothing and takes into account a larger -portion of the window. Larger values make the weight decay quickly, which reduces the impact of older values on the -moving average. This tends to make the moving average track the data more closely but with less smoothing. - -The default value of `alpha` is `0.3`, and the setting accepts any float from 0-1 inclusive. - -The EWMA model can be <> - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "ewma", - "settings" : { - "alpha" : 0.5 - } - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -[[single_0.2alpha]] -.EWMA with window of size 10, alpha = 0.2 -image::images/pipeline_movavg/single_0.2alpha.png[] - -[[single_0.7alpha]] -.EWMA with window of size 10, alpha = 0.7 -image::images/pipeline_movavg/single_0.7alpha.png[] - -==== Holt-Linear - -The `holt` model (aka "double exponential") incorporates a second exponential term which -tracks the data's trend. Single exponential does not perform well when the data has an underlying linear trend. The -double exponential model calculates two values internally: a "level" and a "trend". - -The level calculation is similar to `ewma`, and is an exponentially weighted view of the data. The difference is -that the previously smoothed value is used instead of the raw value, which allows it to stay close to the original series. -The trend calculation looks at the difference between the current and last value (e.g. the slope, or trend, of the -smoothed data). The trend value is also exponentially weighted. - -Values are produced by multiplying the level and trend components. - -The default value of `alpha` is `0.3` and `beta` is `0.1`. The settings accept any float from 0-1 inclusive. - -The Holt-Linear model can be <> - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "holt", - "settings" : { - "alpha" : 0.5, - "beta" : 0.5 - } - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -In practice, the `alpha` value behaves very similarly in `holt` as `ewma`: small values produce more smoothing -and more lag, while larger values produce closer tracking and less lag. The value of `beta` is often difficult -to see. Small values emphasize long-term trends (such as a constant linear trend in the whole series), while larger -values emphasize short-term trends. This will become more apparently when you are predicting values. - -[[double_0.2beta]] -.Holt-Linear moving average with window of size 100, alpha = 0.5, beta = 0.2 -image::images/pipeline_movavg/double_0.2beta.png[] - -[[double_0.7beta]] -.Holt-Linear moving average with window of size 100, alpha = 0.5, beta = 0.7 -image::images/pipeline_movavg/double_0.7beta.png[] - -==== Holt-Winters - -The `holt_winters` model (aka "triple exponential") incorporates a third exponential term which -tracks the seasonal aspect of your data. This aggregation therefore smooths based on three components: "level", "trend" -and "seasonality". - -The level and trend calculation is identical to `holt` The seasonal calculation looks at the difference between -the current point, and the point one period earlier. - -Holt-Winters requires a little more handholding than the other moving averages. You need to specify the "periodicity" -of your data: e.g. if your data has cyclic trends every 7 days, you would set `period: 7`. Similarly if there was -a monthly trend, you would set it to `30`. There is currently no periodicity detection, although that is planned -for future enhancements. - -There are two varieties of Holt-Winters: additive and multiplicative. - -===== "Cold Start" - -Unfortunately, due to the nature of Holt-Winters, it requires two periods of data to "bootstrap" the algorithm. This -means that your `window` must always be *at least* twice the size of your period. An exception will be thrown if it -isn't. It also means that Holt-Winters will not emit a value for the first `2 * period` buckets; the current algorithm -does not backcast. - -[[holt_winters_cold_start]] -.Holt-Winters showing a "cold" start where no values are emitted -image::images/pipeline_movavg/triple_untruncated.png[] - -Because the "cold start" obscures what the moving average looks like, the rest of the Holt-Winters images are truncated -to not show the "cold start". Just be aware this will always be present at the beginning of your moving averages! - -===== Additive Holt-Winters - -Additive seasonality is the default; it can also be specified by setting `"type": "add"`. This variety is preferred -when the seasonal affect is additive to your data. E.g. you could simply subtract the seasonal effect to "de-seasonalize" -your data into a flat trend. - -The default values of `alpha` and `gamma` are `0.3` while `beta` is `0.1`. The settings accept any float from 0-1 inclusive. -The default value of `period` is `1`. - -The additive Holt-Winters model can be <> - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "holt_winters", - "settings" : { - "type" : "add", - "alpha" : 0.5, - "beta" : 0.5, - "gamma" : 0.5, - "period" : 7 - } - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -[[holt_winters_add]] -.Holt-Winters moving average with window of size 120, alpha = 0.5, beta = 0.7, gamma = 0.3, period = 30 -image::images/pipeline_movavg/triple.png[] - -===== Multiplicative Holt-Winters - -Multiplicative is specified by setting `"type": "mult"`. This variety is preferred when the seasonal affect is -multiplied against your data. E.g. if the seasonal affect is x5 the data, rather than simply adding to it. - -The default values of `alpha` and `gamma` are `0.3` while `beta` is `0.1`. The settings accept any float from 0-1 inclusive. -The default value of `period` is `1`. - -The multiplicative Holt-Winters model can be <> - -[WARNING] -====== -Multiplicative Holt-Winters works by dividing each data point by the seasonal value. This is problematic if any of -your data is zero, or if there are gaps in the data (since this results in a divid-by-zero). To combat this, the -`mult` Holt-Winters pads all values by a very small amount (1*10^-10^) so that all values are non-zero. This affects -the result, but only minimally. If your data is non-zero, or you prefer to see `NaN` when zero's are encountered, -you can disable this behavior with `pad: false` -====== - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "holt_winters", - "settings" : { - "type" : "mult", - "alpha" : 0.5, - "beta" : 0.5, - "gamma" : 0.5, - "period" : 7, - "pad" : true - } - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -==== Prediction - -experimental[] - -All the moving average model support a "prediction" mode, which will attempt to extrapolate into the future given the -current smoothed, moving average. Depending on the model and parameter, these predictions may or may not be accurate. - -Predictions are enabled by adding a `predict` parameter to any moving average aggregation, specifying the number of -predictions you would like appended to the end of the series. These predictions will be spaced out at the same interval -as your buckets: - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "window" : 30, - "model" : "simple", - "predict" : 10 - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -The `simple`, `linear` and `ewma` models all produce "flat" predictions: they essentially converge on the mean -of the last value in the series, producing a flat: - -[[simple_prediction]] -.Simple moving average with window of size 10, predict = 50 -image::images/pipeline_movavg/simple_prediction.png[] - -In contrast, the `holt` model can extrapolate based on local or global constant trends. If we set a high `beta` -value, we can extrapolate based on local constant trends (in this case the predictions head down, because the data at the end -of the series was heading in a downward direction): - -[[double_prediction_local]] -.Holt-Linear moving average with window of size 100, predict = 20, alpha = 0.5, beta = 0.8 -image::images/pipeline_movavg/double_prediction_local.png[] - -In contrast, if we choose a small `beta`, the predictions are based on the global constant trend. In this series, the -global trend is slightly positive, so the prediction makes a sharp u-turn and begins a positive slope: - -[[double_prediction_global]] -.Double Exponential moving average with window of size 100, predict = 20, alpha = 0.5, beta = 0.1 -image::images/pipeline_movavg/double_prediction_global.png[] - -The `holt_winters` model has the potential to deliver the best predictions, since it also incorporates seasonal -fluctuations into the model: - -[[holt_winters_prediction_global]] -.Holt-Winters moving average with window of size 120, predict = 25, alpha = 0.8, beta = 0.2, gamma = 0.7, period = 30 -image::images/pipeline_movavg/triple_prediction.png[] - -[[movavg-minimizer]] -==== Minimization - -Some of the models (EWMA, Holt-Linear, Holt-Winters) require one or more parameters to be configured. Parameter choice -can be tricky and sometimes non-intuitive. Furthermore, small deviations in these parameters can sometimes have a drastic -effect on the output moving average. - -For that reason, the three "tunable" models can be algorithmically *minimized*. Minimization is a process where parameters -are tweaked until the predictions generated by the model closely match the output data. Minimization is not fullproof -and can be susceptible to overfitting, but it often gives better results than hand-tuning. - -Minimization is disabled by default for `ewma` and `holt_linear`, while it is enabled by default for `holt_winters`. -Minimization is most useful with Holt-Winters, since it helps improve the accuracy of the predictions. EWMA and -Holt-Linear are not great predictors, and mostly used for smoothing data, so minimization is less useful on those -models. - -Minimization is enabled/disabled via the `minimize` parameter: - -[source,js] --------------------------------------------------- -POST /_search -{ - "size": 0, - "aggs": { - "my_date_histo":{ - "date_histogram":{ - "field":"date", - "interval":"1M" - }, - "aggs":{ - "the_sum":{ - "sum":{ "field": "price" } - }, - "the_movavg": { - "moving_avg":{ - "buckets_path": "the_sum", - "model" : "holt_winters", - "window" : 30, - "minimize" : true, <1> - "settings" : { - "period" : 7 - } - } - } - } - } - } -} --------------------------------------------------- -// CONSOLE -// TEST[setup:sales] -// TEST[warning:The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.] - -<1> Minimization is enabled with the `minimize` parameter - -When enabled, minimization will find the optimal values for `alpha`, `beta` and `gamma`. The user should still provide -appropriate values for `window`, `period` and `type`. - -[WARNING] -====== -Minimization works by running a stochastic process called *simulated annealing*. This process will usually generate -a good solution, but is not guaranteed to find the global optimum. It also requires some amount of additional -computational power, since the model needs to be re-run multiple times as the values are tweaked. The run-time of -minimization is linear to the size of the window being processed: excessively large windows may cause latency. - -Finally, minimization fits the model to the last `n` values, where `n = window`. This generally produces -better forecasts into the future, since the parameters are tuned around the end of the series. It can, however, generate -poorer fitting moving averages at the beginning of the series. -====== +The Moving Average aggregation has been removed. Use the more general +<> instead. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml index 7897d1feb5aa6..ab17ddb66587e 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/80_typed_keys.yml @@ -210,8 +210,6 @@ setup: version: " - 6.3.99" reason: "deprecation added in 6.4.0" - do: - warnings: - - 'The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.' search: rest_total_hits_as_int: true typed_keys: true @@ -226,8 +224,8 @@ setup: test_sum: sum: field: num - test_moving_avg: - moving_avg: + test_deriv: + derivative: buckets_path: "test_sum" test_max_bucket: max_bucket: @@ -236,5 +234,5 @@ setup: - is_true: aggregations.date_histogram#test_created_histogram - is_true: aggregations.date_histogram#test_created_histogram.buckets.0.sum#test_sum - is_true: aggregations.date_histogram#test_created_histogram.buckets.1.sum#test_sum - - is_true: aggregations.date_histogram#test_created_histogram.buckets.1.simple_value#test_moving_avg + - is_true: aggregations.date_histogram#test_created_histogram.buckets.1.derivative#test_deriv - is_true: aggregations.bucket_metric_value#test_max_bucket diff --git a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java index d07467e5d1f8f..1b1e9961cf50f 100644 --- a/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java +++ b/server/src/main/java/org/elasticsearch/plugins/SearchPlugin.java @@ -41,8 +41,6 @@ import org.elasticsearch.search.aggregations.bucket.significant.SignificantTerms; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristic; import org.elasticsearch.search.aggregations.bucket.significant.heuristics.SignificanceHeuristicParser; -import org.elasticsearch.search.aggregations.pipeline.MovAvgModel; -import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.fetch.FetchSubPhase; import org.elasticsearch.search.fetch.subphase.highlight.Highlighter; @@ -77,13 +75,6 @@ default List> getScoreFunctions() { default List> getSignificanceHeuristics() { return emptyList(); } - /** - * The new {@link MovAvgModel}s defined by this plugin. {@linkplain MovAvgModel}s are used by the {@link MovAvgPipelineAggregator} to - * model trends in data. - */ - default List> getMovingAverageModels() { - return emptyList(); - } /** * The new {@link FetchSubPhase}s defined by this plugin. */ @@ -400,7 +391,7 @@ public RescorerSpec(String name, Writeable.Reader reader, CheckedFu } /** - * Specification of search time behavior extension like a custom {@link MovAvgModel} or {@link ScoreFunction}. + * Specification of search time behavior extension like a custom {@link ScoreFunction}. * * @param the type of the main {@link NamedWriteable} for this spec. All specs have this but it isn't always *for* the same thing * though, usually it is some sort of builder sent from the coordinating node to the data nodes executing the behavior diff --git a/server/src/main/java/org/elasticsearch/search/SearchModule.java b/server/src/main/java/org/elasticsearch/search/SearchModule.java index bd1bbb98281cc..de4f548f6cf08 100644 --- a/server/src/main/java/org/elasticsearch/search/SearchModule.java +++ b/server/src/main/java/org/elasticsearch/search/SearchModule.java @@ -110,8 +110,8 @@ import org.elasticsearch.search.aggregations.bucket.filter.InternalFilter; import org.elasticsearch.search.aggregations.bucket.filter.InternalFilters; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoHashGrid; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.elasticsearch.search.aggregations.bucket.global.GlobalAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.global.InternalGlobal; @@ -203,26 +203,19 @@ import org.elasticsearch.search.aggregations.pipeline.CumulativeSumPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregator; -import org.elasticsearch.search.aggregations.pipeline.EwmaModel; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketParser; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregator; -import org.elasticsearch.search.aggregations.pipeline.HoltLinearModel; -import org.elasticsearch.search.aggregations.pipeline.HoltWintersModel; import org.elasticsearch.search.aggregations.pipeline.InternalBucketMetricValue; import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; import org.elasticsearch.search.aggregations.pipeline.InternalExtendedStatsBucket; import org.elasticsearch.search.aggregations.pipeline.InternalPercentilesBucket; import org.elasticsearch.search.aggregations.pipeline.InternalSimpleValue; import org.elasticsearch.search.aggregations.pipeline.InternalStatsBucket; -import org.elasticsearch.search.aggregations.pipeline.LinearModel; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregator; -import org.elasticsearch.search.aggregations.pipeline.MovAvgModel; -import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.MovFnPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MovFnPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder; @@ -230,7 +223,6 @@ import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.SerialDiffPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.SerialDiffPipelineAggregator; -import org.elasticsearch.search.aggregations.pipeline.SimpleModel; import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.SumBucketPipelineAggregationBuilder; @@ -291,8 +283,6 @@ public class SearchModule { private final Map highlighters; private final ParseFieldRegistry significanceHeuristicParserRegistry = new ParseFieldRegistry<>( "significance_heuristic"); - private final ParseFieldRegistry movingAverageModelParserRegistry = new ParseFieldRegistry<>( - "moving_avg_model"); private final List fetchSubPhases = new ArrayList<>(); @@ -321,7 +311,6 @@ public SearchModule(Settings settings, boolean transportClient, List getSignificanceHeuristicP return significanceHeuristicParserRegistry; } - /** - * The registry of {@link MovAvgModel}s. - */ - public ParseFieldRegistry getMovingAverageModelParserRegistry() { - return movingAverageModelParserRegistry; - } - private void registerAggregations(List plugins) { registerAggregation(new AggregationSpec(AvgAggregationBuilder.NAME, AvgAggregationBuilder::new, AvgAggregationBuilder::parse) .addResultReader(InternalAvg::new)); @@ -521,12 +503,6 @@ private void registerPipelineAggregations(List plugins) { PercentilesBucketPipelineAggregator::new, PercentilesBucketPipelineAggregationBuilder.PARSER) .addResultReader(InternalPercentilesBucket::new)); - registerPipelineAggregation(new PipelineAggregationSpec( - MovAvgPipelineAggregationBuilder.NAME, - MovAvgPipelineAggregationBuilder::new, - MovAvgPipelineAggregator::new, - (n, c) -> MovAvgPipelineAggregationBuilder.parse(movingAverageModelParserRegistry, n, c)) - /* Uses InternalHistogram for buckets */); registerPipelineAggregation(new PipelineAggregationSpec( CumulativeSumPipelineAggregationBuilder.NAME, CumulativeSumPipelineAggregationBuilder::new, @@ -721,22 +697,6 @@ private void registerSignificanceHeuristic(SearchExtensionSpec plugins) { - registerMovingAverageModel(new SearchExtensionSpec<>(SimpleModel.NAME, SimpleModel::new, SimpleModel.PARSER)); - registerMovingAverageModel(new SearchExtensionSpec<>(LinearModel.NAME, LinearModel::new, LinearModel.PARSER)); - registerMovingAverageModel(new SearchExtensionSpec<>(EwmaModel.NAME, EwmaModel::new, EwmaModel.PARSER)); - registerMovingAverageModel(new SearchExtensionSpec<>(HoltLinearModel.NAME, HoltLinearModel::new, HoltLinearModel.PARSER)); - registerMovingAverageModel(new SearchExtensionSpec<>(HoltWintersModel.NAME, HoltWintersModel::new, HoltWintersModel.PARSER)); - - registerFromPlugin(plugins, SearchPlugin::getMovingAverageModels, this::registerMovingAverageModel); - } - - private void registerMovingAverageModel(SearchExtensionSpec movAvgModel) { - movingAverageModelParserRegistry.register(movAvgModel.getParser(), movAvgModel.getName()); - namedWriteables.add( - new NamedWriteableRegistry.Entry(MovAvgModel.class, movAvgModel.getName().getPreferredName(), movAvgModel.getReader())); - } - private void registerFetchSubPhases(List plugins) { registerFetchSubPhase(new ExplainFetchSubPhase()); registerFetchSubPhase(new DocValueFieldsFetchSubPhase()); diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java index a5f89f82f6b35..eb786bbbe01f4 100644 --- a/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java +++ b/server/src/main/java/org/elasticsearch/search/aggregations/PipelineAggregatorBuilders.java @@ -29,7 +29,6 @@ import org.elasticsearch.search.aggregations.pipeline.ExtendedStatsBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.MovAvgPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.MovFnPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.PercentilesBucketPipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.SerialDiffPipelineAggregationBuilder; @@ -77,14 +76,6 @@ public static PercentilesBucketPipelineAggregationBuilder percentilesBucket(Stri return new PercentilesBucketPipelineAggregationBuilder(name, bucketsPath); } - /** - * @deprecated use {@link #movingFunction(String, Script, String, int)} instead - */ - @Deprecated - public static MovAvgPipelineAggregationBuilder movingAvg(String name, String bucketsPath) { - return new MovAvgPipelineAggregationBuilder(name, bucketsPath); - } - public static BucketScriptPipelineAggregationBuilder bucketScript(String name, Map bucketsPathsMap, Script script) { return new BucketScriptPipelineAggregationBuilder(name, bucketsPathsMap, script); } diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/EwmaModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/EwmaModel.java deleted file mode 100644 index ad2532c3b5049..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/EwmaModel.java +++ /dev/null @@ -1,175 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; -import java.util.Objects; - -/** - * Calculate a exponentially weighted moving average - */ -public class EwmaModel extends MovAvgModel { - public static final String NAME = "ewma"; - - private static final double DEFAULT_ALPHA = 0.3; - - /** - * Controls smoothing of data. Also known as "level" value. - * Alpha = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double alpha; - - public EwmaModel() { - this(DEFAULT_ALPHA); - } - - public EwmaModel(double alpha) { - this.alpha = alpha; - } - - /** - * Read from a stream. - */ - public EwmaModel(StreamInput in) throws IOException { - alpha = in.readDouble(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(alpha); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public boolean canBeMinimized() { - return true; - } - - @Override - public MovAvgModel neighboringModel() { - double alpha = Math.random(); - return new EwmaModel(alpha); - } - - @Override - public MovAvgModel clone() { - return new EwmaModel(this.alpha); - } - - @Override - protected double[] doPredict(Collection values, int numPredictions) { - double[] predictions = new double[numPredictions]; - - // EWMA just emits the same final prediction repeatedly. - Arrays.fill(predictions, next(values)); - - return predictions; - } - - @Override - public double next(Collection values) { - return MovingFunctions.ewma(values.stream().mapToDouble(Double::doubleValue).toArray(), alpha); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - builder.endObject(); - return builder; - } - - public static final AbstractModelParser PARSER = new AbstractModelParser() { - @Override - public MovAvgModel parse(@Nullable Map settings, String pipelineName, int windowSize) throws ParseException { - double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA); - checkUnrecognizedParams(settings); - return new EwmaModel(alpha); - } - }; - - @Override - public int hashCode() { - return Objects.hash(alpha); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - EwmaModel other = (EwmaModel) obj; - return Objects.equals(alpha, other.alpha); - } - - public static class EWMAModelBuilder implements MovAvgModelBuilder { - - private double alpha = DEFAULT_ALPHA; - - /** - * Alpha controls the smoothing of the data. Alpha = 1 retains no memory of past values - * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g. - * the series mean). Useful values are somewhere in between. Defaults to 0.5. - * - * @param alpha A double between 0-1 inclusive, controls data smoothing - * - * @return The builder to continue chaining - */ - public EWMAModelBuilder alpha(double alpha) { - this.alpha = alpha; - return this; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - - builder.endObject(); - return builder; - } - - @Override - public MovAvgModel build() { - return new EwmaModel(alpha); - } - } -} - diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltLinearModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltLinearModel.java deleted file mode 100644 index ec40d2b18b56a..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltLinearModel.java +++ /dev/null @@ -1,233 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Collection; -import java.util.Map; -import java.util.Objects; - -/** - * Calculate a doubly exponential weighted moving average - */ -public class HoltLinearModel extends MovAvgModel { - public static final String NAME = "holt"; - - private static final double DEFAULT_ALPHA = 0.3; - private static final double DEFAULT_BETA = 0.1; - - /** - * Controls smoothing of data. Also known as "level" value. - * Alpha = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double alpha; - - /** - * Controls smoothing of trend. - * Beta = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double beta; - - public HoltLinearModel() { - this(DEFAULT_ALPHA, DEFAULT_BETA); - } - - public HoltLinearModel(double alpha, double beta) { - this.alpha = alpha; - this.beta = beta; - } - - /** - * Read from a stream. - */ - public HoltLinearModel(StreamInput in) throws IOException { - alpha = in.readDouble(); - beta = in.readDouble(); - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(alpha); - out.writeDouble(beta); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public boolean canBeMinimized() { - return true; - } - - @Override - public MovAvgModel neighboringModel() { - double newValue = Math.random(); - switch ((int) (Math.random() * 2)) { - case 0: - return new HoltLinearModel(newValue, this.beta); - case 1: - return new HoltLinearModel(this.alpha, newValue); - default: - assert (false): "Random value fell outside of range [0-1]"; - return new HoltLinearModel(newValue, this.beta); // This should never technically happen... - } - } - - @Override - public MovAvgModel clone() { - return new HoltLinearModel(this.alpha, this.beta); - } - - /** - * Predicts the next `n` values in the series, using the smoothing model to generate new values. - * Unlike the other moving averages, Holt-Linear has forecasting/prediction built into the algorithm. - * Prediction is more than simply adding the next prediction to the window and repeating. Holt-Linear - * will extrapolate into the future by applying the trend information to the smoothed data. - * - * @param values Collection of numerics to movingAvg, usually windowed - * @param numPredictions Number of newly generated predictions to return - * @return Returns an array of doubles, since most smoothing methods operate on floating points - */ - @Override - protected double[] doPredict(Collection values, int numPredictions) { - return next(values, numPredictions); - } - - @Override - public double next(Collection values) { - return next(values, 1)[0]; - } - - /** - * Calculate a Holt-Linear (doubly exponential weighted) moving average - * - * @param values Collection of values to calculate avg for - * @param numForecasts number of forecasts into the future to return - * - * @return Returns a Double containing the moving avg for the window - */ - public double[] next(Collection values, int numForecasts) { - if (values.size() == 0) { - return emptyPredictions(numForecasts); - } - return MovingFunctions.holtForecast(values.stream().mapToDouble(Double::doubleValue).toArray(), alpha, beta, numForecasts); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - builder.field("beta", beta); - builder.endObject(); - return builder; - } - - public static final AbstractModelParser PARSER = new AbstractModelParser() { - @Override - public MovAvgModel parse(@Nullable Map settings, String pipelineName, int windowSize) throws ParseException { - - double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA); - double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA); - checkUnrecognizedParams(settings); - return new HoltLinearModel(alpha, beta); - } - }; - - - @Override - public int hashCode() { - return Objects.hash(alpha, beta); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - HoltLinearModel other = (HoltLinearModel) obj; - return Objects.equals(alpha, other.alpha) - && Objects.equals(beta, other.beta); - } - - - public static class HoltLinearModelBuilder implements MovAvgModelBuilder { - private double alpha = DEFAULT_ALPHA; - private double beta = DEFAULT_BETA; - - /** - * Alpha controls the smoothing of the data. Alpha = 1 retains no memory of past values - * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g. - * the series mean). Useful values are somewhere in between. Defaults to 0.5. - * - * @param alpha A double between 0-1 inclusive, controls data smoothing - * - * @return The builder to continue chaining - */ - public HoltLinearModelBuilder alpha(double alpha) { - this.alpha = alpha; - return this; - } - - /** - * Equivalent to alpha, but controls the smoothing of the trend instead of the data - * - * @param beta a double between 0-1 inclusive, controls trend smoothing - * - * @return The builder to continue chaining - */ - public HoltLinearModelBuilder beta(double beta) { - this.beta = beta; - return this; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - builder.field("beta", beta); - - builder.endObject(); - return builder; - } - - @Override - public MovAvgModel build() { - return new HoltLinearModel(alpha, beta); - } - } -} - diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltWintersModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltWintersModel.java deleted file mode 100644 index df42689a2116e..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/HoltWintersModel.java +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - - -import org.elasticsearch.ElasticsearchParseException; -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.LoggingDeprecationHandler; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -/** - * Calculate a triple exponential weighted moving average - */ -public class HoltWintersModel extends MovAvgModel { - public static final String NAME = "holt_winters"; - - private static final double DEFAULT_ALPHA = 0.3; - private static final double DEFAULT_BETA = 0.1; - private static final double DEFAULT_GAMMA = 0.3; - private static final int DEFAULT_PERIOD = 1; - private static final SeasonalityType DEFAULT_SEASONALITY_TYPE = SeasonalityType.ADDITIVE; - private static final boolean DEFAULT_PAD = false; - - public enum SeasonalityType { - ADDITIVE((byte) 0, "add"), MULTIPLICATIVE((byte) 1, "mult"); - - /** - * Parse a string SeasonalityType into the byte enum - * - * @param text SeasonalityType in string format (e.g. "add") - * @return SeasonalityType enum - */ - @Nullable - public static SeasonalityType parse(String text) { - if (text == null) { - return null; - } - SeasonalityType result = null; - for (SeasonalityType policy : values()) { - if (policy.parseField.match(text, LoggingDeprecationHandler.INSTANCE)) { - result = policy; - break; - } - } - if (result == null) { - final List validNames = new ArrayList<>(); - for (SeasonalityType policy : values()) { - validNames.add(policy.getName()); - } - throw new ElasticsearchParseException("failed to parse seasonality type [{}]. accepted values are [{}]", text, validNames); - } - return result; - } - - private final byte id; - private final ParseField parseField; - - SeasonalityType(byte id, String name) { - this.id = id; - this.parseField = new ParseField(name); - } - - /** - * Serialize the SeasonalityType to the output stream - */ - public void writeTo(StreamOutput out) throws IOException { - out.writeByte(id); - } - - /** - * Deserialize the SeasonalityType from the input stream - * - * @param in the input stream - * @return SeasonalityType Enum - */ - public static SeasonalityType readFrom(StreamInput in) throws IOException { - byte id = in.readByte(); - for (SeasonalityType seasonalityType : values()) { - if (id == seasonalityType.id) { - return seasonalityType; - } - } - throw new IllegalStateException("Unknown Seasonality Type with id [" + id + "]"); - } - - /** - * Return the english-formatted name of the SeasonalityType - * - * @return English representation of SeasonalityType - */ - public String getName() { - return parseField.getPreferredName(); - } - } - - /** - * Controls smoothing of data. Also known as "level" value. - * Alpha = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double alpha; - - /** - * Controls smoothing of trend. - * Beta = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double beta; - - /** - * Controls smoothing of seasonality. - * Gamma = 1 retains no memory of past values - * (e.g. random walk), while alpha = 0 retains infinite memory of past values (e.g. - * mean of the series). - */ - private final double gamma; - - /** - * Periodicity of the data - */ - private final int period; - - /** - * Whether this is a multiplicative or additive HW - */ - private final SeasonalityType seasonalityType; - - /** - * Padding is used to add a very small amount to values, so that zeroes do not interfere - * with multiplicative seasonality math (e.g. division by zero) - */ - private final boolean pad; - private final double padding; - - public HoltWintersModel() { - this(DEFAULT_ALPHA, DEFAULT_BETA, DEFAULT_GAMMA, DEFAULT_PERIOD, DEFAULT_SEASONALITY_TYPE, DEFAULT_PAD); - } - - public HoltWintersModel(double alpha, double beta, double gamma, int period, SeasonalityType seasonalityType, boolean pad) { - this.alpha = alpha; - this.beta = beta; - this.gamma = gamma; - this.period = period; - this.seasonalityType = seasonalityType; - this.pad = pad; - this.padding = inferPadding(); - } - - /** - * Read from a stream. - */ - public HoltWintersModel(StreamInput in) throws IOException { - alpha = in.readDouble(); - beta = in.readDouble(); - gamma = in.readDouble(); - period = in.readVInt(); - seasonalityType = SeasonalityType.readFrom(in); - pad = in.readBoolean(); - this.padding = inferPadding(); - } - - /** - * Only pad if we are multiplicative and padding is enabled. the padding amount is not currently user-configurable. - */ - private double inferPadding() { - return seasonalityType.equals(SeasonalityType.MULTIPLICATIVE) && pad ? 0.0000000001 : 0; - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - out.writeDouble(alpha); - out.writeDouble(beta); - out.writeDouble(gamma); - out.writeVInt(period); - seasonalityType.writeTo(out); - out.writeBoolean(pad); - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public boolean minimizeByDefault() { - return true; - } - - @Override - public boolean canBeMinimized() { - return true; - } - - @Override - public MovAvgModel neighboringModel() { - double newValue = Math.random(); - switch ((int) (Math.random() * 3)) { - case 0: - return new HoltWintersModel(newValue, beta, gamma, period, seasonalityType, pad); - case 1: - return new HoltWintersModel(alpha, newValue, gamma, period, seasonalityType, pad); - case 2: - return new HoltWintersModel(alpha, beta, newValue, period, seasonalityType, pad); - default: - assert (false): "Random value fell outside of range [0-2]"; - return new HoltWintersModel(newValue, beta, gamma, period, seasonalityType, pad); // This should never technically happen... - } - } - - @Override - public MovAvgModel clone() { - return new HoltWintersModel(alpha, beta, gamma, period, seasonalityType, pad); - } - - @Override - public boolean hasValue(int valuesAvailable) { - // We need at least (period * 2) data-points (e.g. two "seasons") - return valuesAvailable >= period * 2; - } - - /** - * Predicts the next `n` values in the series, using the smoothing model to generate new values. - * Unlike the other moving averages, HoltWinters has forecasting/prediction built into the algorithm. - * Prediction is more than simply adding the next prediction to the window and repeating. HoltWinters - * will extrapolate into the future by applying the trend and seasonal information to the smoothed data. - * - * @param values Collection of numerics to movingAvg, usually windowed - * @param numPredictions Number of newly generated predictions to return - * @return Returns an array of doubles, since most smoothing methods operate on floating points - */ - @Override - protected double[] doPredict(Collection values, int numPredictions) { - return next(values, numPredictions); - } - - @Override - public double next(Collection values) { - return next(values, 1)[0]; - } - - /** - * Calculate a doubly exponential weighted moving average - * - * @param values Collection of values to calculate avg for - * @param numForecasts number of forecasts into the future to return - * - * @return Returns a Double containing the moving avg for the window - */ - public double[] next(Collection values, int numForecasts) { - return MovingFunctions.holtWintersForecast(values.stream().mapToDouble(Double::doubleValue).toArray(), - alpha, beta, gamma, period, padding, seasonalityType.equals(SeasonalityType.MULTIPLICATIVE), numForecasts); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - builder.field("beta", beta); - builder.field("gamma", gamma); - builder.field("period", period); - builder.field("pad", pad); - builder.field("type", seasonalityType.getName()); - builder.endObject(); - return builder; - } - - public static final AbstractModelParser PARSER = new AbstractModelParser() { - @Override - public MovAvgModel parse(@Nullable Map settings, String pipelineName, int windowSize) throws ParseException { - - double alpha = parseDoubleParam(settings, "alpha", DEFAULT_ALPHA); - double beta = parseDoubleParam(settings, "beta", DEFAULT_BETA); - double gamma = parseDoubleParam(settings, "gamma", DEFAULT_GAMMA); - int period = parseIntegerParam(settings, "period", DEFAULT_PERIOD); - - if (windowSize < 2 * period) { - throw new ParseException("Field [window] must be at least twice as large as the period when " + - "using Holt-Winters. Value provided was [" + windowSize + "], which is less than (2*period) == " - + (2 * period), 0); - } - - SeasonalityType seasonalityType = DEFAULT_SEASONALITY_TYPE; - - if (settings != null) { - Object value = settings.get("type"); - if (value != null) { - if (value instanceof String) { - seasonalityType = SeasonalityType.parse((String)value); - settings.remove("type"); - } else { - throw new ParseException("Parameter [type] must be a String, type `" - + value.getClass().getSimpleName() + "` provided instead", 0); - } - } - } - - boolean pad = parseBoolParam(settings, "pad", seasonalityType.equals(SeasonalityType.MULTIPLICATIVE)); - - checkUnrecognizedParams(settings); - return new HoltWintersModel(alpha, beta, gamma, period, seasonalityType, pad); - } - }; - - @Override - public int hashCode() { - return Objects.hash(alpha, beta, gamma, period, seasonalityType, pad); - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - HoltWintersModel other = (HoltWintersModel) obj; - return Objects.equals(alpha, other.alpha) - && Objects.equals(beta, other.beta) - && Objects.equals(gamma, other.gamma) - && Objects.equals(period, other.period) - && Objects.equals(seasonalityType, other.seasonalityType) - && Objects.equals(pad, other.pad); - } - - public static class HoltWintersModelBuilder implements MovAvgModelBuilder { - - private double alpha = DEFAULT_ALPHA; - private double beta = DEFAULT_BETA; - private double gamma = DEFAULT_GAMMA; - private int period = DEFAULT_PERIOD; - private SeasonalityType seasonalityType = DEFAULT_SEASONALITY_TYPE; - private Boolean pad = null; - - /** - * Alpha controls the smoothing of the data. Alpha = 1 retains no memory of past values - * (e.g. a random walk), while alpha = 0 retains infinite memory of past values (e.g. - * the series mean). Useful values are somewhere in between. Defaults to 0.5. - * - * @param alpha A double between 0-1 inclusive, controls data smoothing - * - * @return The builder to continue chaining - */ - public HoltWintersModelBuilder alpha(double alpha) { - this.alpha = alpha; - return this; - } - - /** - * Equivalent to alpha, but controls the smoothing of the trend instead of the data - * - * @param beta a double between 0-1 inclusive, controls trend smoothing - * - * @return The builder to continue chaining - */ - public HoltWintersModelBuilder beta(double beta) { - this.beta = beta; - return this; - } - - public HoltWintersModelBuilder gamma(double gamma) { - this.gamma = gamma; - return this; - } - - public HoltWintersModelBuilder period(int period) { - this.period = period; - return this; - } - - public HoltWintersModelBuilder seasonalityType(SeasonalityType type) { - this.seasonalityType = type; - return this; - } - - public HoltWintersModelBuilder pad(boolean pad) { - this.pad = pad; - return this; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - builder.startObject(MovAvgPipelineAggregationBuilder.SETTINGS.getPreferredName()); - builder.field("alpha", alpha); - builder.field("beta", beta); - builder.field("gamma", gamma); - builder.field("period", period); - if (pad != null) { - builder.field("pad", pad); - } - builder.field("type", seasonalityType.getName()); - - builder.endObject(); - return builder; - } - - @Override - public MovAvgModel build() { - boolean pad = this.pad == null ? (seasonalityType == SeasonalityType.MULTIPLICATIVE) : this.pad; - return new HoltWintersModel(alpha, beta, gamma, period, seasonalityType, pad); - } - } -} - diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/LinearModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/LinearModel.java deleted file mode 100644 index 310403fca83d9..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/LinearModel.java +++ /dev/null @@ -1,132 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; - -/** - * Calculate a linearly weighted moving average, such that older values are - * linearly less important. "Time" is determined by position in collection - */ -public class LinearModel extends MovAvgModel { - public static final String NAME = "linear"; - - public LinearModel() { - } - - /** - * Read from a stream. - */ - public LinearModel(StreamInput in) { - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - // No state to write - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public boolean canBeMinimized() { - return false; - } - - @Override - public MovAvgModel neighboringModel() { - return new LinearModel(); - } - - @Override - public MovAvgModel clone() { - return new LinearModel(); - } - - @Override - protected double[] doPredict(Collection values, int numPredictions) { - double[] predictions = new double[numPredictions]; - - // EWMA just emits the same final prediction repeatedly. - Arrays.fill(predictions, next(values)); - - return predictions; - } - - @Override - public double next(Collection values) { - return MovingFunctions.linearWeightedAvg(values.stream().mapToDouble(Double::doubleValue).toArray()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - return builder; - } - - public static final AbstractModelParser PARSER = new AbstractModelParser() { - @Override - public MovAvgModel parse(@Nullable Map settings, String pipelineName, int windowSize) throws ParseException { - checkUnrecognizedParams(settings); - return new LinearModel(); - } - }; - - public static class LinearModelBuilder implements MovAvgModelBuilder { - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - return builder; - } - - @Override - public MovAvgModel build() { - return new LinearModel(); - } - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - return true; - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModel.java deleted file mode 100644 index 7c47d2eadf1a6..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModel.java +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.NamedWriteable; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ToXContentFragment; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; - -public abstract class MovAvgModel implements NamedWriteable, ToXContentFragment { - - /** - * Should this model be fit to the data via a cost minimizing algorithm by default? - */ - public boolean minimizeByDefault() { - return false; - } - - /** - * Returns if the model can be cost minimized. Not all models have parameters - * which can be tuned / optimized. - */ - public abstract boolean canBeMinimized(); - - /** - * Generates a "neighboring" model, where one of the tunable parameters has been - * randomly mutated within the allowed range. Used for minimization - */ - public abstract MovAvgModel neighboringModel(); - - /** - * Checks to see this model can produce a new value, without actually running the algo. - * This can be used for models that have certain preconditions that need to be met in order - * to short-circuit execution - * - * @param valuesAvailable Number of values in the current window of values - * @return Returns `true` if calling next() will produce a value, `false` otherwise - */ - public boolean hasValue(int valuesAvailable) { - // Default implementation can always provide a next() value - return valuesAvailable > 0; - } - - /** - * Returns the next value in the series, according to the underlying smoothing model - * - * @param values Collection of numerics to movingAvg, usually windowed - * @return Returns a double, since most smoothing methods operate on floating points - */ - public abstract double next(Collection values); - - /** - * Predicts the next `n` values in the series. - * - * @param values Collection of numerics to movingAvg, usually windowed - * @param numPredictions Number of newly generated predictions to return - * @return Returns an array of doubles, since most smoothing methods operate on floating points - */ - public double[] predict(Collection values, int numPredictions) { - assert(numPredictions >= 1); - - // If there are no values, we can't do anything. Return an array of NaNs. - if (values.isEmpty()) { - return emptyPredictions(numPredictions); - } - - return doPredict(values, numPredictions); - } - - /** - * Calls to the model-specific implementation which actually generates the predictions - * - * @param values Collection of numerics to movingAvg, usually windowed - * @param numPredictions Number of newly generated predictions to return - * @return Returns an array of doubles, since most smoothing methods operate on floating points - */ - protected abstract double[] doPredict(Collection values, int numPredictions); - - /** - * Returns an empty set of predictions, filled with NaNs - * @param numPredictions Number of empty predictions to generate - */ - protected double[] emptyPredictions(int numPredictions) { - double[] predictions = new double[numPredictions]; - Arrays.fill(predictions, Double.NaN); - return predictions; - } - - /** - * Write the model to the output stream - * - * @param out Output stream - */ - @Override - public abstract void writeTo(StreamOutput out) throws IOException; - - /** - * Clone the model, returning an exact copy - */ - @Override - public abstract MovAvgModel clone(); - - @Override - public abstract int hashCode(); - - @Override - public abstract boolean equals(Object obj); - - /** - * Abstract class which also provides some concrete parsing functionality. - */ - public abstract static class AbstractModelParser { - /** - * Parse a settings hash that is specific to this model - * - * @param settings Map of settings, extracted from the request - * @param pipelineName Name of the parent pipeline agg - * @param windowSize Size of the window for this moving avg - * @return A fully built moving average model - */ - public abstract MovAvgModel parse(@Nullable Map settings, String pipelineName, - int windowSize) throws ParseException; - - - /** - * Extracts a 0-1 inclusive double from the settings map, otherwise throws an exception - * - * @param settings Map of settings provided to this model - * @param name Name of parameter we are attempting to extract - * @param defaultValue Default value to be used if value does not exist in map - * @return Double value extracted from settings map - */ - protected double parseDoubleParam(@Nullable Map settings, String name, double defaultValue) throws ParseException { - if (settings == null) { - return defaultValue; - } - - Object value = settings.get(name); - if (value == null) { - return defaultValue; - } else if (value instanceof Number) { - double v = ((Number) value).doubleValue(); - if (v >= 0 && v <= 1) { - settings.remove(name); - return v; - } - - throw new ParseException("Parameter [" + name + "] must be between 0-1 inclusive. Provided" - + "value was [" + v + "]", 0); - } - - throw new ParseException("Parameter [" + name + "] must be a double, type `" - + value.getClass().getSimpleName() + "` provided instead", 0); - } - - /** - * Extracts an integer from the settings map, otherwise throws an exception - * - * @param settings Map of settings provided to this model - * @param name Name of parameter we are attempting to extract - * @param defaultValue Default value to be used if value does not exist in map - * @return Integer value extracted from settings map - */ - protected int parseIntegerParam(@Nullable Map settings, String name, int defaultValue) throws ParseException { - if (settings == null) { - return defaultValue; - } - - Object value = settings.get(name); - if (value == null) { - return defaultValue; - } else if (value instanceof Number) { - settings.remove(name); - return ((Number) value).intValue(); - } - - throw new ParseException("Parameter [" + name + "] must be an integer, type `" - + value.getClass().getSimpleName() + "` provided instead", 0); - } - - /** - * Extracts a boolean from the settings map, otherwise throws an exception - * - * @param settings Map of settings provided to this model - * @param name Name of parameter we are attempting to extract - * @param defaultValue Default value to be used if value does not exist in map - * @return Boolean value extracted from settings map - */ - protected boolean parseBoolParam(@Nullable Map settings, String name, boolean defaultValue) throws ParseException { - if (settings == null) { - return defaultValue; - } - - Object value = settings.get(name); - if (value == null) { - return defaultValue; - } else if (value instanceof Boolean) { - settings.remove(name); - return (Boolean)value; - } - - throw new ParseException("Parameter [" + name + "] must be a boolean, type `" - + value.getClass().getSimpleName() + "` provided instead", 0); - } - - protected void checkUnrecognizedParams(@Nullable Map settings) throws ParseException { - if (settings != null && settings.size() > 0) { - throw new ParseException("Unrecognized parameter(s): [" + settings.keySet() + "]", 0); - } - } - } - -} - - - - diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModelBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModelBuilder.java deleted file mode 100644 index 1cb13dd200c7d..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgModelBuilder.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.xcontent.ToXContentFragment; - -/** - * Represents the common interface that all moving average models share. Moving - * average models are used by the MovAvg aggregation - */ -public interface MovAvgModelBuilder extends ToXContentFragment { - - MovAvgModel build(); -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java deleted file mode 100644 index 7258a02d56637..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregationBuilder.java +++ /dev/null @@ -1,427 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.apache.logging.log4j.LogManager; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.logging.DeprecationLogger; -import org.elasticsearch.common.xcontent.ParseFieldRegistry; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; - -import java.io.IOException; -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.BUCKETS_PATH; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.FORMAT; -import static org.elasticsearch.search.aggregations.pipeline.PipelineAggregator.Parser.GAP_POLICY; - -public class MovAvgPipelineAggregationBuilder extends AbstractPipelineAggregationBuilder { - public static final String NAME = "moving_avg"; - - static final ParseField MODEL = new ParseField("model"); - private static final ParseField WINDOW = new ParseField("window"); - public static final ParseField SETTINGS = new ParseField("settings"); - private static final ParseField PREDICT = new ParseField("predict"); - private static final ParseField MINIMIZE = new ParseField("minimize"); - private static final DeprecationLogger DEPRECATION_LOGGER - = new DeprecationLogger(LogManager.getLogger(MovAvgPipelineAggregationBuilder.class)); - - private String format; - private GapPolicy gapPolicy = GapPolicy.SKIP; - private int window = 5; - private MovAvgModel model = new SimpleModel(); - private int predict = 0; - private Boolean minimize; - - public MovAvgPipelineAggregationBuilder(String name, String bucketsPath) { - super(name, NAME, new String[] { bucketsPath }); - } - - /** - * Read from a stream. - */ - public MovAvgPipelineAggregationBuilder(StreamInput in) throws IOException { - super(in, NAME); - format = in.readOptionalString(); - gapPolicy = GapPolicy.readFrom(in); - window = in.readVInt(); - model = in.readNamedWriteable(MovAvgModel.class); - predict = in.readVInt(); - minimize = in.readOptionalBoolean(); - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - out.writeOptionalString(format); - gapPolicy.writeTo(out); - out.writeVInt(window); - out.writeNamedWriteable(model); - out.writeVInt(predict); - out.writeOptionalBoolean(minimize); - } - - /** - * Sets the format to use on the output of this aggregation. - */ - public MovAvgPipelineAggregationBuilder format(String format) { - if (format == null) { - throw new IllegalArgumentException("[format] must not be null: [" + name + "]"); - } - this.format = format; - return this; - } - - /** - * Gets the format to use on the output of this aggregation. - */ - public String format() { - return format; - } - - /** - * Sets the GapPolicy to use on the output of this aggregation. - */ - public MovAvgPipelineAggregationBuilder gapPolicy(GapPolicy gapPolicy) { - if (gapPolicy == null) { - throw new IllegalArgumentException("[gapPolicy] must not be null: [" + name + "]"); - } - this.gapPolicy = gapPolicy; - return this; - } - - /** - * Gets the GapPolicy to use on the output of this aggregation. - */ - public GapPolicy gapPolicy() { - return gapPolicy; - } - - protected DocValueFormat formatter() { - if (format != null) { - return new DocValueFormat.Decimal(format); - } else { - return DocValueFormat.RAW; - } - } - - /** - * Sets the window size for the moving average. This window will "slide" - * across the series, and the values inside that window will be used to - * calculate the moving avg value - * - * @param window - * Size of window - */ - public MovAvgPipelineAggregationBuilder window(int window) { - if (window <= 0) { - throw new IllegalArgumentException("[window] must be a positive integer: [" + name + "]"); - } - this.window = window; - return this; - } - - /** - * Gets the window size for the moving average. This window will "slide" - * across the series, and the values inside that window will be used to - * calculate the moving avg value - */ - public int window() { - return window; - } - - /** - * Sets a MovAvgModel for the Moving Average. The model is used to - * define what type of moving average you want to use on the series - * - * @param model - * A MovAvgModel which has been prepopulated with settings - */ - public MovAvgPipelineAggregationBuilder modelBuilder(MovAvgModelBuilder model) { - if (model == null) { - throw new IllegalArgumentException("[model] must not be null: [" + name + "]"); - } - this.model = model.build(); - return this; - } - - /** - * Sets a MovAvgModel for the Moving Average. The model is used to - * define what type of moving average you want to use on the series - * - * @param model - * A MovAvgModel which has been prepopulated with settings - */ - public MovAvgPipelineAggregationBuilder model(MovAvgModel model) { - if (model == null) { - throw new IllegalArgumentException("[model] must not be null: [" + name + "]"); - } - this.model = model; - return this; - } - - /** - * Gets a MovAvgModel for the Moving Average. The model is used to - * define what type of moving average you want to use on the series - */ - public MovAvgModel model() { - return model; - } - - /** - * Sets the number of predictions that should be returned. Each - * prediction will be spaced at the intervals specified in the - * histogram. E.g "predict: 2" will return two new buckets at the end of - * the histogram with the predicted values. - * - * @param predict - * Number of predictions to make - */ - public MovAvgPipelineAggregationBuilder predict(int predict) { - if (predict <= 0) { - throw new IllegalArgumentException("predict must be greater than 0. Found [" + predict + "] in [" + name + "]"); - } - this.predict = predict; - return this; - } - - /** - * Gets the number of predictions that should be returned. Each - * prediction will be spaced at the intervals specified in the - * histogram. E.g "predict: 2" will return two new buckets at the end of - * the histogram with the predicted values. - */ - public int predict() { - return predict; - } - - /** - * Sets whether the model should be fit to the data using a cost - * minimizing algorithm. - * - * @param minimize - * If the model should be fit to the underlying data - */ - public MovAvgPipelineAggregationBuilder minimize(boolean minimize) { - this.minimize = minimize; - return this; - } - - /** - * Gets whether the model should be fit to the data using a cost - * minimizing algorithm. - */ - public Boolean minimize() { - return minimize; - } - - @Override - protected PipelineAggregator createInternal(Map metaData) throws IOException { - // If the user doesn't set a preference for cost minimization, ask - // what the model prefers - boolean minimize = this.minimize == null ? model.minimizeByDefault() : this.minimize; - return new MovAvgPipelineAggregator(name, bucketsPaths, formatter(), gapPolicy, window, predict, model, minimize, metaData); - } - - @Override - public void doValidate(AggregatorFactory parent, Collection aggFactories, - Collection pipelineAggregatoractories) { - if (minimize != null && minimize && !model.canBeMinimized()) { - // If the user asks to minimize, but this model doesn't support - // it, throw exception - throw new IllegalStateException("The [" + model + "] model cannot be minimized for aggregation [" + name + "]"); - } - if (bucketsPaths.length != 1) { - throw new IllegalStateException(PipelineAggregator.Parser.BUCKETS_PATH.getPreferredName() - + " must contain a single entry for aggregation [" + name + "]"); - } - - validateSequentiallyOrderedParentAggs(parent, NAME, name); - } - - @Override - protected XContentBuilder internalXContent(XContentBuilder builder, Params params) throws IOException { - if (format != null) { - builder.field(FORMAT.getPreferredName(), format); - } - builder.field(GAP_POLICY.getPreferredName(), gapPolicy.getName()); - model.toXContent(builder, params); - builder.field(WINDOW.getPreferredName(), window); - if (predict > 0) { - builder.field(PREDICT.getPreferredName(), predict); - } - if (minimize != null) { - builder.field(MINIMIZE.getPreferredName(), minimize); - } - return builder; - } - - public static MovAvgPipelineAggregationBuilder parse( - ParseFieldRegistry movingAverageMdelParserRegistry, - String pipelineAggregatorName, XContentParser parser) throws IOException { - XContentParser.Token token; - String currentFieldName = null; - String[] bucketsPaths = null; - String format = null; - - GapPolicy gapPolicy = null; - Integer window = null; - Map settings = null; - String model = null; - Integer predict = null; - Boolean minimize = null; - - DEPRECATION_LOGGER.deprecated("The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation."); - - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - } else if (token == XContentParser.Token.VALUE_NUMBER) { - if (WINDOW.match(currentFieldName, parser.getDeprecationHandler())) { - window = parser.intValue(); - if (window <= 0) { - throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive, " - + "non-zero integer. Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "]."); - } - } else if (PREDICT.match(currentFieldName, parser.getDeprecationHandler())) { - predict = parser.intValue(); - if (predict <= 0) { - throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive integer." - + " Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "]."); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "]."); - } - } else if (token == XContentParser.Token.VALUE_STRING) { - if (FORMAT.match(currentFieldName, parser.getDeprecationHandler())) { - format = parser.text(); - } else if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) { - bucketsPaths = new String[] { parser.text() }; - } else if (GAP_POLICY.match(currentFieldName, parser.getDeprecationHandler())) { - gapPolicy = GapPolicy.parse(parser.text(), parser.getTokenLocation()); - } else if (MODEL.match(currentFieldName, parser.getDeprecationHandler())) { - model = parser.text(); - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "]."); - } - } else if (token == XContentParser.Token.START_ARRAY) { - if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) { - List paths = new ArrayList<>(); - while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) { - String path = parser.text(); - paths.add(path); - } - bucketsPaths = paths.toArray(new String[paths.size()]); - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "]."); - } - } else if (token == XContentParser.Token.START_OBJECT) { - if (SETTINGS.match(currentFieldName, parser.getDeprecationHandler())) { - settings = parser.map(); - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "]."); - } - } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - if (MINIMIZE.match(currentFieldName, parser.getDeprecationHandler())) { - minimize = parser.booleanValue(); - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "]."); - } - } else { - throw new ParsingException(parser.getTokenLocation(), - "Unexpected token " + token + " in [" + pipelineAggregatorName + "]."); - } - } - - if (bucketsPaths == null) { - throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH.getPreferredName() - + "] for movingAvg aggregation [" + pipelineAggregatorName + "]"); - } - - MovAvgPipelineAggregationBuilder factory = - new MovAvgPipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]); - if (format != null) { - factory.format(format); - } - if (gapPolicy != null) { - factory.gapPolicy(gapPolicy); - } - if (window != null) { - factory.window(window); - } - if (predict != null) { - factory.predict(predict); - } - if (model != null) { - MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, - parser.getTokenLocation(), parser.getDeprecationHandler()); - MovAvgModel movAvgModel; - try { - movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window()); - } catch (ParseException exception) { - throw new ParsingException(parser.getTokenLocation(), "Could not parse settings for model [" + model + "].", exception); - } - factory.model(movAvgModel); - } - if (minimize != null) { - factory.minimize(minimize); - } - return factory; - } - - @Override - protected int doHashCode() { - return Objects.hash(format, gapPolicy, window, model, predict, minimize); - } - - @Override - protected boolean doEquals(Object obj) { - MovAvgPipelineAggregationBuilder other = (MovAvgPipelineAggregationBuilder) obj; - return Objects.equals(format, other.format) - && Objects.equals(gapPolicy, other.gapPolicy) - && Objects.equals(window, other.window) - && Objects.equals(model, other.model) - && Objects.equals(predict, other.predict) - && Objects.equals(minimize, other.minimize); - } - - @Override - public String getWriteableName() { - return NAME; - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregator.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregator.java deleted file mode 100644 index 1f36c989c163d..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/MovAvgPipelineAggregator.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.collect.EvictingQueue; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.InternalAggregation.ReduceContext; -import org.elasticsearch.search.aggregations.InternalAggregations; -import org.elasticsearch.search.aggregations.InternalMultiBucketAggregation; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation; -import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket; -import org.elasticsearch.search.aggregations.bucket.histogram.HistogramFactory; -import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.ListIterator; -import java.util.Map; -import java.util.stream.Collectors; -import java.util.stream.StreamSupport; - -import static org.elasticsearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue; - -public class MovAvgPipelineAggregator extends PipelineAggregator { - private final DocValueFormat formatter; - private final GapPolicy gapPolicy; - private final int window; - private MovAvgModel model; - private final int predict; - private final boolean minimize; - - MovAvgPipelineAggregator(String name, String[] bucketsPaths, DocValueFormat formatter, GapPolicy gapPolicy, - int window, int predict, MovAvgModel model, boolean minimize, Map metadata) { - super(name, bucketsPaths, metadata); - this.formatter = formatter; - this.gapPolicy = gapPolicy; - this.window = window; - this.model = model; - this.predict = predict; - this.minimize = minimize; - } - - /** - * Read from a stream. - */ - public MovAvgPipelineAggregator(StreamInput in) throws IOException { - super(in); - formatter = in.readNamedWriteable(DocValueFormat.class); - gapPolicy = GapPolicy.readFrom(in); - window = in.readVInt(); - predict = in.readVInt(); - model = in.readNamedWriteable(MovAvgModel.class); - minimize = in.readBoolean(); - } - - @Override - public void doWriteTo(StreamOutput out) throws IOException { - out.writeNamedWriteable(formatter); - gapPolicy.writeTo(out); - out.writeVInt(window); - out.writeVInt(predict); - out.writeNamedWriteable(model); - out.writeBoolean(minimize); - } - - @Override - public String getWriteableName() { - return MovAvgPipelineAggregationBuilder.NAME; - } - - @Override - public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) { - InternalMultiBucketAggregation - histo = (InternalMultiBucketAggregation) aggregation; - List buckets = histo.getBuckets(); - HistogramFactory factory = (HistogramFactory) histo; - - List newBuckets = new ArrayList<>(); - EvictingQueue values = new EvictingQueue<>(this.window); - - Number lastValidKey = 0; - int lastValidPosition = 0; - int counter = 0; - - // Do we need to fit the model parameters to the data? - if (minimize) { - assert (model.canBeMinimized()); - model = minimize(buckets, histo, model); - } - - for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) { - Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy); - - // Default is to reuse existing bucket. Simplifies the rest of the logic, - // since we only change newBucket if we can add to it - Bucket newBucket = bucket; - - if ((thisBucketValue == null || thisBucketValue.equals(Double.NaN)) == false) { - - // Some models (e.g. HoltWinters) have certain preconditions that must be met - if (model.hasValue(values.size())) { - double movavg = model.next(values); - - List aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false) - .map((p) -> (InternalAggregation) p) - .collect(Collectors.toList()); - aggs.add(new InternalSimpleValue(name(), movavg, formatter, new ArrayList<>(), metaData())); - newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs)); - } - - if (predict > 0) { - lastValidKey = factory.getKey(bucket); - lastValidPosition = counter; - } - - values.offer(thisBucketValue); - } - counter += 1; - newBuckets.add(newBucket); - - } - - if (buckets.size() > 0 && predict > 0) { - double[] predictions = model.predict(values, predict); - for (int i = 0; i < predictions.length; i++) { - - List aggs; - Number newKey = factory.nextKey(lastValidKey); - - if (lastValidPosition + i + 1 < newBuckets.size()) { - Bucket bucket = newBuckets.get(lastValidPosition + i + 1); - - // Get the existing aggs in the bucket so we don't clobber data - aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false) - .map((p) -> (InternalAggregation) p) - .collect(Collectors.toList()); - aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<>(), metaData())); - - Bucket newBucket = factory.createBucket(newKey, bucket.getDocCount(), new InternalAggregations(aggs)); - - // Overwrite the existing bucket with the new version - newBuckets.set(lastValidPosition + i + 1, newBucket); - - } else { - // Not seen before, create fresh - aggs = new ArrayList<>(); - aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<>(), metaData())); - - Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs)); - - // Since this is a new bucket, simply append it - newBuckets.add(newBucket); - } - lastValidKey = newKey; - } - } - - return factory.createAggregation(newBuckets); - } - - private MovAvgModel minimize(List buckets, - MultiBucketsAggregation histo, MovAvgModel model) { - - int counter = 0; - EvictingQueue values = new EvictingQueue<>(this.window); - - double[] test = new double[window]; - ListIterator iter = buckets.listIterator(buckets.size()); - - // We have to walk the iterator backwards because we don't know if/how many buckets are empty. - while (iter.hasPrevious() && counter < window) { - - Double thisBucketValue = resolveBucketValue(histo, iter.previous(), bucketsPaths()[0], gapPolicy); - - if (!(thisBucketValue == null || thisBucketValue.equals(Double.NaN))) { - test[window - counter - 1] = thisBucketValue; - counter += 1; - } - } - - // If we didn't fill the test set, we don't have enough data to minimize. - // Just return the model with the starting coef - if (counter < window) { - return model; - } - - //And do it again, for the train set. Unfortunately we have to fill an array and then - //fill an evicting queue backwards :( - - counter = 0; - double[] train = new double[window]; - - while (iter.hasPrevious() && counter < window) { - - Double thisBucketValue = resolveBucketValue(histo, iter.previous(), bucketsPaths()[0], gapPolicy); - - if (!(thisBucketValue == null || thisBucketValue.equals(Double.NaN))) { - train[window - counter - 1] = thisBucketValue; - counter += 1; - } - } - - // If we didn't fill the train set, we don't have enough data to minimize. - // Just return the model with the starting coef - if (counter < window) { - return model; - } - - for (double v : train) { - values.add(v); - } - - return SimulatedAnealingMinimizer.minimize(model, values, test); - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimpleModel.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimpleModel.java deleted file mode 100644 index a64131278a563..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimpleModel.java +++ /dev/null @@ -1,130 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.Nullable; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; - -import java.io.IOException; -import java.text.ParseException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; - -/** - * Calculate a simple unweighted (arithmetic) moving average - */ -public class SimpleModel extends MovAvgModel { - public static final String NAME = "simple"; - - public SimpleModel() { - } - - /** - * Read from a stream. - */ - public SimpleModel(StreamInput in) throws IOException { - } - - @Override - public void writeTo(StreamOutput out) throws IOException { - // Nothing to write - } - - @Override - public String getWriteableName() { - return NAME; - } - - @Override - public boolean canBeMinimized() { - return false; - } - - @Override - public MovAvgModel neighboringModel() { - return new SimpleModel(); - } - - @Override - public MovAvgModel clone() { - return new SimpleModel(); - } - - @Override - protected double[] doPredict(Collection values, int numPredictions) { - double[] predictions = new double[numPredictions]; - - // Simple just emits the same final prediction repeatedly. - Arrays.fill(predictions, next(values)); - - return predictions; - } - - @Override - public double next(Collection values) { - return MovingFunctions.unweightedAvg(values.stream().mapToDouble(Double::doubleValue).toArray()); - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - return builder; - } - - public static final AbstractModelParser PARSER = new AbstractModelParser() { - @Override - public MovAvgModel parse(@Nullable Map settings, String pipelineName, int windowSize) throws ParseException { - checkUnrecognizedParams(settings); - return new SimpleModel(); - } - }; - - public static class SimpleModelBuilder implements MovAvgModelBuilder { - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.field(MovAvgPipelineAggregationBuilder.MODEL.getPreferredName(), NAME); - return builder; - } - - @Override - public MovAvgModel build() { - return new SimpleModel(); - } - } - - @Override - public int hashCode() { - return 0; - } - - @Override - public boolean equals(Object obj) { - if (obj == null) { - return false; - } - if (getClass() != obj.getClass()) { - return false; - } - return true; - } -} diff --git a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java b/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java deleted file mode 100644 index e157b2cec1c6f..0000000000000 --- a/server/src/main/java/org/elasticsearch/search/aggregations/pipeline/SimulatedAnealingMinimizer.java +++ /dev/null @@ -1,124 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.collect.EvictingQueue; - -/** - * A cost minimizer which will fit a MovAvgModel to the data. - * - * This optimizer uses naive simulated annealing. Random solutions in the problem space - * are generated, compared against the last period of data, and the least absolute deviation - * is recorded as a cost. - * - * If the new cost is better than the old cost, the new coefficients are chosen. If the new - * solution is worse, there is a temperature-dependent probability it will be randomly selected - * anyway. This allows the algo to sample the problem space widely. As iterations progress, - * the temperature decreases and the algorithm rejects poor solutions more regularly, - * theoretically honing in on a global minimum. - */ -public class SimulatedAnealingMinimizer { - - /** - * Runs the simulated annealing algorithm and produces a model with new coefficients that, theoretically - * fit the data better and generalizes to future forecasts without overfitting. - * - * @param model The MovAvgModel to be optimized for - * @param train A training set provided to the model, which predictions will be - * generated from - * @param test A test set of data to compare the predictions against and derive - * a cost for the model - * @return A new, minimized model that (theoretically) better fits the data - */ - public static MovAvgModel minimize(MovAvgModel model, EvictingQueue train, double[] test) { - - double temp = 1; - double minTemp = 0.0001; - int iterations = 100; - double alpha = 0.9; - - MovAvgModel bestModel = model; - MovAvgModel oldModel = model; - - double oldCost = cost(model, train, test); - double bestCost = oldCost; - - while (temp > minTemp) { - for (int i = 0; i < iterations; i++) { - MovAvgModel newModel = oldModel.neighboringModel(); - double newCost = cost(newModel, train, test); - - double ap = acceptanceProbability(oldCost, newCost, temp); - if (ap > Math.random()) { - oldModel = newModel; - oldCost = newCost; - - if (newCost < bestCost) { - bestCost = newCost; - bestModel = newModel; - } - } - } - - temp *= alpha; - } - - return bestModel; - } - - /** - * If the new cost is better than old, return 1.0. Otherwise, return a double that increases - * as the two costs are closer to each other. - * - * @param oldCost Old model cost - * @param newCost New model cost - * @param temp Current annealing temperature - * @return The probability of accepting the new cost over the old - */ - private static double acceptanceProbability(double oldCost, double newCost, double temp) { - return newCost < oldCost ? 1.0 : Math.exp(-(newCost - oldCost) / temp); - } - - /** - * Calculates the "cost" of a model. E.g. when run on the training data, how closely do the predictions - * match the test data - * - * Uses Least Absolute Differences to calculate error. Note that this is not scale free, but seems - * to work fairly well in practice - * - * @param model The MovAvgModel we are fitting - * @param train A training set of data given to the model, which will then generate predictions from - * @param test A test set of data to compare against the predictions - * @return A cost, or error, of the model - */ - private static double cost(MovAvgModel model, EvictingQueue train, double[] test) { - double error = 0; - double[] predictions = model.predict(train, test.length); - - assert(predictions.length == test.length); - - for (int i = 0; i < predictions.length; i++) { - error += Math.abs(test[i] - predictions[i]) ; - } - - return error; - } - -} diff --git a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java index ccdfe28b44fdc..6e5a6d0da5475 100644 --- a/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java +++ b/server/src/test/java/org/elasticsearch/search/SearchModuleTests.java @@ -47,9 +47,7 @@ import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregationBuilder; import org.elasticsearch.search.aggregations.pipeline.DerivativePipelineAggregator; import org.elasticsearch.search.aggregations.pipeline.InternalDerivative; -import org.elasticsearch.search.aggregations.pipeline.MovAvgModel; import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator; -import org.elasticsearch.search.aggregations.pipeline.SimpleModel; import org.elasticsearch.search.aggregations.support.ValuesSource; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; @@ -126,14 +124,6 @@ public List> getMovingAverageModels() { - return singletonList(new SearchExtensionSpec<>(SimpleModel.NAME, SimpleModel::new, SimpleModel.PARSER)); - } - }; - expectThrows(IllegalArgumentException.class, registryForPlugin(registersDupeMovAvgModel)); - SearchPlugin registersDupeFetchSubPhase = new SearchPlugin() { @Override public List getFetchSubPhases(FetchPhaseConstructionContext context) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java index 3ad41bf3dc866..637a2db947dd2 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeAggregatorTests.java @@ -47,8 +47,8 @@ import org.elasticsearch.search.aggregations.metrics.StatsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.Sum; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.aggregations.support.AggregationPath; import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; +import org.elasticsearch.search.aggregations.support.AggregationPath; import java.io.IOException; import java.util.List; @@ -682,20 +682,14 @@ public void testSingleValueAggDerivative_invalidPath() throws IOException { } } - public void testAvgMovavgDerivNPE() throws IOException { + public void testDerivDerivNPE() throws IOException { try (Directory directory = newDirectory()) { Query query = new MatchAllDocsQuery(); HistogramAggregationBuilder aggBuilder = new HistogramAggregationBuilder("histo") .field("tick").interval(1); aggBuilder.subAggregation(new AvgAggregationBuilder("avg").field("value")); - aggBuilder.subAggregation( - new MovAvgPipelineAggregationBuilder("movavg", "avg") - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .window(3) - ); - aggBuilder.subAggregation( - new DerivativePipelineAggregationBuilder("deriv", "movavg") - ); + aggBuilder.subAggregation(new DerivativePipelineAggregationBuilder("deriv1", "avg")); + aggBuilder.subAggregation(new DerivativePipelineAggregationBuilder("deriv2", "deriv1")); try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { Document document = new Document(); for (int i = 0; i < 10; i++) { diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java index df88ba95f5335..209fe6562593e 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/DerivativeIT.java @@ -42,13 +42,12 @@ import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; import static org.elasticsearch.index.query.QueryBuilders.matchAllQuery; +import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.AggregationBuilders.filters; import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; import static org.elasticsearch.search.aggregations.AggregationBuilders.stats; import static org.elasticsearch.search.aggregations.AggregationBuilders.sum; -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.derivative; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.movingAvg; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked; import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; import static org.hamcrest.Matchers.closeTo; @@ -615,8 +614,8 @@ public void testSingleValueAggDerivative_invalidPath() throws Exception { } } - public void testAvgMovavgDerivNPE() throws Exception { - createIndex("movavg_npe"); + public void testDerivDerivNPE() throws Exception { + createIndex("deriv_npe"); for (int i = 0; i < 10; i++) { Integer value = i; @@ -629,18 +628,18 @@ public void testAvgMovavgDerivNPE() throws Exception { .field("tick", i) .field("value", value) .endObject(); - client().prepareIndex("movavg_npe", "type").setSource(doc).get(); + client().prepareIndex("deriv_npe", "type").setSource(doc).get(); } refresh(); SearchResponse response = client() - .prepareSearch("movavg_npe") + .prepareSearch("deriv_npe") .addAggregation( histogram("histo").field("tick").interval(1) .subAggregation(avg("avg").field("value")) - .subAggregation(movingAvg("movavg", "avg").modelBuilder(new SimpleModel.SimpleModelBuilder()).window(3)) - .subAggregation(derivative("deriv", "movavg"))).get(); + .subAggregation(derivative("deriv1", "avg")) + .subAggregation(derivative("deriv2", "deriv1"))).get(); assertSearchResponse(response); } diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgIT.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgIT.java deleted file mode 100644 index 23a7231e269a1..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgIT.java +++ /dev/null @@ -1,1347 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.action.bulk.BulkRequestBuilder; -import org.elasticsearch.action.index.IndexRequestBuilder; -import org.elasticsearch.action.search.SearchPhaseExecutionException; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.support.WriteRequest; -import org.elasticsearch.client.Client; -import org.elasticsearch.common.collect.EvictingQueue; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram; -import org.elasticsearch.search.aggregations.bucket.histogram.Histogram.Bucket; -import org.elasticsearch.search.aggregations.metrics.Avg; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.elasticsearch.test.ESIntegTestCase; -import org.hamcrest.Matchers; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; - -import static org.elasticsearch.common.xcontent.XContentFactory.jsonBuilder; -import static org.elasticsearch.search.aggregations.AggregationBuilders.avg; -import static org.elasticsearch.search.aggregations.AggregationBuilders.histogram; -import static org.elasticsearch.search.aggregations.AggregationBuilders.max; -import static org.elasticsearch.search.aggregations.AggregationBuilders.min; -import static org.elasticsearch.search.aggregations.AggregationBuilders.range; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.derivative; -import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.movingAvg; -import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertSearchResponse; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.core.IsNull.notNullValue; -import static org.hamcrest.core.IsNull.nullValue; - -@ESIntegTestCase.SuiteScopeTestCase -public class MovAvgIT extends ESIntegTestCase { - private static final String INTERVAL_FIELD = "l_value"; - private static final String VALUE_FIELD = "v_value"; - private static final String VALUE_FIELD2 = "v_value2"; - - static int interval; - static int numBuckets; - static int windowSize; - static double alpha; - static double beta; - static double gamma; - static int period; - static HoltWintersModel.SeasonalityType seasonalityType; - static BucketHelpers.GapPolicy gapPolicy; - static ValuesSourceAggregationBuilder> metric; - static List mockHisto; - - static Map> testValues; - - - enum MovAvgType { - SIMPLE ("simple"), LINEAR("linear"), EWMA("ewma"), HOLT("holt"), HOLT_WINTERS("holt_winters"), HOLT_BIG_MINIMIZE("holt"); - - private final String name; - - MovAvgType(String s) { - name = s; - } - - @Override - public String toString(){ - return name; - } - } - - enum MetricTarget { - VALUE ("value"), COUNT("count"), METRIC("metric"); - - private final String name; - - MetricTarget(String s) { - name = s; - } - - @Override - public String toString(){ - return name; - } - } - - - @Override - public void setupSuiteScopeCluster() throws Exception { - createIndex("idx"); - createIndex("idx_unmapped"); - List builders = new ArrayList<>(); - - - interval = 5; - numBuckets = randomIntBetween(6, 80); - period = randomIntBetween(1, 5); - windowSize = randomIntBetween(period * 2, 10); // start must be 2*period to play nice with HW - alpha = randomDouble(); - beta = randomDouble(); - gamma = randomDouble(); - seasonalityType = randomBoolean() ? HoltWintersModel.SeasonalityType.ADDITIVE : HoltWintersModel.SeasonalityType.MULTIPLICATIVE; - - - gapPolicy = randomBoolean() ? BucketHelpers.GapPolicy.SKIP : BucketHelpers.GapPolicy.INSERT_ZEROS; - metric = randomMetric("the_metric", VALUE_FIELD); - mockHisto = PipelineAggregationHelperTests.generateHistogram(interval, numBuckets, randomDouble(), randomDouble()); - - testValues = new HashMap<>(8); - - for (MovAvgType type : MovAvgType.values()) { - for (MetricTarget target : MetricTarget.values()) { - if (type.equals(MovAvgType.HOLT_BIG_MINIMIZE)) { - setupExpected(type, target, numBuckets); - } else { - setupExpected(type, target, windowSize); - } - - } - } - - for (PipelineAggregationHelperTests.MockBucket mockBucket : mockHisto) { - for (double value : mockBucket.docValues) { - builders.add(client().prepareIndex("idx", "type").setSource(jsonBuilder().startObject() - .field(INTERVAL_FIELD, mockBucket.key) - .field(VALUE_FIELD, value).endObject())); - } - } - - for (int i = -10; i < 10; i++) { - builders.add(client().prepareIndex("neg_idx", "type").setSource( - jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD, 10).endObject())); - } - - for (int i = 0; i < 12; i++) { - builders.add(client().prepareIndex("double_predict", "type").setSource( - jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD, 10).endObject())); - } - - indexRandom(true, builders); - ensureSearchable(); - } - - /** - * Calculates the moving averages for a specific (model, target) tuple based on the previously generated mock histogram. - * Computed values are stored in the testValues map. - * - * @param type The moving average model to use - * @param target The document field "target", e.g. _count or a field value - */ - private void setupExpected(MovAvgType type, MetricTarget target, int windowSize) { - ArrayList values = new ArrayList<>(numBuckets); - EvictingQueue window = new EvictingQueue<>(windowSize); - - for (PipelineAggregationHelperTests.MockBucket mockBucket : mockHisto) { - double metricValue; - double[] docValues = mockBucket.docValues; - - // Gaps only apply to metric values, not doc _counts - if (mockBucket.count == 0 && target.equals(MetricTarget.VALUE)) { - // If there was a gap in doc counts and we are ignoring, just skip this bucket - if (gapPolicy.equals(BucketHelpers.GapPolicy.SKIP)) { - values.add(null); - continue; - } else if (gapPolicy.equals(BucketHelpers.GapPolicy.INSERT_ZEROS)) { - // otherwise insert a zero instead of the true value - metricValue = 0.0; - } else { - metricValue = PipelineAggregationHelperTests.calculateMetric(docValues, metric); - } - - } else { - // If this isn't a gap, or is a _count, just insert the value - metricValue = target.equals(MetricTarget.VALUE) - ? PipelineAggregationHelperTests.calculateMetric(docValues, metric) - : mockBucket.count; - } - - if (window.size() > 0) { - switch (type) { - case SIMPLE: - values.add(simple(window)); - break; - case LINEAR: - values.add(linear(window)); - break; - case EWMA: - values.add(ewma(window)); - break; - case HOLT: - values.add(holt(window)); - break; - case HOLT_BIG_MINIMIZE: - values.add(holt(window)); - break; - case HOLT_WINTERS: - // HW needs at least 2 periods of data to start - if (window.size() >= period * 2) { - values.add(holtWinters(window)); - } else { - values.add(null); - } - - break; - } - } else { - values.add(null); - } - - window.offer(metricValue); - - } - testValues.put(type.name() + "_" + target.name(), values); - } - - /** - * Simple, unweighted moving average - * - * @param window Window of values to compute movavg for - */ - private double simple(Collection window) { - double movAvg = 0; - for (double value : window) { - movAvg += value; - } - movAvg /= window.size(); - return movAvg; - } - - /** - * Linearly weighted moving avg - * - * @param window Window of values to compute movavg for - */ - private double linear(Collection window) { - double avg = 0; - long totalWeight = 1; - long current = 1; - - for (double value : window) { - avg += value * current; - totalWeight += current; - current += 1; - } - return avg / totalWeight; - } - - /** - * Exponentially weighted (EWMA, Single exponential) moving avg - * - * @param window Window of values to compute movavg for - */ - private double ewma(Collection window) { - double avg = 0; - boolean first = true; - - for (double value : window) { - if (first) { - avg = value; - first = false; - } else { - avg = (value * alpha) + (avg * (1 - alpha)); - } - } - return avg; - } - - /** - * Holt-Linear (Double exponential) moving avg - * @param window Window of values to compute movavg for - */ - private double holt(Collection window) { - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - int counter = 0; - - double last; - for (double value : window) { - last = value; - if (counter == 0) { - s = value; - b = value - last; - } else { - s = alpha * value + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - } - - counter += 1; - last_s = s; - last_b = b; - } - - return s + (0 * b) ; - } - - /** - * Holt winters (triple exponential) moving avg - * @param window Window of values to compute movavg for - */ - private double holtWinters(Collection window) { - // Smoothed value - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - // Seasonal value - double[] seasonal = new double[window.size()]; - - double padding = seasonalityType.equals(HoltWintersModel.SeasonalityType.MULTIPLICATIVE) ? 0.0000000001 : 0; - - int counter = 0; - double[] vs = new double[window.size()]; - for (double v : window) { - vs[counter] = v + padding; - counter += 1; - } - - - // Initial level value is average of first season - // Calculate the slopes between first and second season for each period - for (int i = 0; i < period; i++) { - s += vs[i]; - b += (vs[i + period] - vs[i]) / period; - } - s /= period; - b /= period; - last_s = s; - - // Calculate first seasonal - if (Double.compare(s, 0.0) == 0 || Double.compare(s, -0.0) == 0) { - Arrays.fill(seasonal, 0.0); - } else { - for (int i = 0; i < period; i++) { - seasonal[i] = vs[i] / s; - } - } - - for (int i = period; i < vs.length; i++) { - if (seasonalityType.equals(HoltWintersModel.SeasonalityType.MULTIPLICATIVE)) { - s = alpha * (vs[i] / seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - } else { - s = alpha * (vs[i] - seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - } - - b = beta * (s - last_s) + (1 - beta) * last_b; - - if (seasonalityType.equals(HoltWintersModel.SeasonalityType.MULTIPLICATIVE)) { - seasonal[i] = gamma * (vs[i] / (last_s + last_b )) + (1 - gamma) * seasonal[i - period]; - } else { - seasonal[i] = gamma * (vs[i] - (last_s - last_b )) + (1 - gamma) * seasonal[i - period]; - } - - last_s = s; - last_b = b; - } - - int idx = window.size() - period + (0 % period); - - // TODO perhaps pad out seasonal to a power of 2 and use a mask instead of modulo? - if (seasonalityType.equals(HoltWintersModel.SeasonalityType.MULTIPLICATIVE)) { - return (s + (1 * b)) * seasonal[idx]; - } else { - return s + (1 * b) + seasonal[idx]; - } - } - - - /** - * test simple moving average on single value field - */ - public void testSimpleSingleValuedField() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts","_count") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - .subAggregation(movingAvg("movavg_values","the_metric") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.SIMPLE.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.SIMPLE.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - public void testLinearSingleValuedField() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(windowSize) - .modelBuilder(new LinearModel.LinearModelBuilder()) - .gapPolicy(gapPolicy)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new LinearModel.LinearModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.LINEAR.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.LINEAR.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - public void testEwmaSingleValuedField() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(windowSize) - .modelBuilder(new EwmaModel.EWMAModelBuilder().alpha(alpha)) - .gapPolicy(gapPolicy)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new EwmaModel.EWMAModelBuilder().alpha(alpha)) - .gapPolicy(gapPolicy)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.EWMA.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.EWMA.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - public void testHoltSingleValuedField() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(windowSize) - .modelBuilder(new HoltLinearModel.HoltLinearModelBuilder().alpha(alpha).beta(beta)) - .gapPolicy(gapPolicy)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new HoltLinearModel.HoltLinearModelBuilder().alpha(alpha).beta(beta)) - .gapPolicy(gapPolicy)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.HOLT.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.HOLT.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - - public void testHoltWintersValuedField() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(windowSize) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .alpha(alpha).beta(beta).gamma(gamma).period(period).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy) - .minimize(false)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .alpha(alpha).beta(beta).gamma(gamma).period(period).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy) - .minimize(false)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.HOLT_WINTERS.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.HOLT_WINTERS.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - public void testPredictNegativeKeysAtStart() { - - SearchResponse response = client() - .prepareSearch("neg_idx") - .setTypes("type") - .addAggregation( - histogram("histo") - .field(INTERVAL_FIELD) - .interval(1) - .subAggregation(avg("avg").field(VALUE_FIELD)) - .subAggregation( - movingAvg("movavg_values", "avg") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy).predict(5))) - .get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(25)); - - SimpleValue current = buckets.get(0).getAggregations().get("movavg_values"); - assertThat(current, nullValue()); - - for (int i = 1; i < 20; i++) { - Bucket bucket = buckets.get(i); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo(i - 10d)); - assertThat(bucket.getDocCount(), equalTo(1L)); - Avg avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.value(), equalTo(10d)); - SimpleValue movAvgAgg = bucket.getAggregations().get("movavg_values"); - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - } - - for (int i = 20; i < 25; i++) { - Bucket bucket = buckets.get(i); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo(i - 10d)); - assertThat(bucket.getDocCount(), equalTo(0L)); - Avg avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, nullValue()); - SimpleValue movAvgAgg = bucket.getAggregations().get("movavg_values"); - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - } - } - - public void testSizeZeroWindow() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(0) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - fail("MovingAvg should not accept a window that is zero"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("[window] must be a positive integer: [movavg_counts]")); - } - } - - public void testBadParent() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - range("histo").field(INTERVAL_FIELD).addRange(0, 10) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - fail("MovingAvg should not accept non-histogram as parent"); - - } catch (SearchPhaseExecutionException exception) { - // All good - } - } - - public void testNegativeWindow() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(-10) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - fail("MovingAvg should not accept a window that is negative"); - } catch (IllegalArgumentException e) { - assertThat(e.getMessage(), is("[window] must be a positive integer: [movavg_counts]")); - } - } - - public void testNoBucketsInHistogram() { - - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field("test").interval(interval) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat(buckets.size(), equalTo(0)); - } - - public void testNoBucketsInHistogramWithPredict() { - int numPredictions = randomIntBetween(1,10); - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field("test").interval(interval) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy) - .predict(numPredictions)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat(buckets.size(), equalTo(0)); - } - - public void testZeroPrediction() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(windowSize) - .modelBuilder(randomModelBuilder()) - .gapPolicy(gapPolicy) - .predict(0)) - ).get(); - fail("MovingAvg should not accept a prediction size that is zero"); - - } catch (IllegalArgumentException exception) { - // All Good - } - } - - public void testNegativePrediction() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(randomMetric("the_metric", VALUE_FIELD)) - .subAggregation(movingAvg("movavg_counts", "the_metric") - .window(windowSize) - .modelBuilder(randomModelBuilder()) - .gapPolicy(gapPolicy) - .predict(-10)) - ).get(); - fail("MovingAvg should not accept a prediction size that is negative"); - - } catch (IllegalArgumentException exception) { - // All Good - } - } - - @AwaitsFix(bugUrl = "https://github.com/elastic/elasticsearch/issues/34046") - public void testHoltWintersNotEnoughData() { - Client client = client(); - expectThrows(SearchPhaseExecutionException.class, () -> client.prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(10) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .alpha(alpha).beta(beta).gamma(gamma).period(20).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .alpha(alpha).beta(beta).gamma(gamma).period(20).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy)) - ).get()); - } - - public void testTwoMovAvgsWithPredictions() { - SearchResponse response = client() - .prepareSearch("double_predict") - .setTypes("type") - .addAggregation( - histogram("histo") - .field(INTERVAL_FIELD) - .interval(1) - .subAggregation(avg("avg").field(VALUE_FIELD)) - .subAggregation(derivative("deriv", "avg").gapPolicy(gapPolicy)) - .subAggregation( - movingAvg("avg_movavg", "avg") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy) - .predict(12)) - .subAggregation( - movingAvg("deriv_movavg", "deriv") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy) - .predict(12)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(24)); - - Bucket bucket = buckets.get(0); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo(0d)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - Avg avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.value(), equalTo(10d)); - - SimpleValue movAvgAgg = bucket.getAggregations().get("avg_movavg"); - assertThat(movAvgAgg, nullValue()); - - Derivative deriv = bucket.getAggregations().get("deriv"); - assertThat(deriv, nullValue()); - - SimpleValue derivMovAvg = bucket.getAggregations().get("deriv_movavg"); - assertThat(derivMovAvg, nullValue()); - - // Second bucket - bucket = buckets.get(1); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo(1d)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.value(), equalTo(10d)); - - deriv = bucket.getAggregations().get("deriv"); - assertThat(deriv, notNullValue()); - assertThat(deriv.value(), equalTo(0d)); - - movAvgAgg = bucket.getAggregations().get("avg_movavg"); - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - - derivMovAvg = bucket.getAggregations().get("deriv_movavg"); - assertThat(derivMovAvg, Matchers.nullValue()); // still null because of movavg delay - - for (int i = 2; i < 12; i++) { - bucket = buckets.get(i); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo((double) i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - - avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, notNullValue()); - assertThat(avgAgg.value(), equalTo(10d)); - - deriv = bucket.getAggregations().get("deriv"); - assertThat(deriv, notNullValue()); - assertThat(deriv.value(), equalTo(0d)); - - movAvgAgg = bucket.getAggregations().get("avg_movavg"); - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - - derivMovAvg = bucket.getAggregations().get("deriv_movavg"); - assertThat(derivMovAvg, notNullValue()); - assertThat(derivMovAvg.value(), equalTo(0d)); - } - - // Predictions - for (int i = 12; i < 24; i++) { - bucket = buckets.get(i); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo((double) i)); - assertThat(bucket.getDocCount(), equalTo(0L)); - - avgAgg = bucket.getAggregations().get("avg"); - assertThat(avgAgg, nullValue()); - - deriv = bucket.getAggregations().get("deriv"); - assertThat(deriv, nullValue()); - - movAvgAgg = bucket.getAggregations().get("avg_movavg"); - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - - derivMovAvg = bucket.getAggregations().get("deriv_movavg"); - assertThat(derivMovAvg, notNullValue()); - assertThat(derivMovAvg.value(), equalTo(0d)); - } - } - - @AwaitsFix(bugUrl="https://github.com/elastic/elasticsearch/issues/34046") - public void testBadModelParams() { - expectThrows(SearchPhaseExecutionException.class, () -> client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(10) - .modelBuilder(randomModelBuilder(100)) - .gapPolicy(gapPolicy)) - ).get()); - } - - public void testHoltWintersMinimization() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(windowSize) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .period(period).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy) - .minimize(true)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(windowSize) - .modelBuilder(new HoltWintersModel.HoltWintersModelBuilder() - .period(period).seasonalityType(seasonalityType)) - .gapPolicy(gapPolicy) - .minimize(true)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - - List expectedCounts = testValues.get(MovAvgType.HOLT_WINTERS.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.HOLT_WINTERS.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValueIter = expectedValues.iterator(); - - // The minimizer is stochastic, so just make sure all the values coming back aren't null - while (actualIter.hasNext()) { - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValueIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - SimpleValue countMovAvg = actual.getAggregations().get("movavg_counts"); - SimpleValue valuesMovAvg = actual.getAggregations().get("movavg_values"); - - if (expectedCount == null) { - //this bucket wasn't supposed to have a value (empty, skipped, etc), so - //movavg should be null too - assertThat(countMovAvg, nullValue()); - } else { - - // Note that we don't compare against the mock values, since those are assuming - // a non-minimized set of coefficients. Just check for not-nullness - assertThat(countMovAvg, notNullValue()); - } - - if (expectedValue == null) { - //this bucket wasn't supposed to have a value (empty, skipped, etc), so - //movavg should be null too - assertThat(valuesMovAvg, nullValue()); - } else { - - // Note that we don't compare against the mock values, since those are assuming - // a non-minimized set of coefficients. Just check for not-nullness - assertThat(valuesMovAvg, notNullValue()); - } - } - - } - - - /** - * If the minimizer is turned on, but there isn't enough data to minimize with, it will simply use - * the default settings. Which means our mock histo will match the generated result (which it won't - * if the minimizer is actually working, since the coefficients will be different and thus generate different - * data) - * - * We can simulate this by setting the window size == size of histo - */ - public void testMinimizeNotEnoughData() { - SearchResponse response = client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(numBuckets) - .modelBuilder(new HoltLinearModel.HoltLinearModelBuilder().alpha(alpha).beta(beta)) - .gapPolicy(gapPolicy) - .minimize(true)) - .subAggregation(movingAvg("movavg_values", "the_metric") - .window(numBuckets) - .modelBuilder(new HoltLinearModel.HoltLinearModelBuilder().alpha(alpha).beta(beta)) - .gapPolicy(gapPolicy) - .minimize(true)) - ).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(mockHisto.size())); - - List expectedCounts = testValues.get(MovAvgType.HOLT_BIG_MINIMIZE.name() + "_" + MetricTarget.COUNT.name()); - List expectedValues = testValues.get(MovAvgType.HOLT_BIG_MINIMIZE.name() + "_" + MetricTarget.VALUE.name()); - - Iterator actualIter = buckets.iterator(); - Iterator expectedBucketIter = mockHisto.iterator(); - Iterator expectedCountsIter = expectedCounts.iterator(); - Iterator expectedValuesIter = expectedValues.iterator(); - - while (actualIter.hasNext()) { - assertValidIterators(expectedBucketIter, expectedCountsIter, expectedValuesIter); - - Histogram.Bucket actual = actualIter.next(); - PipelineAggregationHelperTests.MockBucket expected = expectedBucketIter.next(); - Double expectedCount = expectedCountsIter.next(); - Double expectedValue = expectedValuesIter.next(); - - assertThat("keys do not match", ((Number) actual.getKey()).longValue(), equalTo(expected.key)); - assertThat("doc counts do not match", actual.getDocCount(), equalTo((long)expected.count)); - - assertBucketContents(actual, expectedCount, expectedValue); - } - } - - /** - * Only some models can be minimized, should throw exception for: simple, linear - */ - public void testCheckIfNonTunableCanBeMinimized() { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(numBuckets) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(gapPolicy) - .minimize(true)) - ).get(); - fail("Simple Model cannot be minimized, but an exception was not thrown"); - } catch (SearchPhaseExecutionException e) { - // All good - } - - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(numBuckets) - .modelBuilder(new LinearModel.LinearModelBuilder()) - .gapPolicy(gapPolicy) - .minimize(true)) - ).get(); - fail("Linear Model cannot be minimized, but an exception was not thrown"); - } catch (SearchPhaseExecutionException e) { - // all good - } - } - - /** - * These models are all minimizable, so they should not throw exceptions - */ - public void testCheckIfTunableCanBeMinimized() { - MovAvgModelBuilder[] builders = new MovAvgModelBuilder[]{ - new EwmaModel.EWMAModelBuilder(), - new HoltLinearModel.HoltLinearModelBuilder(), - new HoltWintersModel.HoltWintersModelBuilder() - }; - - for (MovAvgModelBuilder builder : builders) { - try { - client() - .prepareSearch("idx").setTypes("type") - .addAggregation( - histogram("histo").field(INTERVAL_FIELD).interval(interval) - .extendedBounds(0L, interval * (numBuckets - 1)) - .subAggregation(metric) - .subAggregation(movingAvg("movavg_counts", "_count") - .window(numBuckets) - .modelBuilder(builder) - .gapPolicy(gapPolicy) - .minimize(true)) - ).get(); - } catch (SearchPhaseExecutionException e) { - fail("Model [" + builder.toString() + "] can be minimized, but an exception was thrown"); - } - } - } - - public void testPredictWithNonEmptyBuckets() throws Exception { - - createIndex("predict_non_empty"); - BulkRequestBuilder bulkBuilder = client().prepareBulk().setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); - - for (int i = 0; i < 10; i++) { - bulkBuilder.add(client().prepareIndex("predict_non_empty", "type").setSource( - jsonBuilder().startObject().field(INTERVAL_FIELD, i) - .field(VALUE_FIELD, 10) - .field(VALUE_FIELD2, 10) - .endObject())); - } - for (int i = 10; i < 20; i++) { - // Extra so there is a bucket that only has second field - bulkBuilder.add(client().prepareIndex("predict_non_empty", "type").setSource( - jsonBuilder().startObject().field(INTERVAL_FIELD, i).field(VALUE_FIELD2, 10).endObject())); - } - - bulkBuilder.get(); - ensureSearchable(); - - SearchResponse response = client() - .prepareSearch("predict_non_empty") - .setTypes("type") - .addAggregation( - histogram("histo") - .field(INTERVAL_FIELD) - .interval(1) - .subAggregation(max("max").field(VALUE_FIELD)) - .subAggregation(max("max2").field(VALUE_FIELD2)) - .subAggregation( - movingAvg("movavg_values", "max") - .window(windowSize) - .modelBuilder(new SimpleModel.SimpleModelBuilder()) - .gapPolicy(BucketHelpers.GapPolicy.SKIP).predict(5))).get(); - - assertSearchResponse(response); - - Histogram histo = response.getAggregations().get("histo"); - assertThat(histo, notNullValue()); - assertThat(histo.getName(), equalTo("histo")); - List buckets = histo.getBuckets(); - assertThat("Size of buckets array is not correct.", buckets.size(), equalTo(20)); - - SimpleValue current = buckets.get(0).getAggregations().get("movavg_values"); - assertThat(current, nullValue()); - - for (int i = 1; i < 20; i++) { - Bucket bucket = buckets.get(i); - assertThat(bucket, notNullValue()); - assertThat(bucket.getKey(), equalTo((double)i)); - assertThat(bucket.getDocCount(), equalTo(1L)); - SimpleValue movAvgAgg = bucket.getAggregations().get("movavg_values"); - if (i < 15) { - assertThat(movAvgAgg, notNullValue()); - assertThat(movAvgAgg.value(), equalTo(10d)); - } else { - assertThat(movAvgAgg, nullValue()); - } - } - } - - private void assertValidIterators(Iterator expectedBucketIter, Iterator expectedCountsIter, Iterator expectedValuesIter) { - if (!expectedBucketIter.hasNext()) { - fail("`expectedBucketIter` iterator ended before `actual` iterator, size mismatch"); - } - if (!expectedCountsIter.hasNext()) { - fail("`expectedCountsIter` iterator ended before `actual` iterator, size mismatch"); - } - if (!expectedValuesIter.hasNext()) { - fail("`expectedValuesIter` iterator ended before `actual` iterator, size mismatch"); - } - } - - private void assertBucketContents(Histogram.Bucket actual, Double expectedCount, Double expectedValue) { - // This is a gap bucket - SimpleValue countMovAvg = actual.getAggregations().get("movavg_counts"); - if (expectedCount == null) { - assertThat("[_count] movavg is not null", countMovAvg, nullValue()); - } else if (Double.isNaN(expectedCount)) { - assertThat("[_count] movavg should be NaN, but is ["+countMovAvg.value()+"] instead", - countMovAvg.value(), equalTo(Double.NaN)); - } else { - assertThat("[_count] movavg is null", countMovAvg, notNullValue()); - assertEquals("[_count] movavg does not match expected [" + countMovAvg.value() + " vs " + expectedCount + "]", - countMovAvg.value(), expectedCount, 0.1 * Math.abs(countMovAvg.value())); - } - - // This is a gap bucket - SimpleValue valuesMovAvg = actual.getAggregations().get("movavg_values"); - if (expectedValue == null) { - assertThat("[value] movavg is not null", valuesMovAvg, Matchers.nullValue()); - } else if (Double.isNaN(expectedValue)) { - assertThat("[value] movavg should be NaN, but is ["+valuesMovAvg.value()+"] instead", - valuesMovAvg.value(), equalTo(Double.NaN)); - } else { - assertThat("[value] movavg is null", valuesMovAvg, notNullValue()); - assertEquals("[value] movavg does not match expected [" + valuesMovAvg.value() + " vs " + expectedValue + "]", - valuesMovAvg.value(), expectedValue, 0.1 * Math.abs(valuesMovAvg.value())); - } - } - - private MovAvgModelBuilder randomModelBuilder() { - return randomModelBuilder(0); - } - - private MovAvgModelBuilder randomModelBuilder(double padding) { - int rand = randomIntBetween(0,3); - - // HoltWinters is excluded from random generation, because it's "cold start" behavior makes - // randomized testing too tricky. Should probably add dedicated, randomized tests just for HoltWinters, - // which can compensate for the idiosyncrasies - switch (rand) { - case 0: - return new SimpleModel.SimpleModelBuilder(); - case 1: - return new LinearModel.LinearModelBuilder(); - case 2: - return new EwmaModel.EWMAModelBuilder().alpha(alpha + padding); - case 3: - return new HoltLinearModel.HoltLinearModelBuilder().alpha(alpha + padding).beta(beta + padding); - default: - return new SimpleModel.SimpleModelBuilder(); - } - } - - private ValuesSourceAggregationBuilder> randomMetric(String name, String field) { - int rand = randomIntBetween(0,3); - - switch (rand) { - case 0: - return min(name).field(field); - case 2: - return max(name).field(field); - case 3: - return avg(name).field(field); - default: - return avg(name).field(field); - } - } - -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgTests.java deleted file mode 100644 index 490d1f22ff329..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgTests.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.search.aggregations.BasePipelineAggregationTestCase; -import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.TestAggregatorFactory; -import org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy; -import org.elasticsearch.search.aggregations.pipeline.HoltWintersModel.SeasonalityType; - -import java.io.IOException; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -public class MovAvgTests extends BasePipelineAggregationTestCase { - - @Override - protected MovAvgPipelineAggregationBuilder createTestAggregatorFactory() { - String name = randomAlphaOfLengthBetween(3, 20); - String bucketsPath = randomAlphaOfLengthBetween(3, 20); - MovAvgPipelineAggregationBuilder factory = new MovAvgPipelineAggregationBuilder(name, bucketsPath); - if (randomBoolean()) { - factory.format(randomAlphaOfLengthBetween(1, 10)); - } - if (randomBoolean()) { - factory.gapPolicy(randomFrom(GapPolicy.values())); - } - if (randomBoolean()) { - switch (randomInt(4)) { - case 0: - factory.modelBuilder(new SimpleModel.SimpleModelBuilder()); - factory.window(randomIntBetween(1, 100)); - break; - case 1: - factory.modelBuilder(new LinearModel.LinearModelBuilder()); - factory.window(randomIntBetween(1, 100)); - break; - case 2: - if (randomBoolean()) { - factory.modelBuilder(new EwmaModel.EWMAModelBuilder()); - factory.window(randomIntBetween(1, 100)); - } else { - factory.modelBuilder(new EwmaModel.EWMAModelBuilder().alpha(randomDouble())); - factory.window(randomIntBetween(1, 100)); - } - break; - case 3: - if (randomBoolean()) { - factory.modelBuilder(new HoltLinearModel.HoltLinearModelBuilder()); - factory.window(randomIntBetween(1, 100)); - } else { - factory.modelBuilder(new HoltLinearModel.HoltLinearModelBuilder().alpha(randomDouble()).beta(randomDouble())); - factory.window(randomIntBetween(1, 100)); - } - break; - case 4: - default: - if (randomBoolean()) { - factory.modelBuilder(new HoltWintersModel.HoltWintersModelBuilder()); - factory.window(randomIntBetween(2, 100)); - } else { - int period = randomIntBetween(1, 100); - factory.modelBuilder( - new HoltWintersModel.HoltWintersModelBuilder().alpha(randomDouble()).beta(randomDouble()).gamma(randomDouble()) - .period(period).seasonalityType(randomFrom(SeasonalityType.values())).pad(randomBoolean())); - factory.window(randomIntBetween(2 * period, 200 * period)); - } - break; - } - } - factory.predict(randomIntBetween(1, 50)); - if (factory.model().canBeMinimized() && randomBoolean()) { - factory.minimize(randomBoolean()); - } - return factory; - } - - @Override - public void testFromXContent() throws IOException { - super.testFromXContent(); - assertWarnings("The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation."); - } - - public void testDefaultParsing() throws Exception { - MovAvgPipelineAggregationBuilder expected = new MovAvgPipelineAggregationBuilder("commits_moving_avg", "commits"); - String json = "{" + - " \"commits_moving_avg\": {" + - " \"moving_avg\": {" + - " \"buckets_path\": \"commits\"" + - " }" + - " }" + - "}"; - PipelineAggregationBuilder newAgg = parse(createParser(JsonXContent.jsonXContent, json)); - assertWarnings("The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation."); - assertNotSame(newAgg, expected); - assertEquals(expected, newAgg); - assertEquals(expected.hashCode(), newAgg.hashCode()); - } - - /** - * The validation should verify the parent aggregation is allowed. - */ - public void testValidate() throws IOException { - final Set aggBuilders = new HashSet<>(); - aggBuilders.add(createTestAggregatorFactory()); - - final MovAvgPipelineAggregationBuilder builder = new MovAvgPipelineAggregationBuilder("name", "valid"); - builder.validate(PipelineAggregationHelperTests.getRandomSequentiallyOrderedParentAgg(), Collections.emptySet(), aggBuilders); - } - - /** - * The validation should throw an IllegalArgumentException, since parent - * aggregation is not a type of HistogramAggregatorFactory, - * DateHistogramAggregatorFactory or AutoDateHistogramAggregatorFactory. - */ - public void testValidateException() throws IOException { - final Set aggBuilders = new HashSet<>(); - aggBuilders.add(createTestAggregatorFactory()); - TestAggregatorFactory parentFactory = TestAggregatorFactory.createInstance(); - - final MovAvgPipelineAggregationBuilder builder = new MovAvgPipelineAggregationBuilder("name", "invalid_agg>metric"); - IllegalStateException ex = expectThrows(IllegalStateException.class, - () -> builder.validate(parentFactory, Collections.emptySet(), aggBuilders)); - assertEquals("moving_avg aggregation [name] must have a histogram, date_histogram or auto_date_histogram as parent", - ex.getMessage()); - } -} diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgUnitTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgUnitTests.java deleted file mode 100644 index 38ed1c1dc3f24..0000000000000 --- a/server/src/test/java/org/elasticsearch/search/aggregations/pipeline/MovAvgUnitTests.java +++ /dev/null @@ -1,630 +0,0 @@ -/* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.elasticsearch.search.aggregations.pipeline; - -import org.elasticsearch.common.collect.EvictingQueue; -import org.elasticsearch.search.aggregations.pipeline.EwmaModel; -import org.elasticsearch.search.aggregations.pipeline.HoltLinearModel; -import org.elasticsearch.search.aggregations.pipeline.HoltWintersModel; -import org.elasticsearch.search.aggregations.pipeline.LinearModel; -import org.elasticsearch.search.aggregations.pipeline.MovAvgModel; -import org.elasticsearch.search.aggregations.pipeline.SimpleModel; -import org.elasticsearch.test.ESTestCase; - -import java.text.ParseException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static org.hamcrest.Matchers.equalTo; - -public class MovAvgUnitTests extends ESTestCase { - public void testSimpleMovAvgModel() { - MovAvgModel model = new SimpleModel(); - - int numValues = randomIntBetween(1, 100); - int windowSize = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < numValues; i++) { - - double randValue = randomDouble(); - double expected = 0; - - if (i == 0) { - window.offer(randValue); - continue; - } - - for (double value : window) { - expected += value; - } - expected /= window.size(); - - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - window.offer(randValue); - } - } - - public void testSimplePredictionModel() { - MovAvgModel model = new SimpleModel(); - - int windowSize = randomIntBetween(1, 50); - int numPredictions = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - - double expected[] = new double[numPredictions]; - double t = 0; - for (double value : window) { - t += value; - } - t /= window.size(); - Arrays.fill(expected, t); - - for (int i = 0; i < numPredictions; i++) { - assertThat(Double.compare(expected[i], actual[i]), equalTo(0)); - } - } - - public void testLinearMovAvgModel() { - MovAvgModel model = new LinearModel(); - - int numValues = randomIntBetween(1, 100); - int windowSize = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < numValues; i++) { - double randValue = randomDouble(); - - if (i == 0) { - window.offer(randValue); - continue; - } - - double avg = 0; - long totalWeight = 1; - long current = 1; - - for (double value : window) { - avg += value * current; - totalWeight += current; - current += 1; - } - double expected = avg / totalWeight; - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - window.offer(randValue); - } - } - - public void testLinearPredictionModel() { - MovAvgModel model = new LinearModel(); - - int windowSize = randomIntBetween(1, 50); - int numPredictions = randomIntBetween(1,50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - double expected[] = new double[numPredictions]; - - double avg = 0; - long totalWeight = 1; - long current = 1; - - for (double value : window) { - avg += value * current; - totalWeight += current; - current += 1; - } - avg = avg / totalWeight; - Arrays.fill(expected, avg); - - for (int i = 0; i < numPredictions; i++) { - assertThat(Double.compare(expected[i], actual[i]), equalTo(0)); - } - } - - public void testEWMAMovAvgModel() { - double alpha = randomDouble(); - MovAvgModel model = new EwmaModel(alpha); - - int numValues = randomIntBetween(1, 100); - int windowSize = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < numValues; i++) { - double randValue = randomDouble(); - - if (i == 0) { - window.offer(randValue); - continue; - } - - double avg = 0; - boolean first = true; - - for (double value : window) { - if (first) { - avg = value; - first = false; - } else { - avg = (value * alpha) + (avg * (1 - alpha)); - } - } - double expected = avg; - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - window.offer(randValue); - } - } - - public void testEWMAPredictionModel() { - double alpha = randomDouble(); - MovAvgModel model = new EwmaModel(alpha); - - int windowSize = randomIntBetween(1, 50); - int numPredictions = randomIntBetween(1,50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - double expected[] = new double[numPredictions]; - - double avg = 0; - boolean first = true; - - for (double value : window) { - if (first) { - avg = value; - first = false; - } else { - avg = (value * alpha) + (avg * (1 - alpha)); - } - } - Arrays.fill(expected, avg); - - for (int i = 0; i < numPredictions; i++) { - assertThat(Double.compare(expected[i], actual[i]), equalTo(0)); - } - } - - public void testHoltLinearMovAvgModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - MovAvgModel model = new HoltLinearModel(alpha, beta); - - int numValues = randomIntBetween(1, 100); - int windowSize = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < numValues; i++) { - double randValue = randomDouble(); - - if (i == 0) { - window.offer(randValue); - continue; - } - - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - int counter = 0; - - double last; - for (double value : window) { - last = value; - if (counter == 0) { - s = value; - b = value - last; - } else { - s = alpha * value + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - } - - counter += 1; - last_s = s; - last_b = b; - } - - double expected = s + (0 * b) ; - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - window.offer(randValue); - } - } - - public void testHoltLinearPredictionModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - MovAvgModel model = new HoltLinearModel(alpha, beta); - - int windowSize = randomIntBetween(1, 50); - int numPredictions = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - double expected[] = new double[numPredictions]; - - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - int counter = 0; - - double last; - for (double value : window) { - last = value; - if (counter == 0) { - s = value; - b = value - last; - } else { - s = alpha * value + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - } - - counter += 1; - last_s = s; - last_b = b; - } - - for (int i = 0; i < numPredictions; i++) { - expected[i] = s + (i * b); - assertThat(Double.compare(expected[i], actual[i]), equalTo(0)); - } - } - - public void testHoltWintersMultiplicativePadModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - double gamma = randomDouble(); - int period = randomIntBetween(1,10); - MovAvgModel model = new HoltWintersModel(alpha, beta, gamma, period, HoltWintersModel.SeasonalityType.MULTIPLICATIVE, true); - - int windowSize = randomIntBetween(period * 2, 50); // HW requires at least two periods of data - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - - // Smoothed value - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - // Seasonal value - double[] seasonal = new double[windowSize]; - - int counter = 0; - double[] vs = new double[windowSize]; - for (double v : window) { - vs[counter] = v + 0.0000000001; - counter += 1; - } - - - // Initial level value is average of first season - // Calculate the slopes between first and second season for each period - for (int i = 0; i < period; i++) { - s += vs[i]; - b += (vs[i + period] - vs[i]) / period; - } - s /= period; - b /= period; - last_s = s; - - // Calculate first seasonal - if (Double.compare(s, 0.0) == 0 || Double.compare(s, -0.0) == 0) { - Arrays.fill(seasonal, 0.0); - } else { - for (int i = 0; i < period; i++) { - seasonal[i] = vs[i] / s; - } - } - - for (int i = period; i < vs.length; i++) { - s = alpha * (vs[i] / seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - - seasonal[i] = gamma * (vs[i] / (last_s + last_b )) + (1 - gamma) * seasonal[i - period]; - last_s = s; - last_b = b; - } - - int idx = window.size() - period + (0 % period); - double expected = (s + (1 * b)) * seasonal[idx]; - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - } - - public void testHoltWintersMultiplicativePadPredictionModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - double gamma = randomDouble(); - int period = randomIntBetween(1,10); - MovAvgModel model = new HoltWintersModel(alpha, beta, gamma, period, HoltWintersModel.SeasonalityType.MULTIPLICATIVE, true); - - int windowSize = randomIntBetween(period * 2, 50); // HW requires at least two periods of data - int numPredictions = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - double expected[] = new double[numPredictions]; - - // Smoothed value - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - // Seasonal value - double[] seasonal = new double[windowSize]; - - int counter = 0; - double[] vs = new double[windowSize]; - for (double v : window) { - vs[counter] = v + 0.0000000001; - counter += 1; - } - - - // Initial level value is average of first season - // Calculate the slopes between first and second season for each period - for (int i = 0; i < period; i++) { - s += vs[i]; - b += (vs[i + period] - vs[i]) / period; - } - s /= period; - b /= period; - last_s = s; - - // Calculate first seasonal - if (Double.compare(s, 0.0) == 0 || Double.compare(s, -0.0) == 0) { - Arrays.fill(seasonal, 0.0); - } else { - for (int i = 0; i < period; i++) { - seasonal[i] = vs[i] / s; - } - } - - for (int i = period; i < vs.length; i++) { - s = alpha * (vs[i] / seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - - seasonal[i] = gamma * (vs[i] / (last_s + last_b )) + (1 - gamma) * seasonal[i - period]; - last_s = s; - last_b = b; - } - - - for (int i = 1; i <= numPredictions; i++) { - int idx = window.size() - period + ((i - 1) % period); - expected[i-1] = (s + (i * b)) * seasonal[idx]; - assertThat(Double.compare(expected[i-1], actual[i-1]), equalTo(0)); - } - - } - - public void testHoltWintersAdditiveModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - double gamma = randomDouble(); - int period = randomIntBetween(1,10); - MovAvgModel model = new HoltWintersModel(alpha, beta, gamma, period, HoltWintersModel.SeasonalityType.ADDITIVE, false); - - int windowSize = randomIntBetween(period * 2, 50); // HW requires at least two periods of data - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - - // Smoothed value - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - // Seasonal value - double[] seasonal = new double[windowSize]; - - int counter = 0; - double[] vs = new double[windowSize]; - for (double v : window) { - vs[counter] = v; - counter += 1; - } - - // Initial level value is average of first season - // Calculate the slopes between first and second season for each period - for (int i = 0; i < period; i++) { - s += vs[i]; - b += (vs[i + period] - vs[i]) / period; - } - s /= period; - b /= period; - last_s = s; - - // Calculate first seasonal - if (Double.compare(s, 0.0) == 0 || Double.compare(s, -0.0) == 0) { - Arrays.fill(seasonal, 0.0); - } else { - for (int i = 0; i < period; i++) { - seasonal[i] = vs[i] / s; - } - } - - for (int i = period; i < vs.length; i++) { - s = alpha * (vs[i] - seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - - seasonal[i] = gamma * (vs[i] - (last_s - last_b )) + (1 - gamma) * seasonal[i - period]; - last_s = s; - last_b = b; - } - - int idx = window.size() - period + (0 % period); - double expected = s + (1 * b) + seasonal[idx]; - double actual = model.next(window); - assertThat(Double.compare(expected, actual), equalTo(0)); - } - - public void testHoltWintersAdditivePredictionModel() { - double alpha = randomDouble(); - double beta = randomDouble(); - double gamma = randomDouble(); - int period = randomIntBetween(1,10); - MovAvgModel model = new HoltWintersModel(alpha, beta, gamma, period, HoltWintersModel.SeasonalityType.ADDITIVE, false); - - int windowSize = randomIntBetween(period * 2, 50); // HW requires at least two periods of data - int numPredictions = randomIntBetween(1, 50); - - EvictingQueue window = new EvictingQueue<>(windowSize); - for (int i = 0; i < windowSize; i++) { - window.offer(randomDouble()); - } - double actual[] = model.predict(window, numPredictions); - double expected[] = new double[numPredictions]; - - // Smoothed value - double s = 0; - double last_s = 0; - - // Trend value - double b = 0; - double last_b = 0; - - // Seasonal value - double[] seasonal = new double[windowSize]; - - int counter = 0; - double[] vs = new double[windowSize]; - for (double v : window) { - vs[counter] = v; - counter += 1; - } - - // Initial level value is average of first season - // Calculate the slopes between first and second season for each period - for (int i = 0; i < period; i++) { - s += vs[i]; - b += (vs[i + period] - vs[i]) / period; - } - s /= period; - b /= period; - last_s = s; - - // Calculate first seasonal - if (Double.compare(s, 0.0) == 0 || Double.compare(s, -0.0) == 0) { - Arrays.fill(seasonal, 0.0); - } else { - for (int i = 0; i < period; i++) { - seasonal[i] = vs[i] / s; - } - } - - for (int i = period; i < vs.length; i++) { - s = alpha * (vs[i] - seasonal[i - period]) + (1.0d - alpha) * (last_s + last_b); - b = beta * (s - last_s) + (1 - beta) * last_b; - - seasonal[i] = gamma * (vs[i] - (last_s - last_b )) + (1 - gamma) * seasonal[i - period]; - last_s = s; - last_b = b; - } - - for (int i = 1; i <= numPredictions; i++) { - int idx = window.size() - period + ((i - 1) % period); - expected[i-1] = s + (i * b) + seasonal[idx]; - assertThat(Double.compare(expected[i-1], actual[i-1]), equalTo(0)); - } - - } - - public void testNumericValidation() { - List parsers = new ArrayList<>(3); - - // Simple and Linear don't have any settings to test - parsers.add(EwmaModel.PARSER); - parsers.add(HoltWintersModel.PARSER); - parsers.add(HoltLinearModel.PARSER); - - Object[] values = {(byte)1, 1, 1L, (short)1, (double)1}; - Map settings = new HashMap<>(2); - - for (MovAvgModel.AbstractModelParser parser : parsers) { - for (Object v : values) { - settings.put("alpha", v); - - try { - parser.parse(settings, "pipeline", 10); - } catch (ParseException e) { - fail(parser + " parser should not have thrown SearchParseException while parsing [" + - v.getClass().getSimpleName() +"]"); - } - } - } - - for (MovAvgModel.AbstractModelParser parser : parsers) { - settings.put("alpha", "abc"); - settings.put("beta", "abc"); - settings.put("gamma", "abc"); - - try { - parser.parse(settings, "pipeline", 10); - } catch (ParseException e) { - //all good - continue; - } - - fail(parser + " parser should have thrown SearchParseException while parsing [String]"); - } - } -}