From a99c7db6dc2430b4b0fdfbcffcb682e7e815bd8a Mon Sep 17 00:00:00 2001 From: vanjaftn Date: Mon, 13 Nov 2023 14:37:29 +0100 Subject: [PATCH 1/4] Support Extended stats aggregation --- .../elastic_aggregation_extended_stats.md | 41 ++++++++ .../aggregations/elastic_aggregation_stats.md | 2 +- .../zio/elasticsearch/HttpExecutorSpec.scala | 42 ++++++++ .../elasticsearch/ElasticAggregation.scala | 34 +++++++ .../aggregation/Aggregations.scala | 42 ++++++++ .../response/AggregationResponse.scala | 84 ++++++++++++++++ .../SearchWithAggregationsResponse.scala | 6 +- .../scala/zio/elasticsearch/package.scala | 12 +++ .../result/AggregationResult.scala | 17 ++++ .../elasticsearch/result/ElasticResult.scala | 3 + .../ElasticAggregationSpec.scala | 96 +++++++++++++++++++ website/sidebars.js | 1 + 12 files changed, 377 insertions(+), 3 deletions(-) create mode 100644 docs/overview/aggregations/elastic_aggregation_extended_stats.md diff --git a/docs/overview/aggregations/elastic_aggregation_extended_stats.md b/docs/overview/aggregations/elastic_aggregation_extended_stats.md new file mode 100644 index 000000000..2e0413183 --- /dev/null +++ b/docs/overview/aggregations/elastic_aggregation_extended_stats.md @@ -0,0 +1,41 @@ +--- +id: elastic_aggregation_extended_stats +title: "Extended stats Aggregation" +--- + +The `Extended stats` aggregation is a multi-value metrics aggregation that provides statistical information (count, sum, min, max, average, sum od squares, variance and std deviation of a field) over numeric values extracted from the aggregated documents. +The `Extended stats` aggregation is an extended version of the [`Stats`](https://lambdaworks.github.io/zio-elasticsearch/overview/aggregations/elastic_aggregation_stats) aggregation. + +In order to use the `Extended stats` aggregation import the following: +```scala +import zio.elasticsearch.aggregation.ExtendedStatsAggregation +import zio.elasticsearch.ElasticAggregation.extendedStatsAggregation +``` + +You can create a `Extended stats` aggregation using the `extendedStatsAggregation` method this way: +```scala +val aggregation: ExtendedStatsAggregation = extendedStatsAggregation(name = "extendedStatsAggregation", field = "intField") +``` + +You can create a [type-safe](https://lambdaworks.github.io/zio-elasticsearch/overview/overview_zio_prelude_schema) `Extended stats` aggregation using the `extendedStatsAggregation` method this way: +```scala +// Document.intField must be number value, because of Stats aggregation +val aggregation: ExtendedStatsAggregation = extendedStatsAggregation(name = "extendedStatsAggregation", field = Document.intField) +``` + +If you want to change the `missing` parameter, you can use `missing` method: +```scala +val aggregationWithMissing: ExtendedStatsAggregation = extendedStatsAggregation(name = "extendedStatsAggregation", field = Document.intField).missing(10.0) +``` + +If you want to change the `sigma` parameter, you can use `sigma` method: +```scala +val aggregationWithSigma: ExtendedStatsAggregation = extendedStatsAggregation(name = "extendedStatsAggregation", field = Document.intField).sigma(3.0) +``` + +If you want to add aggregation (on the same level), you can use `withAgg` method: +```scala +val multipleAggregations: MultipleAggregations = extendedStatsAggregation(name = "extendedStatsAggregation", field = Document.intField).withAgg(extendedStatsAggregation(name = "extendedStatsAggregation2", field = Document.doubleField)) +``` + +You can find more information about `Extended stats` aggregation [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-extendedstats-aggregation.html#search-aggregations-metrics-extendedstats-aggregation). diff --git a/docs/overview/aggregations/elastic_aggregation_stats.md b/docs/overview/aggregations/elastic_aggregation_stats.md index 23f037ede..255ee634a 100644 --- a/docs/overview/aggregations/elastic_aggregation_stats.md +++ b/docs/overview/aggregations/elastic_aggregation_stats.md @@ -3,7 +3,7 @@ id: elastic_aggregation_stats title: "Stats Aggregation" --- -The `Stats` aggregation is a multi-value metrics aggregation that provides statistical information (count, sum, min, max, and average of a field) over numeric values extracted from the aggregated documents. +The `Stats` aggregation is a multi-value metrics aggregation that provides statistical information (count, sum, min, max and average of a field) over numeric values extracted from the aggregated documents. In order to use the `Stats` aggregation import the following: ```scala diff --git a/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala index 358324ec2..f85f07294 100644 --- a/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -104,6 +104,48 @@ object HttpExecutorSpec extends IntegrationSpec { Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ), + test("aggregate using extended stats aggregation ttt") { + checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument) { + (firstDocumentId, firstDocument, secondDocumentId, secondDocument) => + for { + _ <- Executor.execute(ElasticRequest.deleteByQuery(firstSearchIndex, matchAll)) + _ <- Executor.execute( + ElasticRequest + .upsert[TestDocument](firstSearchIndex, firstDocumentId, firstDocument.copy(intField = 100)) + ) + _ <- Executor.execute( + ElasticRequest + .upsert[TestDocument](firstSearchIndex, secondDocumentId, secondDocument.copy(intField = 50)) + .refreshTrue + ) + aggregation = extendedStatsAggregation(name = "aggregation", field = TestDocument.intField).sigma(3) + aggsRes <- + Executor + .execute(ElasticRequest.aggregate(selectors = firstSearchIndex, aggregation = aggregation)) + .asExtendedStatsAggregation("aggregation") + } yield assert(aggsRes.head.count)(equalTo(2)) && + assert(aggsRes.head.min)(equalTo(50.0)) && + assert(aggsRes.head.max)(equalTo(100.0)) && + assert(aggsRes.head.avg)(equalTo(75.0)) && + assert(aggsRes.head.sum)(equalTo(150.0)) && + assert(aggsRes.head.sumOfSquares)(equalTo(12500.0)) && + assert(aggsRes.head.variance)(equalTo(625.0)) && + assert(aggsRes.head.variancePopulation)(equalTo(625.0)) && + assert(aggsRes.head.varianceSampling)(equalTo(1250.0)) && + assert(aggsRes.head.stdDeviation)(equalTo(25.0)) && + assert(aggsRes.head.stdDeviationPopulation)(equalTo(25.0)) && + assert(aggsRes.head.stdDeviationSampling)(equalTo(35.35533905932738)) && + assert(aggsRes.head.stdDeviationBounds.upper)(equalTo(150.0)) && + assert(aggsRes.head.stdDeviationBounds.lower)(equalTo(0.0)) && + assert(aggsRes.head.stdDeviationBounds.upper_population)(equalTo(150.0)) && + assert(aggsRes.head.stdDeviationBounds.lower_population)(equalTo(0.0)) && + assert(aggsRes.head.stdDeviationBounds.upper_sampling)(equalTo(181.06601717798213)) && + assert(aggsRes.head.stdDeviationBounds.lower_sampling)(equalTo(-31.066017177982133)) + } + } @@ around( + Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), + Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie + ), test("aggregate using max aggregation") { val expectedResponse = ("aggregationInt", MaxAggregationResult(value = 20.0)) checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument) { diff --git a/modules/library/src/main/scala/zio/elasticsearch/ElasticAggregation.scala b/modules/library/src/main/scala/zio/elasticsearch/ElasticAggregation.scala index 7d4e27021..bd3aea156 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/ElasticAggregation.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/ElasticAggregation.scala @@ -113,6 +113,38 @@ object ElasticAggregation { final def cardinalityAggregation(name: String, field: String): CardinalityAggregation = Cardinality(name = name, field = field, missing = None) + /** + * Constructs a type-safe instance of [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]] using the specified + * parameters. + * + * @param name + * aggregation name + * @param field + * the type-safe field for which extended stats aggregation will be executed + * @tparam A + * expected number type + * @return + * an instance of [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]] that represents extended stats + * aggregation to be performed. + */ + final def extendedStatsAggregation[A: Numeric](name: String, field: Field[_, A]): ExtendedStatsAggregation = + ExtendedStats(name = name, field = field.toString, missing = None, sigma = None) + + /** + * Constructs an instance of [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]] using the specified + * parameters. + * + * @param name + * aggregation name + * @param field + * the field for which extended stats aggregation will be executed + * @return + * an instance of [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]] that represents extended stats + * aggregation to be performed. + */ + final def extendedStatsAggregation(name: String, field: String): ExtendedStatsAggregation = + ExtendedStats(name = name, field = field, missing = None, sigma = None) + /** * Constructs a type-safe instance of [[zio.elasticsearch.aggregation.MaxAggregation]] using the specified parameters. * @@ -246,6 +278,8 @@ object ElasticAggregation { * the name of the aggregation * @param field * the type-safe field for which stats aggregation will be executed + * @tparam A + * expected number type * @return * an instance of [[zio.elasticsearch.aggregation.StatsAggregation]] that represents stats aggregation to be * performed. diff --git a/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala b/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala index 4dbe6cadd..006516815 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala @@ -143,6 +143,48 @@ private[elasticsearch] final case class Cardinality(name: String, field: String, } } +sealed trait ExtendedStatsAggregation + extends SingleElasticAggregation + with HasMissing[ExtendedStatsAggregation] + with WithAgg { + + /** + * Sets the `sigma` parameter for the [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]]. The`sigma` parameter + * controls how many standard deviations plus/minus from the mean should std_deviation_bounds object display + * + * @param value + * the value to use for sigma parameter + * @return + * an instance of the [[zio.elasticsearch.aggregation.ElasticAggregation]] enriched with the `sigma` parameter. + */ + def sigma(value: Double): ExtendedStatsAggregation +} + +private[elasticsearch] final case class ExtendedStats( + name: String, + field: String, + missing: Option[Double], + sigma: Option[Double] +) extends ExtendedStatsAggregation { self => + + def missing(value: Double): ExtendedStatsAggregation = + self.copy(missing = Some(value)) + + def sigma(value: Double): ExtendedStatsAggregation = + self.copy(sigma = Some(value)) + + def withAgg(agg: SingleElasticAggregation): MultipleAggregations = + multipleAggregations.aggregations(self, agg) + + private[elasticsearch] def toJson: Json = { + val missingJson: Json = missing.fold(Obj())(m => Obj("missing" -> m.toJson)) + + val sigmaJson: Json = sigma.fold(Obj())(m => Obj("sigma" -> m.toJson)) + + Obj(name -> Obj("extended_stats" -> (Obj("field" -> field.toJson) merge missingJson merge sigmaJson))) + } +} + sealed trait MaxAggregation extends SingleElasticAggregation with HasMissing[MaxAggregation] with WithAgg private[elasticsearch] final case class Max(name: String, field: String, missing: Option[Double]) diff --git a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala index fd6adf43a..bb8d1d757 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala @@ -31,6 +31,36 @@ object AggregationResponse { AvgAggregationResult(value) case CardinalityAggregationResponse(value) => CardinalityAggregationResult(value) + case ExtendedStatsAggregationResponse( + count, + min, + max, + avg, + sum, + sumOfSquares, + variance, + variancePopulation, + varianceSampling, + stdDeviation, + stdDeviationPopulation, + stdDeviationSampling, + stdDeviationBounds + ) => + ExtendedStatsAggregationResult( + count, + min, + max, + avg, + sum, + sumOfSquares, + variance, + variancePopulation, + varianceSampling, + stdDeviation, + stdDeviationPopulation, + stdDeviationSampling, + stdDeviationBounds + ) case MaxAggregationResponse(value) => MaxAggregationResult(value) case MinAggregationResponse(value) => @@ -77,6 +107,38 @@ private[elasticsearch] object CardinalityAggregationResponse { DeriveJsonDecoder.gen[CardinalityAggregationResponse] } +case class StdDeviationBounds private[elasticsearch] ( + upper: Double, + lower: Double, + upper_population: Double, + lower_population: Double, + upper_sampling: Double, + lower_sampling: Double +) + +private[elasticsearch] final case class ExtendedStatsAggregationResponse( + count: Int, + min: Double, + max: Double, + avg: Double, + sum: Double, + sum_of_squares: Double, + variance: Double, + variance_population: Double, + variance_sampling: Double, + std_deviation: Double, + std_deviation_population: Double, + std_deviation_sampling: Double, + std_deviation_bounds: StdDeviationBounds +) extends AggregationResponse + +private[elasticsearch] object ExtendedStatsAggregationResponse { + implicit val boundsDecoder: JsonDecoder[StdDeviationBounds] = + DeriveJsonDecoder.gen[StdDeviationBounds] + implicit val decoder: JsonDecoder[ExtendedStatsAggregationResponse] = + DeriveJsonDecoder.gen[ExtendedStatsAggregationResponse] +} + private[elasticsearch] final case class MaxAggregationResponse(value: Double) extends AggregationResponse private[elasticsearch] object MaxAggregationResponse { @@ -161,6 +223,26 @@ private[elasticsearch] object TermsAggregationBucket { Some(field -> AvgAggregationResponse(value = objFields("value").unsafeAs[Double])) case str if str.contains("cardinality#") => Some(field -> CardinalityAggregationResponse(value = objFields("value").unsafeAs[Int])) + case str if str.contains("extended_stats#") => + Some( + field -> ExtendedStatsAggregationResponse( + count = objFields("count").unsafeAs[Int], + min = objFields("min").unsafeAs[Double], + max = objFields("max").unsafeAs[Double], + avg = objFields("avg").unsafeAs[Double], + sum = objFields("sum").unsafeAs[Double], + sum_of_squares = objFields("sum_of_squares").unsafeAs[Double], + variance = objFields("variance").unsafeAs[Double], + variance_population = objFields("variance_population").unsafeAs[Double], + variance_sampling = objFields("variance_sampling").unsafeAs[Double], + std_deviation = objFields("std_deviation").unsafeAs[Double], + std_deviation_population = objFields("std_deviation_population").unsafeAs[Double], + std_deviation_sampling = objFields("std_deviation_sampling").unsafeAs[Double], + std_deviation_bounds = objFields("std_deviation_sampling").unsafeAs[StdDeviationBounds]( + ExtendedStatsAggregationResponse.boundsDecoder + ) + ) + ) case str if str.contains("max#") => Some(field -> MaxAggregationResponse(value = objFields("value").unsafeAs[Double])) case str if str.contains("min#") => @@ -208,6 +290,8 @@ private[elasticsearch] object TermsAggregationBucket { (field.split("#")(1), data.asInstanceOf[AvgAggregationResponse]) case str if str.contains("cardinality#") => (field.split("#")(1), data.asInstanceOf[CardinalityAggregationResponse]) + case str if str.contains("extended_stats#") => + (field.split("#")(1), data.asInstanceOf[ExtendedStatsAggregationResponse]) case str if str.contains("max#") => (field.split("#")(1), data.asInstanceOf[MaxAggregationResponse]) case str if str.contains("min#") => diff --git a/modules/library/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala index 591ed8ad6..c856e5deb 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/executor/response/SearchWithAggregationsResponse.scala @@ -76,6 +76,10 @@ private[elasticsearch] final case class SearchWithAggregationsResponse( WeightedAvgAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("avg#") => AvgAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) + case str if str.contains("cardinality#") => + CardinalityAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) + case str if str.contains("extended_stats#") => + ExtendedStatsAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("max#") => MaxAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("min#") => @@ -88,8 +92,6 @@ private[elasticsearch] final case class SearchWithAggregationsResponse( StatsAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("sum#") => SumAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) - case str if str.contains("cardinality#") => - CardinalityAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("terms#") => TermsAggregationResponse.decoder.decodeJson(data.toString).map(field.split("#")(1) -> _) case str if str.contains("value_count#") => diff --git a/modules/library/src/main/scala/zio/elasticsearch/package.scala b/modules/library/src/main/scala/zio/elasticsearch/package.scala index 5902d7c4c..071a89df7 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/package.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/package.scala @@ -84,6 +84,18 @@ package object elasticsearch extends IndexNameNewtype with IndexPatternNewtype w def asCardinalityAggregation(name: String): RIO[R, Option[CardinalityAggregationResult]] = aggregationAs[CardinalityAggregationResult](name) + /** + * Executes the [[ElasticRequest.SearchRequest]] or the [[ElasticRequest.SearchAndAggregateRequest]]. + * + * @param name + * the name of the aggregation to retrieve + * @return + * a [[RIO]] effect that, when executed, will produce the aggregation as instance of + * [[result.ExtendedStatsAggregationResult]]. + */ + def asExtendedStatsAggregation(name: String): RIO[R, Option[ExtendedStatsAggregationResult]] = + aggregationAs[ExtendedStatsAggregationResult](name) + /** * Executes the [[ElasticRequest.SearchRequest]] or the [[ElasticRequest.SearchAndAggregateRequest]]. * diff --git a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala index d17d9e02e..b8ed7337c 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala @@ -17,6 +17,7 @@ package zio.elasticsearch.result import zio.Chunk +import zio.elasticsearch.executor.response.StdDeviationBounds import scala.util.{Failure, Success, Try} @@ -26,6 +27,22 @@ final case class AvgAggregationResult private[elasticsearch] (value: Double) ext final case class CardinalityAggregationResult private[elasticsearch] (value: Int) extends AggregationResult +final case class ExtendedStatsAggregationResult private[elasticsearch] ( + count: Int, + min: Double, + max: Double, + avg: Double, + sum: Double, + sumOfSquares: Double, + variance: Double, + variancePopulation: Double, + varianceSampling: Double, + stdDeviation: Double, + stdDeviationPopulation: Double, + stdDeviationSampling: Double, + stdDeviationBounds: StdDeviationBounds +) extends AggregationResult + final case class MaxAggregationResult private[elasticsearch] (value: Double) extends AggregationResult final case class MinAggregationResult private[elasticsearch] (value: Double) extends AggregationResult diff --git a/modules/library/src/main/scala/zio/elasticsearch/result/ElasticResult.scala b/modules/library/src/main/scala/zio/elasticsearch/result/ElasticResult.scala index 7b230b832..3eefd130f 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/result/ElasticResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/result/ElasticResult.scala @@ -48,6 +48,9 @@ private[elasticsearch] sealed trait ResultWithAggregation { def asCardinalityAggregation(name: String): IO[DecodingException, Option[CardinalityAggregationResult]] = aggregationAs[CardinalityAggregationResult](name) + def asExtendedStatsAggregation(name: String): IO[DecodingException, Option[ExtendedStatsAggregationResult]] = + aggregationAs[ExtendedStatsAggregationResult](name) + def asMaxAggregation(name: String): IO[DecodingException, Option[MaxAggregationResult]] = aggregationAs[MaxAggregationResult](name) diff --git a/modules/library/src/test/scala/zio/elasticsearch/ElasticAggregationSpec.scala b/modules/library/src/test/scala/zio/elasticsearch/ElasticAggregationSpec.scala index 052315f9f..1ca40d8cd 100644 --- a/modules/library/src/test/scala/zio/elasticsearch/ElasticAggregationSpec.scala +++ b/modules/library/src/test/scala/zio/elasticsearch/ElasticAggregationSpec.scala @@ -128,6 +128,29 @@ object ElasticAggregationSpec extends ZIOSpecDefault { equalTo(Cardinality(name = "aggregation", field = "intField", missing = Some(20))) ) }, + test("extendedStats") { + val aggregation = extendedStatsAggregation("aggregation", "testField") + val aggregationTs = extendedStatsAggregation("aggregation", TestSubDocument.intField) + val aggregationTsRaw = extendedStatsAggregation("aggregation", TestSubDocument.intField.raw) + val aggregationWithMissing = extendedStatsAggregation("aggregation", TestSubDocument.intField).missing(20.0) + val aggregationWithSigma = extendedStatsAggregation("aggregation", TestSubDocument.intField).sigma(3.0) + val aggregationWithMissingAndSigma = + extendedStatsAggregation("aggregation", TestSubDocument.intField).missing(20.0).sigma(3.0) + + assert(aggregation)( + equalTo(ExtendedStats(name = "aggregation", field = "testField", missing = None, sigma = None)) + ) && assert(aggregationTs)( + equalTo(ExtendedStats(name = "aggregation", field = "intField", missing = None, sigma = None)) + ) && assert(aggregationTsRaw)( + equalTo(ExtendedStats(name = "aggregation", field = "intField.raw", missing = None, sigma = None)) + ) && assert(aggregationWithMissing)( + equalTo(ExtendedStats(name = "aggregation", field = "intField", missing = Some(20.0), sigma = None)) + ) && assert(aggregationWithSigma)( + equalTo(ExtendedStats(name = "aggregation", field = "intField", missing = None, sigma = Some(3.0))) + ) && assert(aggregationWithMissingAndSigma)( + equalTo(ExtendedStats(name = "aggregation", field = "intField", missing = Some(20.0), sigma = Some(3.0))) + ) + }, test("max") { val aggregation = maxAggregation("aggregation", "testField") val aggregationTs = maxAggregation("aggregation", TestSubDocument.intField) @@ -729,6 +752,79 @@ object ElasticAggregationSpec extends ZIOSpecDefault { assert(aggregationTs.toJson)(equalTo(expectedTs.toJson)) && assert(aggregationWithMissing.toJson)(equalTo(expectedWithMissing.toJson)) }, + test("extendedStats") { + val aggregation = extendedStatsAggregation("aggregation", "testField") + val aggregationTs = extendedStatsAggregation("aggregation", TestSubDocument.intField) + val aggregationWithMissing = extendedStatsAggregation("aggregation", TestSubDocument.intField).missing(20.0) + val aggregationWithSigma = extendedStatsAggregation("aggregation", TestSubDocument.intField).sigma(3.0) + val aggregationWithMissingAndSigma = + extendedStatsAggregation("aggregation", TestSubDocument.intField).missing(20.0).sigma(3.0) + + val expected = + """ + |{ + | "aggregation": { + | "extended_stats": { + | "field": "testField" + | } + | } + |} + |""".stripMargin + + val expectedTs = + """ + |{ + | "aggregation": { + | "extended_stats": { + | "field": "intField" + | } + | } + |} + |""".stripMargin + + val expectedWithMissing = + """ + |{ + | "aggregation": { + | "extended_stats": { + | "field": "intField", + | "missing": 20.0 + | } + | } + |} + |""".stripMargin + + val expectedWithSigma = + """ + |{ + | "aggregation": { + | "extended_stats": { + | "field": "intField", + | "sigma": 3.0 + | } + | } + |} + |""".stripMargin + + val expectedWithMissingAndSigma = + """ + |{ + | "aggregation": { + | "extended_stats": { + | "field": "intField", + | "missing": 20.0, + | "sigma": 3.0 + | } + | } + |} + |""".stripMargin + + assert(aggregation.toJson)(equalTo(expected.toJson)) && + assert(aggregationTs.toJson)(equalTo(expectedTs.toJson)) && + assert(aggregationWithMissing.toJson)(equalTo(expectedWithMissing.toJson)) && + assert(aggregationWithSigma.toJson)(equalTo(expectedWithSigma.toJson)) && + assert(aggregationWithMissingAndSigma.toJson)(equalTo(expectedWithMissingAndSigma.toJson)) + }, test("max") { val aggregation = maxAggregation("aggregation", "testField") val aggregationTs = maxAggregation("aggregation", TestDocument.intField) diff --git a/website/sidebars.js b/website/sidebars.js index 89ccf6251..318505b5b 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -48,6 +48,7 @@ module.exports = { 'overview/aggregations/elastic_aggregation_bucket_selector', 'overview/aggregations/elastic_aggregation_bucket_sort', 'overview/aggregations/elastic_aggregation_cardinality', + 'overview/aggregations/elastic_aggregation_extended_stats', 'overview/aggregations/elastic_aggregation_max', 'overview/aggregations/elastic_aggregation_min', 'overview/aggregations/elastic_aggregation_missing', From 004942445263f62ac70e57e2e4e57ab8a4338d9d Mon Sep 17 00:00:00 2001 From: vanjaftn Date: Wed, 15 Nov 2023 10:00:54 +0100 Subject: [PATCH 2/4] Fix code remarks --- .../elastic_aggregation_extended_stats.md | 2 +- .../zio/elasticsearch/HttpExecutorSpec.scala | 14 ++-- .../aggregation/Aggregations.scala | 5 +- .../response/AggregationResponse.scala | 77 ++++++++++++------- .../result/AggregationResult.scala | 12 ++- 5 files changed, 70 insertions(+), 40 deletions(-) diff --git a/docs/overview/aggregations/elastic_aggregation_extended_stats.md b/docs/overview/aggregations/elastic_aggregation_extended_stats.md index 2e0413183..46193facf 100644 --- a/docs/overview/aggregations/elastic_aggregation_extended_stats.md +++ b/docs/overview/aggregations/elastic_aggregation_extended_stats.md @@ -35,7 +35,7 @@ val aggregationWithSigma: ExtendedStatsAggregation = extendedStatsAggregation(na If you want to add aggregation (on the same level), you can use `withAgg` method: ```scala -val multipleAggregations: MultipleAggregations = extendedStatsAggregation(name = "extendedStatsAggregation", field = Document.intField).withAgg(extendedStatsAggregation(name = "extendedStatsAggregation2", field = Document.doubleField)) +val multipleAggregations: MultipleAggregations = extendedStatsAggregation(name = "extendedStatsAggregation1", field = Document.intField).withAgg(extendedStatsAggregation(name = "extendedStatsAggregation2", field = Document.doubleField)) ``` You can find more information about `Extended stats` aggregation [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-metrics-extendedstats-aggregation.html#search-aggregations-metrics-extendedstats-aggregation). diff --git a/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala b/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala index f85f07294..16931dd7c 100644 --- a/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala +++ b/modules/integration/src/test/scala/zio/elasticsearch/HttpExecutorSpec.scala @@ -104,7 +104,7 @@ object HttpExecutorSpec extends IntegrationSpec { Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), Executor.execute(ElasticRequest.deleteIndex(firstSearchIndex)).orDie ), - test("aggregate using extended stats aggregation ttt") { + test("aggregate using extended stats aggregation") { checkOnce(genDocumentId, genTestDocument, genDocumentId, genTestDocument) { (firstDocumentId, firstDocument, secondDocumentId, secondDocument) => for { @@ -135,12 +135,12 @@ object HttpExecutorSpec extends IntegrationSpec { assert(aggsRes.head.stdDeviation)(equalTo(25.0)) && assert(aggsRes.head.stdDeviationPopulation)(equalTo(25.0)) && assert(aggsRes.head.stdDeviationSampling)(equalTo(35.35533905932738)) && - assert(aggsRes.head.stdDeviationBounds.upper)(equalTo(150.0)) && - assert(aggsRes.head.stdDeviationBounds.lower)(equalTo(0.0)) && - assert(aggsRes.head.stdDeviationBounds.upper_population)(equalTo(150.0)) && - assert(aggsRes.head.stdDeviationBounds.lower_population)(equalTo(0.0)) && - assert(aggsRes.head.stdDeviationBounds.upper_sampling)(equalTo(181.06601717798213)) && - assert(aggsRes.head.stdDeviationBounds.lower_sampling)(equalTo(-31.066017177982133)) + assert(aggsRes.head.stdDeviationBoundsResult.upper)(equalTo(150.0)) && + assert(aggsRes.head.stdDeviationBoundsResult.lower)(equalTo(0.0)) && + assert(aggsRes.head.stdDeviationBoundsResult.upperPopulation)(equalTo(150.0)) && + assert(aggsRes.head.stdDeviationBoundsResult.lowerPopulation)(equalTo(0.0)) && + assert(aggsRes.head.stdDeviationBoundsResult.upperSampling)(equalTo(181.06601717798213)) && + assert(aggsRes.head.stdDeviationBoundsResult.lowerSampling)(equalTo(-31.066017177982133)) } } @@ around( Executor.execute(ElasticRequest.createIndex(firstSearchIndex)), diff --git a/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala b/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala index 006516815..cbab4f158 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/aggregation/Aggregations.scala @@ -150,12 +150,13 @@ sealed trait ExtendedStatsAggregation /** * Sets the `sigma` parameter for the [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]]. The`sigma` parameter - * controls how many standard deviations plus/minus from the mean should std_deviation_bounds object display + * controls how many standard deviations plus/minus from the mean should std_deviation_bounds object display. * * @param value * the value to use for sigma parameter * @return - * an instance of the [[zio.elasticsearch.aggregation.ElasticAggregation]] enriched with the `sigma` parameter. + * an instance of the [[zio.elasticsearch.aggregation.ExtendedStatsAggregation]] enriched with the `sigma` + * parameter. */ def sigma(value: Double): ExtendedStatsAggregation } diff --git a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala index bb8d1d757..632ac4106 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala @@ -44,7 +44,7 @@ object AggregationResponse { stdDeviation, stdDeviationPopulation, stdDeviationSampling, - stdDeviationBounds + stdDeviationBoundsResponse ) => ExtendedStatsAggregationResult( count, @@ -59,7 +59,14 @@ object AggregationResponse { stdDeviation, stdDeviationPopulation, stdDeviationSampling, - stdDeviationBounds + StdDeviationBoundsResult( + stdDeviationBoundsResponse.upper, + stdDeviationBoundsResponse.lower, + stdDeviationBoundsResponse.upperPopulation, + stdDeviationBoundsResponse.lowerPopulation, + stdDeviationBoundsResponse.upperSampling, + stdDeviationBoundsResponse.lowerSampling + ) ) case MaxAggregationResponse(value) => MaxAggregationResult(value) @@ -107,34 +114,30 @@ private[elasticsearch] object CardinalityAggregationResponse { DeriveJsonDecoder.gen[CardinalityAggregationResponse] } -case class StdDeviationBounds private[elasticsearch] ( - upper: Double, - lower: Double, - upper_population: Double, - lower_population: Double, - upper_sampling: Double, - lower_sampling: Double -) - private[elasticsearch] final case class ExtendedStatsAggregationResponse( count: Int, min: Double, max: Double, avg: Double, sum: Double, - sum_of_squares: Double, + @jsonField("sum_of_squares") + sumOfSquares: Double, variance: Double, - variance_population: Double, - variance_sampling: Double, - std_deviation: Double, - std_deviation_population: Double, - std_deviation_sampling: Double, - std_deviation_bounds: StdDeviationBounds + @jsonField("variance_population") + variancePopulation: Double, + @jsonField("variance_sampling") + varianceSampling: Double, + @jsonField("std_deviation") + stdDeviation: Double, + @jsonField("std_deviation_population") + stdDeviationPopulation: Double, + @jsonField("std_deviation_sampling") + stdDeviationSampling: Double, + @jsonField("std_deviation_bounds") + stdDeviationBoundsResponse: StdDeviationBoundsResponse ) extends AggregationResponse private[elasticsearch] object ExtendedStatsAggregationResponse { - implicit val boundsDecoder: JsonDecoder[StdDeviationBounds] = - DeriveJsonDecoder.gen[StdDeviationBounds] implicit val decoder: JsonDecoder[ExtendedStatsAggregationResponse] = DeriveJsonDecoder.gen[ExtendedStatsAggregationResponse] } @@ -178,6 +181,24 @@ private[elasticsearch] object StatsAggregationResponse { implicit val decoder: JsonDecoder[StatsAggregationResponse] = DeriveJsonDecoder.gen[StatsAggregationResponse] } +private[elasticsearch] case class StdDeviationBoundsResponse( + upper: Double, + lower: Double, + @jsonField("upper_population") + upperPopulation: Double, + @jsonField("lower_population") + lowerPopulation: Double, + @jsonField("upper_sampling") + upperSampling: Double, + @jsonField("lower_sampling") + lowerSampling: Double +) extends AggregationResponse + +private[elasticsearch] object StdDeviationBoundsResponse { + implicit val decoder: JsonDecoder[StdDeviationBoundsResponse] = + DeriveJsonDecoder.gen[StdDeviationBoundsResponse] +} + private[elasticsearch] final case class SumAggregationResponse(value: Double) extends AggregationResponse private[elasticsearch] object SumAggregationResponse { @@ -231,15 +252,15 @@ private[elasticsearch] object TermsAggregationBucket { max = objFields("max").unsafeAs[Double], avg = objFields("avg").unsafeAs[Double], sum = objFields("sum").unsafeAs[Double], - sum_of_squares = objFields("sum_of_squares").unsafeAs[Double], + sumOfSquares = objFields("sum_of_squares").unsafeAs[Double], variance = objFields("variance").unsafeAs[Double], - variance_population = objFields("variance_population").unsafeAs[Double], - variance_sampling = objFields("variance_sampling").unsafeAs[Double], - std_deviation = objFields("std_deviation").unsafeAs[Double], - std_deviation_population = objFields("std_deviation_population").unsafeAs[Double], - std_deviation_sampling = objFields("std_deviation_sampling").unsafeAs[Double], - std_deviation_bounds = objFields("std_deviation_sampling").unsafeAs[StdDeviationBounds]( - ExtendedStatsAggregationResponse.boundsDecoder + variancePopulation = objFields("variance_population").unsafeAs[Double], + varianceSampling = objFields("variance_sampling").unsafeAs[Double], + stdDeviation = objFields("std_deviation").unsafeAs[Double], + stdDeviationPopulation = objFields("std_deviation_population").unsafeAs[Double], + stdDeviationSampling = objFields("std_deviation_sampling").unsafeAs[Double], + stdDeviationBoundsResponse = objFields("std_deviation_sampling").unsafeAs[StdDeviationBoundsResponse]( + StdDeviationBoundsResponse.decoder ) ) ) diff --git a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala index b8ed7337c..aa9994075 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala @@ -17,7 +17,6 @@ package zio.elasticsearch.result import zio.Chunk -import zio.elasticsearch.executor.response.StdDeviationBounds import scala.util.{Failure, Success, Try} @@ -27,6 +26,15 @@ final case class AvgAggregationResult private[elasticsearch] (value: Double) ext final case class CardinalityAggregationResult private[elasticsearch] (value: Int) extends AggregationResult +private[elasticsearch] case class StdDeviationBoundsResult( + upper: Double, + lower: Double, + upperPopulation: Double, + lowerPopulation: Double, + upperSampling: Double, + lowerSampling: Double +) extends AggregationResult + final case class ExtendedStatsAggregationResult private[elasticsearch] ( count: Int, min: Double, @@ -40,7 +48,7 @@ final case class ExtendedStatsAggregationResult private[elasticsearch] ( stdDeviation: Double, stdDeviationPopulation: Double, stdDeviationSampling: Double, - stdDeviationBounds: StdDeviationBounds + stdDeviationBoundsResult: StdDeviationBoundsResult ) extends AggregationResult final case class MaxAggregationResult private[elasticsearch] (value: Double) extends AggregationResult From df575eb7b35f6f56fdf71ab06efddad64eaf3019 Mon Sep 17 00:00:00 2001 From: vanjaftn Date: Wed, 15 Nov 2023 10:37:19 +0100 Subject: [PATCH 3/4] Implement changes in Extended stats aggregation --- .../result/AggregationResult.scala | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala index aa9994075..a4a4415a8 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/result/AggregationResult.scala @@ -26,15 +26,6 @@ final case class AvgAggregationResult private[elasticsearch] (value: Double) ext final case class CardinalityAggregationResult private[elasticsearch] (value: Int) extends AggregationResult -private[elasticsearch] case class StdDeviationBoundsResult( - upper: Double, - lower: Double, - upperPopulation: Double, - lowerPopulation: Double, - upperSampling: Double, - lowerSampling: Double -) extends AggregationResult - final case class ExtendedStatsAggregationResult private[elasticsearch] ( count: Int, min: Double, @@ -68,6 +59,15 @@ final case class StatsAggregationResult private[elasticsearch] ( sum: Double ) extends AggregationResult +private[elasticsearch] case class StdDeviationBoundsResult( + upper: Double, + lower: Double, + upperPopulation: Double, + lowerPopulation: Double, + upperSampling: Double, + lowerSampling: Double +) extends AggregationResult + final case class SumAggregationResult private[elasticsearch] (value: Double) extends AggregationResult final case class TermsAggregationResult private[elasticsearch] ( From 5cbbd5b767fa63cccca1a17c4eabd9e955b80dca Mon Sep 17 00:00:00 2001 From: vanjaftn Date: Wed, 15 Nov 2023 16:49:20 +0100 Subject: [PATCH 4/4] Fix code remarks --- docs/overview/queries/elastic_query_term.md | 2 +- .../response/AggregationResponse.scala | 36 +++++++++---------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/overview/queries/elastic_query_term.md b/docs/overview/queries/elastic_query_term.md index f511dc7a6..85ca3ceef 100644 --- a/docs/overview/queries/elastic_query_term.md +++ b/docs/overview/queries/elastic_query_term.md @@ -13,7 +13,7 @@ import zio.elasticsearch.ElasticQuery._ You can create a `Term` query using the `term` method this way: ```scala -val query: TermQuery = term(field = Document.name, value = "test") +val query: TermQuery = term(field = "stringField", value = "test") ``` You can create a [type-safe](https://lambdaworks.github.io/zio-elasticsearch/overview/overview_zio_prelude_schema) `Term` query using the `term` method this way: diff --git a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala index 632ac4106..675913500 100644 --- a/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala +++ b/modules/library/src/main/scala/zio/elasticsearch/executor/response/AggregationResponse.scala @@ -47,25 +47,25 @@ object AggregationResponse { stdDeviationBoundsResponse ) => ExtendedStatsAggregationResult( - count, - min, - max, - avg, - sum, - sumOfSquares, - variance, - variancePopulation, - varianceSampling, - stdDeviation, - stdDeviationPopulation, - stdDeviationSampling, + count = count, + min = min, + max = max, + avg = avg, + sum = sum, + sumOfSquares = sumOfSquares, + variance = variance, + variancePopulation = variancePopulation, + varianceSampling = varianceSampling, + stdDeviation = stdDeviation, + stdDeviationPopulation = stdDeviationPopulation, + stdDeviationSampling = stdDeviationSampling, StdDeviationBoundsResult( - stdDeviationBoundsResponse.upper, - stdDeviationBoundsResponse.lower, - stdDeviationBoundsResponse.upperPopulation, - stdDeviationBoundsResponse.lowerPopulation, - stdDeviationBoundsResponse.upperSampling, - stdDeviationBoundsResponse.lowerSampling + upper = stdDeviationBoundsResponse.upper, + lower = stdDeviationBoundsResponse.lower, + upperPopulation = stdDeviationBoundsResponse.upperPopulation, + lowerPopulation = stdDeviationBoundsResponse.lowerPopulation, + upperSampling = stdDeviationBoundsResponse.upperSampling, + lowerSampling = stdDeviationBoundsResponse.lowerSampling ) ) case MaxAggregationResponse(value) =>