Support multi ranges traversal when doing date histogram rewrite opti…

…mization (opensearch-project#13317)

CHANGELOG.md

@@ -36,6 +36,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - [Search Pipeline] Handle default pipeline for multiple indices ([#13276](https://github.com/opensearch-project/OpenSearch/pull/13276))
 - [Batch Ingestion] Add `batch_size` to `_bulk` API. ([#12457](https://github.com/opensearch-project/OpenSearch/issues/12457))
 - [Remote Store] Add capability of doing refresh as determined by the translog ([#12992](https://github.com/opensearch-project/OpenSearch/pull/12992))
+- Add support for deep copying SearchRequest ([#12295](https://github.com/opensearch-project/OpenSearch/pull/12295))
+- Support multi ranges traversal when doing date histogram rewrite optimization. ([#13317](https://github.com/opensearch-project/OpenSearch/pull/13317))
 
 ### Dependencies
 - Bump `org.apache.commons:commons-configuration2` from 2.10.0 to 2.10.1 ([#12896](https://github.com/opensearch-project/OpenSearch/pull/12896))

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/10_histogram.yml

@@ -644,3 +644,59 @@ setup:
  - match: { aggregations.histo.buckets.0.doc_count: 1 }
  - match: { aggregations.histo.buckets.20.key: 20 }
  - match: { aggregations.histo.buckets.20.doc_count: 1 }
+
+---
+"date_histogram profiler shows filter rewrite info":
+ - skip:
+ version: " - 2.99.99"
+ reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+ - do:
+ indices.create:
+ index: test_2
+ body:
+ settings:
+ number_of_replicas: 0
+ number_of_shards: 1
+ mappings:
+ properties:
+ date:
+ type: date
+
+ - do:
+ bulk:
+ index: test_2
+ refresh: true
+ body:
+ - '{"index": {}}'
+ - '{"date": "2016-01-01"}'
+ - '{"index": {}}'
+ - '{"date": "2016-01-02"}'
+ - '{"index": {}}'
+ - '{"date": "2016-02-01"}'
+ - '{"index": {}}'
+ - '{"date": "2016-03-01"}'
+
+ - do:
+ search:
+ index: test_2
+ body:
+ size: 0
+ profile: true
+ aggs:
+ histo:
+ date_histogram:
+ field: date
+ calendar_interval: month
+
+ - match: { hits.total.value: 4 }
+ - length: { aggregations.histo.buckets: 3 }
+ - match: { aggregations.histo.buckets.0.key_as_string: "2016-01-01T00:00:00.000Z" }
+ - match: { aggregations.histo.buckets.0.doc_count: 2 }
+ - match: { profile.shards.0.aggregations.0.type: DateHistogramAggregator }
+ - match: { profile.shards.0.aggregations.0.description: histo }
+ - match: { profile.shards.0.aggregations.0.debug.total_buckets: 3 }
+ - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+ - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/230_composite.yml

@@ -1069,3 +1069,61 @@ setup:
  - match: { aggregations.test.buckets.1.doc_count: 2 }
  - match: { aggregations.test.buckets.2.key.kw: null }
  - match: { aggregations.test.buckets.2.doc_count: 2 }
+
+---
+"composite aggregation date_histogram profile shows filter rewrite info":
+ - skip:
+ version: " - 2.99.99"
+ reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+ - do:
+ indices.create:
+ index: test_2
+ body:
+ settings:
+ number_of_replicas: 0
+ number_of_shards: 1
+ mappings:
+ properties:
+ date:
+ type: date
+ - do:
+ bulk:
+ index: test_2
+ refresh: true
+ body:
+ - '{"index": {}}'
+ - '{"date": "2016-01-01"}'
+ - '{"index": {}}'
+ - '{"date": "2016-01-02"}'
+ - '{"index": {}}'
+ - '{"date": "2016-02-01"}'
+ - '{"index": {}}'
+ - '{"date": "2016-03-01"}'
+ - do:
+ search:
+ index: test_2
+ body:
+ size: 0
+ profile: true
+ aggregations:
+ test:
+ composite:
+ sources: [
+ {
+ "date": {
+ "date_histogram": {
+ "field": "date",
+ "calendar_interval": "1d",
+ "format": "strict_date"
+ }
+ }
+ }
+ ]
+
+ - match: { hits.total.value: 4 }
+ - length: { aggregations.test.buckets: 4 }
+ - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+ - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }

rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/330_auto_date_histogram.yml

@@ -99,3 +99,29 @@ setup:
  - length: { aggregations.histo.buckets: 2 }
  - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
  - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+
+---
+"auto_date_histogram profile shows filter rewrite info":
+ - skip:
+ version: " - 2.99.99"
+ reason: debug info for filter rewrite added in 3.0.0 (to be backported to 2.14.0)
+
+ - do:
+ search:
+ body:
+ profile: true
+ size: 0
+ aggs:
+ histo:
+ auto_date_histogram:
+ field: date
+ buckets: 2
+
+ - match: { hits.total.value: 4 }
+ - length: { aggregations.histo.buckets: 2 }
+ - match: { profile.shards.0.aggregations.0.type: AutoDateHistogramAggregator.FromSingle }
+ - match: { profile.shards.0.aggregations.0.debug.surviving_buckets: 4 }
+ - match: { profile.shards.0.aggregations.0.debug.optimized_segments: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.unoptimized_segments: 0 }
+ - match: { profile.shards.0.aggregations.0.debug.leaf_visited: 1 }
+ - match: { profile.shards.0.aggregations.0.debug.inner_visited: 0 }

server/src/main/java/org/opensearch/search/SearchService.java

@@ -276,7 +276,7 @@ public class SearchService extends AbstractLifecycleComponent implements IndexEv
  // value 0 means rewrite filters optimization in aggregations will be disabled
  public static final Setting<Integer> MAX_AGGREGATION_REWRITE_FILTERS = Setting.intSetting(
  "search.max_aggregation_rewrite_filters",
- 72,
+ 3000,
  0,
  Property.Dynamic,
  Property.NodeScope