diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java index 43df4f0f6e3..f8b235c45ef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java @@ -10,17 +10,11 @@ @FixMethodOrder(MethodSorters.JVM) public class CalcitePPLBig5IT extends PPLBig5IT { - private boolean initialized = false; @Override public void init() throws Exception { super.init(); enableCalcite(); disallowCalciteFallback(); - // warm-up - if (!initialized) { - executeQuery("source=big5 | join on 1=1 big5"); // trigger non-pushdown - initialized = true; - } } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java index cf01704870e..529a7ce8762 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java @@ -6,217 +6,205 @@ package org.opensearch.sql.calcite.big5; import java.io.IOException; -import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; import org.junit.AfterClass; import org.junit.FixMethodOrder; -import org.junit.Ignore; import org.junit.Test; import org.junit.runners.MethodSorters; +import org.opensearch.common.collect.MapBuilder; import org.opensearch.sql.ppl.PPLIntegTestCase; @FixMethodOrder(MethodSorters.JVM) public class PPLBig5IT extends PPLIntegTestCase { - private boolean initialized = false; - private static final Map summary = new LinkedHashMap<>(); + private static final MapBuilder summary = MapBuilder.newMapBuilder(); @Override public void init() throws Exception { super.init(); loadIndex(Index.BIG5); disableCalcite(); - // warm-up - if (!initialized) { - executeQuery("source=big5 | eval a = 1"); // trigger non-pushdown - initialized = true; - } } @AfterClass public static void reset() throws IOException { long total = 0; - for (long duration : summary.values()) { + Map map = summary.immutableMap(); + for (long duration : map.values()) { total += duration; } System.out.println("Summary:"); - for (Map.Entry entry : summary.entrySet()) { - System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue()); - } + map.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach( + entry -> + System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue())); System.out.printf( Locale.ENGLISH, "Total %d queries succeed. Average duration: %d ms%n", - summary.size(), - total / summary.size()); + map.size(), + total / map.size()); System.out.println(); - summary.clear(); - } - - protected void timing(String query, String ppl) throws IOException { - long start = System.currentTimeMillis(); - executeQuery(ppl); - long duration = System.currentTimeMillis() - start; - summary.put(query, duration); } @Test public void asc_sort_timestamp() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp.ppl")); - timing("asc_sort_timestamp", ppl); + timing(summary, "asc_sort_timestamp", ppl); } @Test public void asc_sort_timestamp_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_can_match_shortcut.ppl")); - timing("asc_sort_timestamp_can_match_shortcut", ppl); + timing(summary, "asc_sort_timestamp_can_match_shortcut", ppl); } @Test public void asc_sort_timestamp_no_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl")); - timing("asc_sort_timestamp_no_can_match_shortcut", ppl); + timing(summary, "asc_sort_timestamp_no_can_match_shortcut", ppl); } @Test public void asc_sort_with_after_timestamp() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/asc_sort_with_after_timestamp.ppl")); - timing("asc_sort_with_after_timestamp", ppl); + timing(summary, "asc_sort_with_after_timestamp", ppl); } @Test public void composite_date_histogram_daily() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/composite_date_histogram_daily.ppl")); - timing("composite_date_histogram_daily", ppl); + timing(summary, "composite_date_histogram_daily", ppl); } @Test public void composite_terms_keyword() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/composite_terms_keyword.ppl")); - timing("composite_terms_keyword", ppl); + timing(summary, "composite_terms_keyword", ppl); } @Test public void composite_terms() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/composite_terms.ppl")); - timing("composite_terms", ppl); + timing(summary, "composite_terms", ppl); } @Test public void date_histogram_hourly_agg() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/date_histogram_hourly_agg.ppl")); - timing("date_histogram_hourly_agg", ppl); + timing(summary, "date_histogram_hourly_agg", ppl); } @Test public void date_histogram_minute_agg() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/date_histogram_minute_agg.ppl")); - timing("date_histogram_minute_agg", ppl); + timing(summary, "date_histogram_minute_agg", ppl); } @Test public void test_default() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/default.ppl")); - timing("default", ppl); + timing(summary, "default", ppl); } @Test public void desc_sort_timestamp() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp.ppl")); - timing("desc_sort_timestamp", ppl); + timing(summary, "desc_sort_timestamp", ppl); } @Test public void desc_sort_timestamp_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_can_match_shortcut.ppl")); - timing("desc_sort_timestamp_can_match_shortcut", ppl); + timing(summary, "desc_sort_timestamp_can_match_shortcut", ppl); } @Test public void desc_sort_timestamp_no_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl")); - timing("desc_sort_timestamp_no_can_match_shortcut", ppl); + timing(summary, "desc_sort_timestamp_no_can_match_shortcut", ppl); } @Test public void desc_sort_with_after_timestamp() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/desc_sort_with_after_timestamp.ppl")); - timing("desc_sort_with_after_timestamp", ppl); + timing(summary, "desc_sort_with_after_timestamp", ppl); } @Test public void keyword_in_range() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/keyword_in_range.ppl")); - timing("keyword_in_range", ppl); + timing(summary, "keyword_in_range", ppl); } @Test public void keyword_terms() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/keyword_terms.ppl")); - timing("keyword_terms", ppl); + timing(summary, "keyword_terms", ppl); } @Test public void keyword_terms_low_cardinality() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/keyword_terms_low_cardinality.ppl")); - timing("keyword_terms_low_cardinality", ppl); + timing(summary, "keyword_terms_low_cardinality", ppl); } @Test public void multi_terms_keyword() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/multi_terms_keyword.ppl")); - timing("multi_terms_keyword", ppl); + timing(summary, "multi_terms_keyword", ppl); } @Test public void query_string_on_message() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message.ppl")); - timing("query_string_on_message", ppl); + timing(summary, "query_string_on_message", ppl); } @Test public void query_string_on_message_filtered() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered.ppl")); - timing("query_string_on_message_filtered", ppl); + timing(summary, "query_string_on_message_filtered", ppl); } @Test public void query_string_on_message_filtered_sorted_num() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered_sorted_num.ppl")); - timing("query_string_on_message_filtered_sorted_num", ppl); + timing(summary, "query_string_on_message_filtered_sorted_num", ppl); } @Test public void range() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range.ppl")); - timing("range", ppl); + timing(summary, "range", ppl); } - @Ignore("Failed to parse request payload") + @Test public void range_auto_date_histo() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo.ppl")); - timing("range_auto_date_histo", ppl); + timing(summary, "range_auto_date_histo", ppl); } - @Ignore("Failed to parse request payload") + @Test public void range_auto_date_histo_with_metrics() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo_with_metrics.ppl")); - timing("range_auto_date_histo_with_metrics", ppl); + timing(summary, "range_auto_date_histo_with_metrics", ppl); } @Test public void range_numeric() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_numeric.ppl")); - timing("range_numeric", ppl); + timing(summary, "range_numeric", ppl); } @Test public void range_field_conjunction_big_range_big_term_query() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_field_conjunction_big_range_big_term_query.ppl")); - timing("range_field_conjunction_big_range_big_term_query", ppl); + timing(summary, "range_field_conjunction_big_range_big_term_query", ppl); } @Test @@ -224,7 +212,7 @@ public void range_field_conjunction_small_range_big_term_query() throws IOExcept String ppl = sanitize( loadFromFile("big5/queries/range_field_conjunction_small_range_big_term_query.ppl")); - timing("range_field_conjunction_small_range_big_term_query", ppl); + timing(summary, "range_field_conjunction_small_range_big_term_query", ppl); } @Test @@ -232,7 +220,7 @@ public void range_field_conjunction_small_range_small_term_query() throws IOExce String ppl = sanitize( loadFromFile("big5/queries/range_field_conjunction_small_range_small_term_query.ppl")); - timing("range_field_conjunction_small_range_small_term_query", ppl); + timing(summary, "range_field_conjunction_small_range_small_term_query", ppl); } @Test @@ -240,78 +228,78 @@ public void range_field_disjunction_big_range_small_term_query() throws IOExcept String ppl = sanitize( loadFromFile("big5/queries/range_field_disjunction_big_range_small_term_query.ppl")); - timing("range_field_disjunction_big_range_small_term_query", ppl); + timing(summary, "range_field_disjunction_big_range_small_term_query", ppl); } @Test public void range_with_asc_sort() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_with_asc_sort.ppl")); - timing("range_with_asc_sort", ppl); + timing(summary, "range_with_asc_sort", ppl); } @Test public void range_with_desc_sort() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/range_with_desc_sort.ppl")); - timing("range_with_desc_sort", ppl); + timing(summary, "range_with_desc_sort", ppl); } @Test public void scroll() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/scroll.ppl")); - timing("scroll", ppl); + timing(summary, "scroll", ppl); } @Test public void sort_keyword_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_can_match_shortcut.ppl")); - timing("sort_keyword_can_match_shortcut", ppl); + timing(summary, "sort_keyword_can_match_shortcut", ppl); } @Test public void sort_keyword_no_can_match_shortcut() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_no_can_match_shortcut.ppl")); - timing("sort_keyword_no_can_match_shortcut", ppl); + timing(summary, "sort_keyword_no_can_match_shortcut", ppl); } @Test public void sort_numeric_asc() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc.ppl")); - timing("sort_numeric_asc", ppl); + timing(summary, "sort_numeric_asc", ppl); } @Test public void sort_numeric_asc_with_match() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc_with_match.ppl")); - timing("sort_numeric_asc_with_match", ppl); + timing(summary, "sort_numeric_asc_with_match", ppl); } @Test public void sort_numeric_desc() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc.ppl")); - timing("sort_numeric_desc", ppl); + timing(summary, "sort_numeric_desc", ppl); } @Test public void sort_numeric_desc_with_match() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc_with_match.ppl")); - timing("sort_numeric_desc_with_match", ppl); + timing(summary, "sort_numeric_desc_with_match", ppl); } @Test public void term() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/term.ppl")); - timing("term", ppl); + timing(summary, "term", ppl); } @Test public void terms_significant_1() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/terms_significant_1.ppl")); - timing("terms_significant_1", ppl); + timing(summary, "terms_significant_1", ppl); } @Test public void terms_significant_2() throws IOException { String ppl = sanitize(loadFromFile("big5/queries/terms_significant_2.ppl")); - timing("terms_significant_2", ppl); + timing(summary, "terms_significant_2", ppl); } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/CalcitePPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/CalcitePPLClickBenchIT.java new file mode 100644 index 00000000000..a75a42704f8 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/CalcitePPLClickBenchIT.java @@ -0,0 +1,20 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.clickbench; + +import org.junit.FixMethodOrder; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.JVM) +public class CalcitePPLClickBenchIT extends PPLClickBenchIT { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java new file mode 100644 index 00000000000..9e730a29ddd --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/clickbench/PPLClickBenchIT.java @@ -0,0 +1,66 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.clickbench; + +import java.io.IOException; +import java.util.Locale; +import java.util.Map; +import java.util.Set; +import org.junit.AfterClass; +import org.junit.FixMethodOrder; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.opensearch.common.collect.MapBuilder; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +@FixMethodOrder(MethodSorters.JVM) +public class PPLClickBenchIT extends PPLIntegTestCase { + private static final MapBuilder summary = MapBuilder.newMapBuilder(); + + @Override + public void init() throws Exception { + super.init(); + loadIndex(Index.CLICK_BENCH); + disableCalcite(); + } + + @AfterClass + public static void reset() throws IOException { + long total = 0; + Map map = summary.immutableMap(); + for (long duration : map.values()) { + total += duration; + } + System.out.println("Summary:"); + map.entrySet().stream() + .sorted(Map.Entry.comparingByKey()) + .forEach( + entry -> + System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue())); + System.out.printf( + Locale.ENGLISH, + "Total %d queries succeed. Average duration: %d ms%n", + map.size(), + total / map.size()); + System.out.println(); + } + + /** Ignore queries that are not supported by Calcite. */ + protected Set ignored() { + return Set.of(29); + } + + @Test + public void test() throws IOException { + for (int i = 1; i <= 43; i++) { + if (ignored().contains(i)) { + continue; + } + String ppl = sanitize(loadFromFile("clickbench/queries/q" + i + ".ppl")); + timing(summary, "q" + i, ppl); + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index 3702f687c03..94b61c4bd7d 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -13,6 +13,7 @@ import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBig5MappingFile; +import static org.opensearch.sql.legacy.TestUtils.getClickBenchMappingFile; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDateIndexMapping; @@ -890,6 +891,11 @@ public enum Index { "big5", getBig5MappingFile("big5_index_mapping.json"), "src/test/resources/big5/data/big5.json"), + CLICK_BENCH( + "hits", + "clickbench", + getClickBenchMappingFile("clickbench_index_mapping.json"), + "src/test/resources/clickbench/data/clickbench.json"), ARRAY( TestsConstants.TEST_INDEX_ARRAY, "array", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index 75d4bfd7e03..fe34237fe4f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -484,4 +484,12 @@ public static String getBig5MappingFile(String fileName) { return null; } } + + public static String getClickBenchMappingFile(String fileName) { + try { + return TestUtils.fileToString("src/test/resources/clickbench/mappings/" + fileName, false); + } catch (IOException e) { + return null; + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 96a3e06800a..e799c3535e9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -24,6 +24,7 @@ import org.opensearch.client.RequestOptions; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; +import org.opensearch.common.collect.MapBuilder; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; import org.opensearch.sql.legacy.SQLIntegTestCase; @@ -69,6 +70,15 @@ protected String executeCsvQuery(String query) throws IOException { return executeCsvQuery(query, true); } + protected void timing(MapBuilder builder, String query, String ppl) + throws IOException { + executeQuery(ppl); // warm-up + long start = System.currentTimeMillis(); + executeQuery(ppl); + long duration = System.currentTimeMillis() - start; + builder.put(query, duration); + } + protected void failWithMessage(String query, String message) { try { client().performRequest(buildRequest(query, QUERY_API_ENDPOINT)); diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl index 528337f078d..52a51cf7419 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl @@ -1,10 +1,10 @@ source = big5 | eval range_bucket = case( - `metrics.size` < -10, "range_1", - `metrics.size` >= -10 and `metrics.size` < 10, "range_2", - `metrics.size` >= 10 and `metrics.size` < 100, "range_3", - `metrics.size` >= 100 and `metrics.size` < 1000, "range_4", - `metrics.size` >= 1000 and `metrics.size` < 2000, "range_5", - `metrics.size` >= 2000, "range_6") + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') | stats count() by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl index 673a3892899..506978ace5b 100644 --- a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl @@ -1,10 +1,10 @@ source = big5 | eval range_bucket = case( - `metrics.size` < -10, "range_1", - `metrics.size` >= -10 and `metrics.size` < 10, "range_2", - `metrics.size` >= 10 and `metrics.size` < 100, "range_3", - `metrics.size` >= 100 and `metrics.size` < 1000, "range_4", - `metrics.size` >= 1000 and `metrics.size` < 2000, "range_5", - `metrics.size` >= 2000, "range_6") + `metrics.size` < -10, 'range_1', + `metrics.size` >= -10 and `metrics.size` < 10, 'range_2', + `metrics.size` >= 10 and `metrics.size` < 100, 'range_3', + `metrics.size` >= 100 and `metrics.size` < 1000, 'range_4', + `metrics.size` >= 1000 and `metrics.size` < 2000, 'range_5', + `metrics.size` >= 2000, 'range_6') | stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span | sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/data/clickbench.json b/integ-test/src/test/resources/clickbench/data/clickbench.json new file mode 100644 index 00000000000..7f21bad191f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/data/clickbench.json @@ -0,0 +1,2 @@ +{"index":{}} +{"WatchID":"9110818468285196899","JavaEnable":0,"Title":"","GoodEvent":1,"EventTime":"2013-07-14 20:38:47","EventDate":"2013-07-15","CounterID":17,"ClientIP":-1216690514,"RegionID":839,"UserID":"-2461439046089301801","CounterClass":0,"OS":0,"UserAgent":0,"URL":"","Referer":"https://example.org/about","IsRefresh":0,"RefererCategoryID":0,"RefererRegionID":0,"URLCategoryID":0,"URLRegionID":0,"ResolutionWidth":0,"ResolutionHeight":0,"ResolutionDepth":0,"FlashMajor":0,"FlashMinor":0,"FlashMinor2":"","NetMajor":0,"NetMinor":0,"UserAgentMajor":0,"UserAgentMinor":"�O","CookieEnable":0,"JavascriptEnable":0,"IsMobile":0,"MobilePhone":0,"MobilePhoneModel":"","Params":"","IPNetworkID":3793327,"TraficSourceID":4,"SearchEngineID":0,"SearchPhrase":"","AdvEngineID":0,"IsArtifical":0,"WindowClientWidth":0,"WindowClientHeight":0,"ClientTimeZone":-1,"ClientEventTime":"1971-01-01 14:16:06","SilverlightVersion1":0,"SilverlightVersion2":0,"SilverlightVersion3":0,"SilverlightVersion4":0,"PageCharset":"","CodeVersion":0,"IsLink":0,"IsDownload":0,"IsNotBounce":0,"FUniqID":"0","OriginalURL":"","HID":0,"IsOldCounter":0,"IsEvent":0,"IsParameter":0,"DontCountHits":0,"WithHash":0,"HitColor":"5","LocalEventTime":"2013-07-15 10:47:34","Age":0,"Sex":0,"Income":0,"Interests":0,"Robotness":0,"RemoteIP":-1001831330,"WindowName":-1,"OpenerName":-1,"HistoryLength":-1,"BrowserLanguage":"�","BrowserCountry":"�\f","SocialNetwork":"","SocialAction":"","HTTPError":0,"SendTiming":0,"DNSTiming":0,"ConnectTiming":0,"ResponseStartTiming":0,"ResponseEndTiming":0,"FetchTiming":0,"SocialSourceNetworkID":0,"SocialSourcePage":"","ParamPrice":"0","ParamOrderID":"","ParamCurrency":"NH\u001C","ParamCurrencyID":0,"OpenstatServiceName":"","OpenstatCampaignID":"","OpenstatAdID":"","OpenstatSourceID":"","UTMSource":"","UTMMedium":"","UTMCampaign":"","UTMContent":"","UTMTerm":"","FromTag":"","HasGCLID":0,"RefererHash":"-296158784638538920","URLHash":"-8417682003818480435","CLID":0} diff --git a/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json new file mode 100644 index 00000000000..1482f8d8a5e --- /dev/null +++ b/integ-test/src/test/resources/clickbench/mappings/clickbench_index_mapping.json @@ -0,0 +1,325 @@ +{ + "settings": { + "index": { + "sort.field": [ "CounterID", "EventDate", "UserID", "EventTime", "WatchID" ], + "sort.order": [ "desc", "desc", "desc", "desc", "desc" ] + } + }, + "mappings" : { + "properties" : { + "AdvEngineID" : { + "type" : "short" + }, + "Age" : { + "type" : "short" + }, + "BrowserCountry" : { + "type" : "keyword" + }, + "BrowserLanguage" : { + "type" : "keyword" + }, + "CLID" : { + "type" : "integer" + }, + "ClientEventTime" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "ClientIP" : { + "type" : "integer" + }, + "ClientTimeZone" : { + "type" : "short" + }, + "CodeVersion" : { + "type" : "integer" + }, + "ConnectTiming" : { + "type" : "integer" + }, + "CookieEnable" : { + "type" : "short" + }, + "CounterClass" : { + "type" : "short" + }, + "CounterID" : { + "type" : "integer" + }, + "DNSTiming" : { + "type" : "integer" + }, + "DontCountHits" : { + "type" : "short" + }, + "EventDate" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "EventTime" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "FUniqID" : { + "type" : "long" + }, + "FetchTiming" : { + "type" : "integer" + }, + "FlashMajor" : { + "type" : "short" + }, + "FlashMinor" : { + "type" : "short" + }, + "FlashMinor2" : { + "type" : "short" + }, + "FromTag" : { + "type" : "keyword" + }, + "GoodEvent" : { + "type" : "short" + }, + "HID" : { + "type" : "integer" + }, + "HTTPError" : { + "type" : "short" + }, + "HasGCLID" : { + "type" : "short" + }, + "HistoryLength" : { + "type" : "short" + }, + "HitColor" : { + "type" : "keyword" + }, + "IPNetworkID" : { + "type" : "integer" + }, + "Income" : { + "type" : "short" + }, + "Interests" : { + "type" : "short" + }, + "IsArtifical" : { + "type" : "short" + }, + "IsDownload" : { + "type" : "short" + }, + "IsEvent" : { + "type" : "short" + }, + "IsLink" : { + "type" : "short" + }, + "IsMobile" : { + "type" : "short" + }, + "IsNotBounce" : { + "type" : "short" + }, + "IsOldCounter" : { + "type" : "short" + }, + "IsParameter" : { + "type" : "short" + }, + "IsRefresh" : { + "type" : "short" + }, + "JavaEnable" : { + "type" : "short" + }, + "JavascriptEnable" : { + "type" : "short" + }, + "LocalEventTime" : { + "type" : "date", + "format": "yyyy-MM-dd HH:mm:ss||strict_date_optional_time ||epoch_millis" + }, + "MobilePhone" : { + "type" : "short" + }, + "MobilePhoneModel" : { + "type" : "keyword" + }, + "NetMajor" : { + "type" : "short" + }, + "NetMinor" : { + "type" : "short" + }, + "OS" : { + "type" : "short" + }, + "OpenerName" : { + "type" : "integer" + }, + "OpenstatAdID" : { + "type" : "keyword" + }, + "OpenstatCampaignID" : { + "type" : "keyword" + }, + "OpenstatServiceName" : { + "type" : "keyword" + }, + "OpenstatSourceID" : { + "type" : "keyword" + }, + "OriginalURL" : { + "type" : "keyword" + }, + "PageCharset" : { + "type" : "keyword" + }, + "ParamCurrency" : { + "type" : "keyword" + }, + "ParamCurrencyID" : { + "type" : "short" + }, + "ParamOrderID" : { + "type" : "keyword" + }, + "ParamPrice" : { + "type" : "long" + }, + "Params" : { + "type" : "keyword" + }, + "Referer" : { + "type" : "keyword" + }, + "RefererCategoryID" : { + "type" : "short" + }, + "RefererHash" : { + "type" : "long" + }, + "RefererRegionID" : { + "type" : "integer" + }, + "RegionID" : { + "type" : "integer" + }, + "RemoteIP" : { + "type" : "integer" + }, + "ResolutionDepth" : { + "type" : "short" + }, + "ResolutionHeight" : { + "type" : "short" + }, + "ResolutionWidth" : { + "type" : "short" + }, + "ResponseEndTiming" : { + "type" : "integer" + }, + "ResponseStartTiming" : { + "type" : "integer" + }, + "Robotness" : { + "type" : "short" + }, + "SearchEngineID" : { + "type" : "short" + }, + "SearchPhrase" : { + "type" : "keyword" + }, + "SendTiming" : { + "type" : "integer" + }, + "Sex" : { + "type" : "short" + }, + "SilverlightVersion1" : { + "type" : "short" + }, + "SilverlightVersion2" : { + "type" : "short" + }, + "SilverlightVersion3" : { + "type" : "integer" + }, + "SilverlightVersion4" : { + "type" : "short" + }, + "SocialSourceNetworkID" : { + "type" : "short" + }, + "SocialSourcePage" : { + "type" : "keyword" + }, + "Title" : { + "type" : "keyword" + }, + "TraficSourceID" : { + "type" : "short" + }, + "URL" : { + "type" : "keyword" + }, + "URLCategoryID" : { + "type" : "short" + }, + "URLHash" : { + "type" : "long" + }, + "URLRegionID" : { + "type" : "integer" + }, + "UTMCampaign" : { + "type" : "keyword" + }, + "UTMContent" : { + "type" : "keyword" + }, + "UTMMedium" : { + "type" : "keyword" + }, + "UTMSource" : { + "type" : "keyword" + }, + "UTMTerm" : { + "type" : "keyword" + }, + "UserAgent" : { + "type" : "short" + }, + "UserAgentMajor" : { + "type" : "short" + }, + "UserAgentMinor" : { + "type" : "keyword" + }, + "UserID" : { + "type" : "long" + }, + "WatchID" : { + "type" : "long" + }, + "WindowClientHeight" : { + "type" : "short" + }, + "WindowClientWidth" : { + "type" : "short" + }, + "WindowName" : { + "type" : "integer" + }, + "WithHash" : { + "type" : "short" + } + } + } +} diff --git a/integ-test/src/test/resources/clickbench/queries/q1.ppl b/integ-test/src/test/resources/clickbench/queries/q1.ppl new file mode 100644 index 00000000000..1e310a81935 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q1.ppl @@ -0,0 +1,4 @@ +/* +SELECT COUNT(*) FROM hits; +*/ +source=hits | stats count() \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q10.ppl b/integ-test/src/test/resources/clickbench/queries/q10.ppl new file mode 100644 index 00000000000..11ac3b319db --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q10.ppl @@ -0,0 +1,8 @@ +/* +SELECT RegionID, SUM(AdvEngineID), COUNT(*) AS c, AVG(ResolutionWidth), COUNT(DISTINCT UserID) +FROM hits GROUP BY RegionID ORDER BY c DESC LIMIT 10; +*/ +source=hits +| stats sum(AdvEngineID), count() as c, avg(ResolutionWidth), dc(UserID) by RegionID +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q11.ppl b/integ-test/src/test/resources/clickbench/queries/q11.ppl new file mode 100644 index 00000000000..1df76937bf3 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q11.ppl @@ -0,0 +1,10 @@ +/* +SELECT MobilePhoneModel, COUNT(DISTINCT UserID) AS u +FROM hits WHERE MobilePhoneModel <> '' +GROUP BY MobilePhoneModel ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where MobilePhoneModel != '' +| stats dc(UserID) as u by MobilePhoneModel +| sort - u +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q12.ppl b/integ-test/src/test/resources/clickbench/queries/q12.ppl new file mode 100644 index 00000000000..fd78378c362 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q12.ppl @@ -0,0 +1,10 @@ +/* +SELECT MobilePhone, MobilePhoneModel, COUNT(DISTINCT UserID) AS u +FROM hits WHERE MobilePhoneModel <> '' +GROUP BY MobilePhone, MobilePhoneModel ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where MobilePhoneModel != '' +| stats dc(UserID) as u by MobilePhone, MobilePhoneModel +| sort - u +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q13.ppl b/integ-test/src/test/resources/clickbench/queries/q13.ppl new file mode 100644 index 00000000000..deaad26c24b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q13.ppl @@ -0,0 +1,9 @@ +/* +SELECT SearchPhrase, COUNT(*) AS c FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats count() as c by SearchPhrase +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q14.ppl b/integ-test/src/test/resources/clickbench/queries/q14.ppl new file mode 100644 index 00000000000..80b896bdbd4 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q14.ppl @@ -0,0 +1,10 @@ +/* +SELECT SearchPhrase, COUNT(DISTINCT UserID) AS u +FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY u DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats dc(UserID) as u by SearchPhrase +| sort - u +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q15.ppl b/integ-test/src/test/resources/clickbench/queries/q15.ppl new file mode 100644 index 00000000000..44cbd81b0ab --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q15.ppl @@ -0,0 +1,10 @@ +/* +SELECT SearchEngineID, SearchPhrase, COUNT(*) AS c +FROM hits WHERE SearchPhrase <> '' +GROUP BY SearchEngineID, SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats count() as c by SearchEngineID, SearchPhrase +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q16.ppl b/integ-test/src/test/resources/clickbench/queries/q16.ppl new file mode 100644 index 00000000000..157febefec4 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q16.ppl @@ -0,0 +1,7 @@ +/* +SELECT UserID, COUNT(*) FROM hits GROUP BY UserID ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| stats count() by UserID +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q17.ppl b/integ-test/src/test/resources/clickbench/queries/q17.ppl new file mode 100644 index 00000000000..3dfc82c236a --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q17.ppl @@ -0,0 +1,8 @@ +/* +SELECT UserID, SearchPhrase, COUNT(*) +FROM hits GROUP BY UserID, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| stats count() by UserID, SearchPhrase +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q18.ppl b/integ-test/src/test/resources/clickbench/queries/q18.ppl new file mode 100644 index 00000000000..38b77a5a565 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q18.ppl @@ -0,0 +1,6 @@ +/* +SELECT UserID, SearchPhrase, COUNT(*) FROM hits GROUP BY UserID, SearchPhrase LIMIT 10; +*/ +source=hits +| stats count() by UserID, SearchPhrase +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q19.ppl b/integ-test/src/test/resources/clickbench/queries/q19.ppl new file mode 100644 index 00000000000..edd852f528b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q19.ppl @@ -0,0 +1,9 @@ +/* +SELECT UserID, extract(minute FROM EventTime) AS m, SearchPhrase, COUNT(*) +FROM hits GROUP BY UserID, m, SearchPhrase ORDER BY COUNT(*) DESC LIMIT 10; +*/ +source=hits +| eval m = extract(minute from EventTime) +| stats count() by UserID, m, SearchPhrase +| sort - `count()` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q2.ppl b/integ-test/src/test/resources/clickbench/queries/q2.ppl new file mode 100644 index 00000000000..4ddf419ec86 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q2.ppl @@ -0,0 +1,4 @@ +/* +SELECT COUNT(*) FROM hits WHERE AdvEngineID <> 0; +*/ +source=hits | where AdvEngineID!=0 | stats count() \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q20.ppl b/integ-test/src/test/resources/clickbench/queries/q20.ppl new file mode 100644 index 00000000000..fb214b4bf81 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q20.ppl @@ -0,0 +1,6 @@ +/* +SELECT UserID FROM hits WHERE UserID = 435090932899640449; +*/ +source=hits +| where UserID = 435090932899640449 +| fields UserID \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q21.ppl b/integ-test/src/test/resources/clickbench/queries/q21.ppl new file mode 100644 index 00000000000..657b0e1fc30 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q21.ppl @@ -0,0 +1,6 @@ +/* +SELECT COUNT(*) FROM hits WHERE URL LIKE '%google%'; +*/ +source=hits +| where like(URL, '%google%') +| stats count() \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q22.ppl b/integ-test/src/test/resources/clickbench/queries/q22.ppl new file mode 100644 index 00000000000..3319cac2a9b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q22.ppl @@ -0,0 +1,10 @@ +/* +SELECT SearchPhrase, MIN(URL), COUNT(*) AS c +FROM hits WHERE URL LIKE '%google%' AND SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where like(URL, '%google%') and SearchPhrase != '' +| stats /* min(URL), */ count() as c by SearchPhrase +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q23.ppl b/integ-test/src/test/resources/clickbench/queries/q23.ppl new file mode 100644 index 00000000000..d5c6de41cec --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q23.ppl @@ -0,0 +1,10 @@ +/* +SELECT SearchPhrase, MIN(URL), MIN(Title), COUNT(*) AS c, COUNT(DISTINCT UserID) +FROM hits WHERE Title LIKE '%Google%' AND URL NOT LIKE '%.google.%' AND SearchPhrase <> '' +GROUP BY SearchPhrase ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where like(Title, '%Google%') and not like(URL, '%.google.%') and SearchPhrase != '' +| stats /* min(URL), min(Title), */ count() as c, dc(UserID) by SearchPhrase +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q24.ppl b/integ-test/src/test/resources/clickbench/queries/q24.ppl new file mode 100644 index 00000000000..1323546a687 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q24.ppl @@ -0,0 +1,7 @@ +/* +SELECT * FROM hits WHERE URL LIKE '%google%' ORDER BY EventTime LIMIT 10; +*/ +source=hits +| where like(URL, '%google%') +| sort EventTime +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q25.ppl b/integ-test/src/test/resources/clickbench/queries/q25.ppl new file mode 100644 index 00000000000..c7fbc087ac1 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q25.ppl @@ -0,0 +1,8 @@ +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| sort EventTime +| fields SearchPhrase +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q26.ppl b/integ-test/src/test/resources/clickbench/queries/q26.ppl new file mode 100644 index 00000000000..00060dc105f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q26.ppl @@ -0,0 +1,8 @@ +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY SearchPhrase LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| fields SearchPhrase +| sort SearchPhrase +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q27.ppl b/integ-test/src/test/resources/clickbench/queries/q27.ppl new file mode 100644 index 00000000000..502d0e06f7d --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q27.ppl @@ -0,0 +1,8 @@ +/* +SELECT SearchPhrase FROM hits WHERE SearchPhrase <> '' ORDER BY EventTime, SearchPhrase LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| sort EventTime, SearchPhrase +| fields SearchPhrase +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q28.ppl b/integ-test/src/test/resources/clickbench/queries/q28.ppl new file mode 100644 index 00000000000..925b19fb328 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q28.ppl @@ -0,0 +1,10 @@ +/* +SELECT CounterID, AVG(length(URL)) AS l, COUNT(*) AS c +FROM hits WHERE URL <> '' GROUP BY CounterID HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +*/ +source=hits +| where URL != '' +| stats avg(length(URL)) as l, count() as c by CounterID +| where c > 100000 +| sort - l +| head 25 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q29.ppl b/integ-test/src/test/resources/clickbench/queries/q29.ppl new file mode 100644 index 00000000000..5de7dea5ff0 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q29.ppl @@ -0,0 +1,12 @@ +/* +SELECT REGEXP_REPLACE(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') AS k, +AVG(length(Referer)) AS l, COUNT(*) AS c, MIN(Referer) +FROM hits WHERE Referer <> '' GROUP BY k HAVING COUNT(*) > 100000 ORDER BY l DESC LIMIT 25; +*/ +source=hits +| Referer != '' +| eval k = regexp_replace(Referer, '^https?://(?:www\.)?([^/]+)/.*$', '\1') +| stats avg(length(Referer)) as l, count() as c, min(Referer) by k +| where c > 100000 +| sort - l +| head 25 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q3.ppl b/integ-test/src/test/resources/clickbench/queries/q3.ppl new file mode 100644 index 00000000000..26fa29cd913 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q3.ppl @@ -0,0 +1,4 @@ +/* +SELECT SUM(AdvEngineID), COUNT(*), AVG(ResolutionWidth) FROM hits; +*/ +source=hits | stats sum(AdvEngineID), count() , avg(ResolutionWidth) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q30.ppl b/integ-test/src/test/resources/clickbench/queries/q30.ppl new file mode 100644 index 00000000000..504340e38e7 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q30.ppl @@ -0,0 +1,95 @@ +/* +SELECT SUM(ResolutionWidth), SUM(ResolutionWidth + 1), ... SUM(ResolutionWidth + 89) FROM hits; +*/ +source=hits +| stats + sum(ResolutionWidth), + sum(ResolutionWidth+1), + sum(ResolutionWidth+2), + sum(ResolutionWidth+3), + sum(ResolutionWidth+4), + sum(ResolutionWidth+5), + sum(ResolutionWidth+6), + sum(ResolutionWidth+7), + sum(ResolutionWidth+8), + sum(ResolutionWidth+9), + sum(ResolutionWidth+10), + sum(ResolutionWidth+11), + sum(ResolutionWidth+12), + sum(ResolutionWidth+13), + sum(ResolutionWidth+14), + sum(ResolutionWidth+15), + sum(ResolutionWidth+16), + sum(ResolutionWidth+17), + sum(ResolutionWidth+18), + sum(ResolutionWidth+19), + sum(ResolutionWidth+20), + sum(ResolutionWidth+21), + sum(ResolutionWidth+22), + sum(ResolutionWidth+23), + sum(ResolutionWidth+24), + sum(ResolutionWidth+25), + sum(ResolutionWidth+26), + sum(ResolutionWidth+27), + sum(ResolutionWidth+28), + sum(ResolutionWidth+29), + sum(ResolutionWidth+30), + sum(ResolutionWidth+31), + sum(ResolutionWidth+32), + sum(ResolutionWidth+33), + sum(ResolutionWidth+34), + sum(ResolutionWidth+35), + sum(ResolutionWidth+36), + sum(ResolutionWidth+37), + sum(ResolutionWidth+38), + sum(ResolutionWidth+39), + sum(ResolutionWidth+40), + sum(ResolutionWidth+41), + sum(ResolutionWidth+42), + sum(ResolutionWidth+43), + sum(ResolutionWidth+44), + sum(ResolutionWidth+45), + sum(ResolutionWidth+46), + sum(ResolutionWidth+47), + sum(ResolutionWidth+48), + sum(ResolutionWidth+49), + sum(ResolutionWidth+50), + sum(ResolutionWidth+51), + sum(ResolutionWidth+52), + sum(ResolutionWidth+53), + sum(ResolutionWidth+54), + sum(ResolutionWidth+55), + sum(ResolutionWidth+56), + sum(ResolutionWidth+57), + sum(ResolutionWidth+58), + sum(ResolutionWidth+59), + sum(ResolutionWidth+60), + sum(ResolutionWidth+61), + sum(ResolutionWidth+62), + sum(ResolutionWidth+63), + sum(ResolutionWidth+64), + sum(ResolutionWidth+65), + sum(ResolutionWidth+66), + sum(ResolutionWidth+67), + sum(ResolutionWidth+68), + sum(ResolutionWidth+69), + sum(ResolutionWidth+70), + sum(ResolutionWidth+71), + sum(ResolutionWidth+72), + sum(ResolutionWidth+73), + sum(ResolutionWidth+74), + sum(ResolutionWidth+75), + sum(ResolutionWidth+76), + sum(ResolutionWidth+77), + sum(ResolutionWidth+78), + sum(ResolutionWidth+79), + sum(ResolutionWidth+80), + sum(ResolutionWidth+81), + sum(ResolutionWidth+82), + sum(ResolutionWidth+83), + sum(ResolutionWidth+84), + sum(ResolutionWidth+85), + sum(ResolutionWidth+86), + sum(ResolutionWidth+87), + sum(ResolutionWidth+88), + sum(ResolutionWidth+89) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q31.ppl b/integ-test/src/test/resources/clickbench/queries/q31.ppl new file mode 100644 index 00000000000..1cff19bac32 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q31.ppl @@ -0,0 +1,9 @@ +/* +SELECT SearchEngineID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits WHERE SearchPhrase <> '' GROUP BY SearchEngineID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by SearchEngineID, ClientIP +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q32.ppl b/integ-test/src/test/resources/clickbench/queries/q32.ppl new file mode 100644 index 00000000000..1a9c7214048 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q32.ppl @@ -0,0 +1,9 @@ +/* +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits WHERE SearchPhrase <> '' GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| where SearchPhrase != '' +| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q33.ppl b/integ-test/src/test/resources/clickbench/queries/q33.ppl new file mode 100644 index 00000000000..06ff6329889 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q33.ppl @@ -0,0 +1,8 @@ +/* +SELECT WatchID, ClientIP, COUNT(*) AS c, SUM(IsRefresh), AVG(ResolutionWidth) +FROM hits GROUP BY WatchID, ClientIP ORDER BY c DESC LIMIT 10; +*/ +source=hits +| stats count() as c, sum(IsRefresh), avg(ResolutionWidth) by WatchID, ClientIP +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q34.ppl b/integ-test/src/test/resources/clickbench/queries/q34.ppl new file mode 100644 index 00000000000..b6813b38db6 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q34.ppl @@ -0,0 +1,7 @@ +/* +SELECT URL, COUNT(*) AS c FROM hits GROUP BY URL ORDER BY c DESC LIMIT 10; +*/ +source=hits +| stats count() as c by URL +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q35.ppl b/integ-test/src/test/resources/clickbench/queries/q35.ppl new file mode 100644 index 00000000000..e52640b9dbf --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q35.ppl @@ -0,0 +1,8 @@ +/* +SELECT 1, URL, COUNT(*) AS c FROM hits GROUP BY 1, URL ORDER BY c DESC LIMIT 10; +*/ +source=hits +| eval const = 1 +| stats count() as c by const, URL +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q36.ppl b/integ-test/src/test/resources/clickbench/queries/q36.ppl new file mode 100644 index 00000000000..78f89060945 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q36.ppl @@ -0,0 +1,9 @@ +/* +SELECT ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3, COUNT(*) AS c +FROM hits GROUP BY ClientIP, ClientIP - 1, ClientIP - 2, ClientIP - 3 ORDER BY c DESC LIMIT 10; +*/ +source=hits +| eval `ClientIP - 1` = ClientIP - 1, `ClientIP - 2` = ClientIP - 2, `ClientIP - 3` = ClientIP - 3 +| stats count() as c by `ClientIP`, `ClientIP - 1`, `ClientIP - 2`, `ClientIP - 3` +| sort - c +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q37.ppl b/integ-test/src/test/resources/clickbench/queries/q37.ppl new file mode 100644 index 00000000000..7fe4dc49486 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q37.ppl @@ -0,0 +1,11 @@ +/* +SELECT URL, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND DontCountHits = 0 AND IsRefresh = 0 AND URL <> '' +GROUP BY URL ORDER BY PageViews DESC LIMIT 10; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and URL != '' +| stats count() as PageViews by URL +| sort - PageViews +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q38.ppl b/integ-test/src/test/resources/clickbench/queries/q38.ppl new file mode 100644 index 00000000000..becd5a49a91 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q38.ppl @@ -0,0 +1,11 @@ +/* +SELECT Title, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND DontCountHits = 0 AND IsRefresh = 0 AND Title <> '' +GROUP BY Title ORDER BY PageViews DESC LIMIT 10; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and DontCountHits = 0 and IsRefresh = 0 and Title != '' +| stats count() as PageViews by Title +| sort - PageViews +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q39.ppl b/integ-test/src/test/resources/clickbench/queries/q39.ppl new file mode 100644 index 00000000000..141707fc0a9 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q39.ppl @@ -0,0 +1,11 @@ +/* +SELECT URL, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND IsLink <> 0 AND IsDownload = 0 +GROUP BY URL ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and IsLink != 0 and IsDownload = 0 +| stats count() as PageViews by URL +| sort - PageViews +| head 10 from 1000 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q4.ppl b/integ-test/src/test/resources/clickbench/queries/q4.ppl new file mode 100644 index 00000000000..f783154e9ab --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q4.ppl @@ -0,0 +1,4 @@ +/* +SELECT AVG(UserID) FROM hits; +*/ +source=hits | stats avg(UserID) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q40.ppl b/integ-test/src/test/resources/clickbench/queries/q40.ppl new file mode 100644 index 00000000000..a3481eb2fac --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q40.ppl @@ -0,0 +1,11 @@ +/* +SELECT TraficSourceID, SearchEngineID, AdvEngineID, CASE WHEN (SearchEngineID = 0 AND AdvEngineID = 0) THEN Referer ELSE '' END AS Src, URL AS Dst, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' AND IsRefresh = 0 +GROUP BY TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst ORDER BY PageViews DESC LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 +| eval Src=case(SearchEngineID = 0 and AdvEngineID = 0, Referer else ''), Dst=URL +| stats count() as PageViews by TraficSourceID, SearchEngineID, AdvEngineID, Src, Dst +| sort - PageViews +| head 10 from 1000 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q41.ppl b/integ-test/src/test/resources/clickbench/queries/q41.ppl new file mode 100644 index 00000000000..13028d744bc --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q41.ppl @@ -0,0 +1,11 @@ +/* +SELECT URLHash, EventDate, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND TraficSourceID IN (-1, 6) AND RefererHash = 3594120000172545465 +GROUP BY URLHash, EventDate ORDER BY PageViews DESC LIMIT 10 OFFSET 100; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and TraficSourceID in (-1, 6) and RefererHash = 3594120000172545465 +| stats count() as PageViews by URLHash, EventDate +| sort - PageViews +| head 10 from 100 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q42.ppl b/integ-test/src/test/resources/clickbench/queries/q42.ppl new file mode 100644 index 00000000000..0d5f034c129 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q42.ppl @@ -0,0 +1,11 @@ +/* +SELECT WindowClientWidth, WindowClientHeight, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-01' AND EventDate <= '2013-07-31' +AND IsRefresh = 0 AND DontCountHits = 0 AND URLHash = 2868770270353813622 +GROUP BY WindowClientWidth, WindowClientHeight ORDER BY PageViews DESC LIMIT 10 OFFSET 10000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-31 00:00:00' and IsRefresh = 0 and DontCountHits = 0 and URLHash = 2868770270353813622 +| stats count() as PageViews by WindowClientWidth, WindowClientHeight +| sort - PageViews +| head 10 from 10000 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q43.ppl b/integ-test/src/test/resources/clickbench/queries/q43.ppl new file mode 100644 index 00000000000..b0a697a8367 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q43.ppl @@ -0,0 +1,14 @@ +/* +SELECT DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') AS M, COUNT(*) AS PageViews +FROM hits WHERE CounterID = 62 AND EventDate >= '2013-07-14' AND EventDate <= '2013-07-15' +AND IsRefresh = 0 AND DontCountHits = 0 +GROUP BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') +ORDER BY DATE_FORMAT(EventTime, '%Y-%m-%d %H:00:00') +LIMIT 10 OFFSET 1000; +*/ +source=hits +| where CounterID = 62 and EventDate >= '2013-07-01 00:00:00' and EventDate <= '2013-07-15 00:00:00' and IsRefresh = 0 and DontCountHits = 0 +| eval M = date_format(EventTime, '%Y-%m-%d %H:00:00') +| stats count() as PageViews by M +| sort M +| head 10 from 1000 \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q5.ppl b/integ-test/src/test/resources/clickbench/queries/q5.ppl new file mode 100644 index 00000000000..fde9e9ffcb3 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q5.ppl @@ -0,0 +1,4 @@ +/* +SELECT COUNT(DISTINCT UserID) FROM hits; +*/ +source=hits | stats dc(UserID) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q6.ppl b/integ-test/src/test/resources/clickbench/queries/q6.ppl new file mode 100644 index 00000000000..9863890cb1f --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q6.ppl @@ -0,0 +1,4 @@ +/* +SELECT COUNT(DISTINCT SearchPhrase) FROM hits; +*/ +source=hits | stats dc(SearchPhrase) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q7.ppl b/integ-test/src/test/resources/clickbench/queries/q7.ppl new file mode 100644 index 00000000000..d11949fd07b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q7.ppl @@ -0,0 +1,4 @@ +/* +SELECT MIN(EventDate), MAX(EventDate) FROM hits; +*/ +source=hits | stats min(EventDate), max(EventDate) \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q8.ppl b/integ-test/src/test/resources/clickbench/queries/q8.ppl new file mode 100644 index 00000000000..b61f73a805b --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q8.ppl @@ -0,0 +1,4 @@ +/* +SELECT AdvEngineID, COUNT(*) FROM hits WHERE AdvEngineID <> 0 GROUP BY AdvEngineID ORDER BY COUNT(*) DESC; +*/ +source=hits | where AdvEngineID!=0 | stats count() by AdvEngineID | sort - `count()` \ No newline at end of file diff --git a/integ-test/src/test/resources/clickbench/queries/q9.ppl b/integ-test/src/test/resources/clickbench/queries/q9.ppl new file mode 100644 index 00000000000..aa6f08f8ce0 --- /dev/null +++ b/integ-test/src/test/resources/clickbench/queries/q9.ppl @@ -0,0 +1,4 @@ +/* +SELECT RegionID, COUNT(DISTINCT UserID) AS u FROM hits GROUP BY RegionID ORDER BY u DESC LIMIT 10; +*/ +source=hits | stats dc(UserID) as u by RegionID | sort -u | head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json index 959d2c7c930..40e7ccbd146 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_multi_range.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalProject(age=[$8])\n LogicalFilter(condition=[SEARCH($8, Sarg[0, [1..10], (20..30)])])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[0, [1..10], (20..30)])], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\":{\"age\":{\"value\":0,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":1,\"to\":10,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":20,\"to\":30,\"include_lower\":false,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[0, [1..10], (20..30)])], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"should\":[{\"term\":{\"age\":{\"value\":0.0,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":1.0,\"to\":10.0,\"include_lower\":true,\"include_upper\":true,\"boost\":1.0}}},{\"range\":{\"age\":{\"from\":20.0,\"to\":30.0,\"include_lower\":false,\"include_upper\":false,\"boost\":1.0}}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.json index 25491790385..63f788a53aa 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sarg_filter_push_single_range.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalProject(age=[$8])\n LogicalFilter(condition=[SEARCH($8, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":1.0,\"to\":10,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age], FILTER->SEARCH($0, Sarg[[1.0:DECIMAL(11, 1)..10:DECIMAL(11, 1))]:DECIMAL(11, 1))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":1.0,\"to\":10.0,\"include_lower\":true,\"include_upper\":false,\"boost\":1.0}}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} +} \ No newline at end of file diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index ce2aae6c8bd..0e075300261 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -545,7 +545,7 @@ private QueryExpression binary(RexCall call) { range -> RangeSets.isPoint(range) ? QueryExpression.create(pair.getKey()) - .equals(range.lowerEndpoint(), isTimeStamp) + .equals(sargPointValue(range.lowerEndpoint()), isTimeStamp) : QueryExpression.create(pair.getKey()).between(range, isTimeStamp)) .collect(Collectors.toList()); if (queryExpressions.size() == 1) { @@ -1199,7 +1199,7 @@ public QueryExpression between(Range range, boolean isTimeStamp) { } private Object convertEndpointValue(Object value, boolean isTimeStamp) { - value = (value instanceof NlsString) ? ((NlsString)value).getValue() : value; + value = sargPointValue(value); return isTimeStamp ? timestampValueForPushDown(value.toString()) : value; } } @@ -1413,36 +1413,37 @@ String stringValue() { List sargValue() { final Sarg sarg = requireNonNull(literal.getValueAs(Sarg.class), "Sarg"); - final RelDataType type = literal.getType(); List values = new ArrayList<>(); - final SqlTypeName sqlTypeName = type.getSqlTypeName(); if (sarg.isPoints()) { Set ranges = sarg.rangeSet.asRanges(); - ranges.forEach(range -> values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName))); + ranges.forEach(range -> values.add(sargPointValue(range.lowerEndpoint()))); } else if (sarg.isComplementedPoints()) { Set ranges = sarg.negate().rangeSet.asRanges(); - ranges.forEach(range -> values.add(sargPointValue(range.lowerEndpoint(), sqlTypeName))); + ranges.forEach(range -> values.add(sargPointValue(range.lowerEndpoint()))); } return values; } - Object sargPointValue(Object point, SqlTypeName sqlTypeName) { - switch (sqlTypeName) { - case CHAR: - case VARCHAR: - return ((NlsString) point).getValue(); - case DECIMAL: - return ((BigDecimal) point).doubleValue(); - default: - return point; - } - } - Object rawValue() { return literal.getValue(); } } + /** + * If the sarg point is a NlsString, we should get the value from it. For BigDecimal type, we + * should get the double value from the literal. That's because there is no decimal type in + * OpenSearch. + */ + public static Object sargPointValue(Object point) { + if (point instanceof NlsString) { + return ((NlsString) point).getValue(); + } else if (point instanceof BigDecimal) { + return ((BigDecimal) point).doubleValue(); + } else { + return point; + } + } + /** * If one operand in a binary operator is a DateTime type, but the other isn't, we should not push * down the predicate. diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java index 5e144b23e54..9618b4db8a7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/PredicateAnalyzerTest.java @@ -258,9 +258,9 @@ void search_generatesTermsQuery() throws ExpressionNotAnalyzableException { "{\n" + " \"terms\" : {\n" + " \"a\" : [\n" + - " 12,\n" + - " 13,\n" + - " 14\n" + + " 12.0,\n" + + " 13.0,\n" + + " 14.0\n" + " ],\n" + " \"boost\" : 1.0\n" + " }\n" +