diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java new file mode 100644 index 00000000000..43df4f0f6e3 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/CalcitePPLBig5IT.java @@ -0,0 +1,26 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.big5; + +import org.junit.FixMethodOrder; +import org.junit.runners.MethodSorters; + +@FixMethodOrder(MethodSorters.JVM) +public class CalcitePPLBig5IT extends PPLBig5IT { + private boolean initialized = false; + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + disallowCalciteFallback(); + // warm-up + if (!initialized) { + executeQuery("source=big5 | join on 1=1 big5"); // trigger non-pushdown + initialized = true; + } + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java new file mode 100644 index 00000000000..cf01704870e --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/big5/PPLBig5IT.java @@ -0,0 +1,317 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.big5; + +import java.io.IOException; +import java.util.LinkedHashMap; +import java.util.Locale; +import java.util.Map; +import org.junit.AfterClass; +import org.junit.FixMethodOrder; +import org.junit.Ignore; +import org.junit.Test; +import org.junit.runners.MethodSorters; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +@FixMethodOrder(MethodSorters.JVM) +public class PPLBig5IT extends PPLIntegTestCase { + private boolean initialized = false; + private static final Map summary = new LinkedHashMap<>(); + + @Override + public void init() throws Exception { + super.init(); + loadIndex(Index.BIG5); + disableCalcite(); + // warm-up + if (!initialized) { + executeQuery("source=big5 | eval a = 1"); // trigger non-pushdown + initialized = true; + } + } + + @AfterClass + public static void reset() throws IOException { + long total = 0; + for (long duration : summary.values()) { + total += duration; + } + System.out.println("Summary:"); + for (Map.Entry entry : summary.entrySet()) { + System.out.printf(Locale.ENGLISH, "%s: %d ms%n", entry.getKey(), entry.getValue()); + } + System.out.printf( + Locale.ENGLISH, + "Total %d queries succeed. Average duration: %d ms%n", + summary.size(), + total / summary.size()); + System.out.println(); + summary.clear(); + } + + protected void timing(String query, String ppl) throws IOException { + long start = System.currentTimeMillis(); + executeQuery(ppl); + long duration = System.currentTimeMillis() - start; + summary.put(query, duration); + } + + @Test + public void asc_sort_timestamp() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp.ppl")); + timing("asc_sort_timestamp", ppl); + } + + @Test + public void asc_sort_timestamp_can_match_shortcut() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/asc_sort_timestamp_can_match_shortcut.ppl")); + timing("asc_sort_timestamp_can_match_shortcut", ppl); + } + + @Test + public void asc_sort_timestamp_no_can_match_shortcut() throws IOException { + String ppl = + sanitize(loadFromFile("big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl")); + timing("asc_sort_timestamp_no_can_match_shortcut", ppl); + } + + @Test + public void asc_sort_with_after_timestamp() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/asc_sort_with_after_timestamp.ppl")); + timing("asc_sort_with_after_timestamp", ppl); + } + + @Test + public void composite_date_histogram_daily() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/composite_date_histogram_daily.ppl")); + timing("composite_date_histogram_daily", ppl); + } + + @Test + public void composite_terms_keyword() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/composite_terms_keyword.ppl")); + timing("composite_terms_keyword", ppl); + } + + @Test + public void composite_terms() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/composite_terms.ppl")); + timing("composite_terms", ppl); + } + + @Test + public void date_histogram_hourly_agg() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/date_histogram_hourly_agg.ppl")); + timing("date_histogram_hourly_agg", ppl); + } + + @Test + public void date_histogram_minute_agg() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/date_histogram_minute_agg.ppl")); + timing("date_histogram_minute_agg", ppl); + } + + @Test + public void test_default() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/default.ppl")); + timing("default", ppl); + } + + @Test + public void desc_sort_timestamp() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp.ppl")); + timing("desc_sort_timestamp", ppl); + } + + @Test + public void desc_sort_timestamp_can_match_shortcut() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/desc_sort_timestamp_can_match_shortcut.ppl")); + timing("desc_sort_timestamp_can_match_shortcut", ppl); + } + + @Test + public void desc_sort_timestamp_no_can_match_shortcut() throws IOException { + String ppl = + sanitize(loadFromFile("big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl")); + timing("desc_sort_timestamp_no_can_match_shortcut", ppl); + } + + @Test + public void desc_sort_with_after_timestamp() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/desc_sort_with_after_timestamp.ppl")); + timing("desc_sort_with_after_timestamp", ppl); + } + + @Test + public void keyword_in_range() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/keyword_in_range.ppl")); + timing("keyword_in_range", ppl); + } + + @Test + public void keyword_terms() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/keyword_terms.ppl")); + timing("keyword_terms", ppl); + } + + @Test + public void keyword_terms_low_cardinality() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/keyword_terms_low_cardinality.ppl")); + timing("keyword_terms_low_cardinality", ppl); + } + + @Test + public void multi_terms_keyword() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/multi_terms_keyword.ppl")); + timing("multi_terms_keyword", ppl); + } + + @Test + public void query_string_on_message() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message.ppl")); + timing("query_string_on_message", ppl); + } + + @Test + public void query_string_on_message_filtered() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/query_string_on_message_filtered.ppl")); + timing("query_string_on_message_filtered", ppl); + } + + @Test + public void query_string_on_message_filtered_sorted_num() throws IOException { + String ppl = + sanitize(loadFromFile("big5/queries/query_string_on_message_filtered_sorted_num.ppl")); + timing("query_string_on_message_filtered_sorted_num", ppl); + } + + @Test + public void range() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range.ppl")); + timing("range", ppl); + } + + @Ignore("Failed to parse request payload") + public void range_auto_date_histo() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo.ppl")); + timing("range_auto_date_histo", ppl); + } + + @Ignore("Failed to parse request payload") + public void range_auto_date_histo_with_metrics() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range_auto_date_histo_with_metrics.ppl")); + timing("range_auto_date_histo_with_metrics", ppl); + } + + @Test + public void range_numeric() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range_numeric.ppl")); + timing("range_numeric", ppl); + } + + @Test + public void range_field_conjunction_big_range_big_term_query() throws IOException { + String ppl = + sanitize(loadFromFile("big5/queries/range_field_conjunction_big_range_big_term_query.ppl")); + timing("range_field_conjunction_big_range_big_term_query", ppl); + } + + @Test + public void range_field_conjunction_small_range_big_term_query() throws IOException { + String ppl = + sanitize( + loadFromFile("big5/queries/range_field_conjunction_small_range_big_term_query.ppl")); + timing("range_field_conjunction_small_range_big_term_query", ppl); + } + + @Test + public void range_field_conjunction_small_range_small_term_query() throws IOException { + String ppl = + sanitize( + loadFromFile("big5/queries/range_field_conjunction_small_range_small_term_query.ppl")); + timing("range_field_conjunction_small_range_small_term_query", ppl); + } + + @Test + public void range_field_disjunction_big_range_small_term_query() throws IOException { + String ppl = + sanitize( + loadFromFile("big5/queries/range_field_disjunction_big_range_small_term_query.ppl")); + timing("range_field_disjunction_big_range_small_term_query", ppl); + } + + @Test + public void range_with_asc_sort() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range_with_asc_sort.ppl")); + timing("range_with_asc_sort", ppl); + } + + @Test + public void range_with_desc_sort() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/range_with_desc_sort.ppl")); + timing("range_with_desc_sort", ppl); + } + + @Test + public void scroll() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/scroll.ppl")); + timing("scroll", ppl); + } + + @Test + public void sort_keyword_can_match_shortcut() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_can_match_shortcut.ppl")); + timing("sort_keyword_can_match_shortcut", ppl); + } + + @Test + public void sort_keyword_no_can_match_shortcut() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_keyword_no_can_match_shortcut.ppl")); + timing("sort_keyword_no_can_match_shortcut", ppl); + } + + @Test + public void sort_numeric_asc() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc.ppl")); + timing("sort_numeric_asc", ppl); + } + + @Test + public void sort_numeric_asc_with_match() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_asc_with_match.ppl")); + timing("sort_numeric_asc_with_match", ppl); + } + + @Test + public void sort_numeric_desc() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc.ppl")); + timing("sort_numeric_desc", ppl); + } + + @Test + public void sort_numeric_desc_with_match() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/sort_numeric_desc_with_match.ppl")); + timing("sort_numeric_desc_with_match", ppl); + } + + @Test + public void term() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/term.ppl")); + timing("term", ppl); + } + + @Test + public void terms_significant_1() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/terms_significant_1.ppl")); + timing("terms_significant_1", ppl); + } + + @Test + public void terms_significant_2() throws IOException { + String ppl = sanitize(loadFromFile("big5/queries/terms_significant_2.ppl")); + timing("terms_significant_2", ppl); + } +} diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java index f8ec438de4c..18a95c4ac73 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/tpch/CalcitePPLTpchIT.java @@ -454,15 +454,4 @@ public void testQ22() throws IOException { rows("30", 1, 7638.57), rows("31", 1, 9331.13)); } - - /** - * Sanitizes the PPL query by removing block comments and replacing new lines with spaces. - * - * @param ppl the PPL query string - * @return the sanitized PPL query string - */ - private static String sanitize(String ppl) { - String withoutComments = ppl.replaceAll("(?s)/\\*.*?\\*/", ""); - return withoutComments.replaceAll("\\r\\n", " ").replaceAll("\\n", " ").trim(); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index e7086abb9a9..d52c99f1250 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -12,6 +12,7 @@ import static org.opensearch.sql.legacy.TestUtils.getArrayIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getBankWithNullValuesIndexMapping; +import static org.opensearch.sql.legacy.TestUtils.getBig5MappingFile; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNonnumericIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDataTypeNumericIndexMapping; import static org.opensearch.sql.legacy.TestUtils.getDateIndexMapping; @@ -901,6 +902,11 @@ public enum Index { "tpch", getTpchMappingFile("customer_index_mapping.json"), "src/test/resources/tpch/data/customer.json"), + BIG5( + "big5", + "big5", + getBig5MappingFile("big5_index_mapping.json"), + "src/test/resources/big5/data/big5.json"), ARRAY( TestsConstants.TEST_INDEX_ARRAY, "array", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index cf8c2282836..770be2ebf9a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -477,4 +477,12 @@ public static String getTpchMappingFile(String fileName) { return null; } } + + public static String getBig5MappingFile(String fileName) { + try { + return TestUtils.fileToString("src/test/resources/big5/mappings/" + fileName, false); + } catch (IOException e) { + return null; + } + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index e7bbc5b035d..2b3f0756b08 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -254,6 +254,17 @@ public void updatePushdownSettings() throws IOException { } } + /** + * Sanitizes the PPL query by removing block comments and replacing new lines with spaces. + * + * @param ppl the PPL query string + * @return the sanitized PPL query string + */ + protected static String sanitize(String ppl) { + String withoutComments = ppl.replaceAll("(?s)/\\*.*?\\*/", ""); + return withoutComments.replaceAll("\\r\\n", " ").replaceAll("\\n", " ").trim(); + } + // Utility methods /** diff --git a/integ-test/src/test/resources/big5/data/big5.json b/integ-test/src/test/resources/big5/data/big5.json new file mode 100644 index 00000000000..30a6a81ab8e --- /dev/null +++ b/integ-test/src/test/resources/big5/data/big5.json @@ -0,0 +1,2 @@ +{"index":{}} +{"message":"2023-04-30T21:48:56.160Z Apr 30 21:48:56 ip-66-221-134-40 journal: donkey glazer fly shark whip servant thornfalcon","process":{"name":"journal"},"aws.cloudwatch":{"ingestion_time":"2023-04-30T21:48:56.160Z","log_group":"/var/log/messages","log_stream":"luckcrafter"},"tags":["preserve_original_event"],"meta":{"file":"2023-01-02/1682891301-gotext.ndjson.gz"},"cloud":{"region":"eu-central-1"},"@timestamp":"2023-01-02T22:02:34.000Z","input":{"type":"aws-cloudwatch"},"metrics":{"tmin":849,"size":1981},"log.file.path":"/var/log/messages/luckcrafter","event":{"id":"sunsetmark","dataset":"generic","ingested":"2023-07-20T03:36:30.223806Z"},"agent":{"id":"c315dc22-3ea6-44dc-8d56-fd02f675367b","name":"fancydancer","ephemeral_id":"c315dc22-3ea6-44dc-8d56-fd02f675367b","type":"filebeat","version":"8.8.0"}} diff --git a/integ-test/src/test/resources/big5/mappings/big5_index_mapping.json b/integ-test/src/test/resources/big5/mappings/big5_index_mapping.json new file mode 100644 index 00000000000..ac6ed187037 --- /dev/null +++ b/integ-test/src/test/resources/big5/mappings/big5_index_mapping.json @@ -0,0 +1,198 @@ +{ + "mappings": { + "_data_stream_timestamp": { + "enabled": true + }, + "dynamic_templates": [ + { + "match_ip": { + "match": "ip", + "match_mapping_type": "string", + "mapping": { + "type": "ip" + } + } + }, + { + "match_message": { + "match": "message", + "match_mapping_type": "string", + "mapping": { + "type": "match_only_text" + } + } + }, + { + "strings_as_keyword": { + "match_mapping_type": "string", + "mapping": { + "ignore_above": 1024, + "type": "keyword" + } + } + } + ], + "date_detection": false, + "properties": { + "@timestamp": { + "type": "date" + }, + "agent": { + "type": "object", + "properties": { + "ephemeral_id": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "name": { + "type": "keyword", + "ignore_above": 1024 + }, + "type": { + "type": "keyword", + "ignore_above": 1024 + }, + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "aws": { + "type": "object", + "properties": { + "cloudwatch": { + "type": "object", + "properties": { + "ingestion_time": { + "type": "keyword", + "ignore_above": 1024 + }, + "log_group": { + "type": "keyword", + "ignore_above": 1024 + }, + "log_stream": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "cloud": { + "type": "object", + "properties": { + "region": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "data_stream": { + "properties": { + "dataset": { + "type": "keyword" + }, + "namespace": { + "type": "keyword" + }, + "type": { + "type": "keyword" + } + } + }, + "ecs": { + "type": "object", + "properties": { + "version": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "event": { + "type": "object", + "properties": { + "dataset": { + "type": "keyword", + "ignore_above": 1024 + }, + "id": { + "type": "keyword", + "ignore_above": 1024 + }, + "ingested": { + "type": "date" + } + } + }, + "host": { + "type": "object" + }, + "input": { + "type": "object", + "properties": { + "type": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "log": { + "type": "object", + "properties": { + "file": { + "type": "object", + "properties": { + "path": { + "type": "keyword", + "ignore_above": 1024 + } + } + } + } + }, + "message": { + "type": "match_only_text" + }, + "meta": { + "type": "object", + "properties": { + "file": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "metrics": { + "type": "object", + "properties": { + "size": { + "type": "long" + }, + "tmin": { + "type": "long" + } + } + }, + "process": { + "type": "object", + "properties": { + "name": { + "type": "keyword", + "ignore_above": 1024 + } + } + }, + "tags": { + "type": "keyword", + "ignore_above": 1024 + } + } + } +} diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl new file mode 100644 index 00000000000..7582d40d69a --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl new file mode 100644 index 00000000000..c0ca2fcd093 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl new file mode 100644 index 00000000000..c0ca2fcd093 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/asc_sort_timestamp_no_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl new file mode 100644 index 00000000000..7582d40d69a --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/asc_sort_with_after_timestamp.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl new file mode 100644 index 00000000000..caa27c86fba --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/composite_date_histogram_daily.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-07 12:00:00' +| stats count() by span(`@timestamp`, 1d) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms.ppl b/integ-test/src/test/resources/big5/queries/composite_terms.ppl new file mode 100644 index 00000000000..859e3c87e54 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/composite_terms.ppl @@ -0,0 +1,4 @@ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `process.name`, `cloud.region` +| sort - `process.name`, + `cloud.region` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl new file mode 100644 index 00000000000..5eb03e5fe2a --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/composite_terms_keyword.ppl @@ -0,0 +1,4 @@ +source = big5 +| where `@timestamp` >= '2023-01-02 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `process.name`, `cloud.region`, `aws.cloudwatch.log_stream` +| sort - `process.name`, + `cloud.region`, + `aws.cloudwatch.log_stream` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl new file mode 100644 index 00000000000..054b915b335 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/date_histogram_hourly_agg.ppl @@ -0,0 +1,2 @@ +source = big5 +| stats count() by span(`@timestamp`, 1h) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl new file mode 100644 index 00000000000..b9fd72abfb5 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/date_histogram_minute_agg.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by span(`@timestamp`, 1m) \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/default.ppl b/integ-test/src/test/resources/big5/queries/default.ppl new file mode 100644 index 00000000000..6b63c414ac0 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/default.ppl @@ -0,0 +1,2 @@ +source = big5 +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl new file mode 100644 index 00000000000..af3445efdff --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort - `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl new file mode 100644 index 00000000000..fabaae912e0 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort - `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl new file mode 100644 index 00000000000..fabaae912e0 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/desc_sort_timestamp_no_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort - `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl new file mode 100644 index 00000000000..af3445efdff --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/desc_sort_with_after_timestamp.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort - `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl new file mode 100644 index 00000000000..56f07779432 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/keyword_in_range.ppl @@ -0,0 +1,4 @@ +source = big5 match(`process.name`, 'kernel') +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` < '2023-01-03 00:00:00' +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl new file mode 100644 index 00000000000..99353b5299f --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/keyword_terms.ppl @@ -0,0 +1,4 @@ +source = big5 +| stats count() as station by `aws.cloudwatch.log_stream` +| sort - station +| head 500 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl new file mode 100644 index 00000000000..02e335723d3 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/keyword_terms_low_cardinality.ppl @@ -0,0 +1,4 @@ +source = big5 +| stats count() as country by `aws.cloudwatch.log_stream` +| sort - country +| head 100 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl new file mode 100644 index 00000000000..a148a8bbc90 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/multi_terms_keyword.ppl @@ -0,0 +1,4 @@ +source = big5 +| where `@timestamp` >= '2022-12-30 00:00:00' and `@timestamp` < '2023-01-01 03:00:00' +| stats count() by `process.name`, `event.id`, `cloud.region` +| sort - `count()` \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl new file mode 100644 index 00000000000..730600c1280 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message.ppl @@ -0,0 +1,2 @@ +source = big5 query_string(['message'], 'shield AND carp AND shark') +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl new file mode 100644 index 00000000000..8f00176fa75 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered.ppl @@ -0,0 +1,4 @@ +source = big5 query_string(['message'], 'shield carp shark', default_operator='AND') +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` < '2023-01-03 00:00:00' +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl new file mode 100644 index 00000000000..566a2ef8413 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/query_string_on_message_filtered_sorted_num.ppl @@ -0,0 +1,5 @@ +source = big5 query_string(['message'], 'shield AND carp AND shark') +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` < '2023-01-03 00:00:00' +| sort - `metrics.size` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range.ppl b/integ-test/src/test/resources/big5/queries/range.ppl new file mode 100644 index 00000000000..74eae492541 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' and `@timestamp` < '2023-01-03 00:00:00' +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl new file mode 100644 index 00000000000..528337f078d --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo.ppl @@ -0,0 +1,10 @@ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, "range_1", + `metrics.size` >= -10 and `metrics.size` < 10, "range_2", + `metrics.size` >= 10 and `metrics.size` < 100, "range_3", + `metrics.size` >= 100 and `metrics.size` < 1000, "range_4", + `metrics.size` >= 1000 and `metrics.size` < 2000, "range_5", + `metrics.size` >= 2000, "range_6") +| stats count() by range_bucket, span(`@timestamp`, 1h) as auto_span +| sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl new file mode 100644 index 00000000000..673a3892899 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_auto_date_histo_with_metrics.ppl @@ -0,0 +1,10 @@ +source = big5 +| eval range_bucket = case( + `metrics.size` < -10, "range_1", + `metrics.size` >= -10 and `metrics.size` < 10, "range_2", + `metrics.size` >= 10 and `metrics.size` < 100, "range_3", + `metrics.size` >= 100 and `metrics.size` < 1000, "range_4", + `metrics.size` >= 1000 and `metrics.size` < 2000, "range_5", + `metrics.size` >= 2000, "range_6") +| stats min(`metrics.tmin`) as tmin, avg(`metrics.size`) as tavg, max(`metrics.size`) as tmax by range_bucket, span(`@timestamp`, 1h) as auto_span +| sort + range_bucket, + auto_span \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl new file mode 100644 index 00000000000..e6390a38cbf --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_big_range_big_term_query.ppl @@ -0,0 +1,5 @@ +source = big5 +| where `process.name` = 'systemd' + and `metrics.size` >= 1 + and `metrics.size` <= 1000 +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl new file mode 100644 index 00000000000..f762da83896 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_big_term_query.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `metrics.size` >= 1 and `metrics.size` <= 42 +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl new file mode 100644 index 00000000000..9d0742e122f --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_field_conjunction_small_range_small_term_query.ppl @@ -0,0 +1,4 @@ +source = big5 +| where `aws.cloudwatch.log_stream` = 'indigodagger' + or (`metrics.size` >= 1 and `metrics.size` <= 30) +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl new file mode 100644 index 00000000000..4ea1dcfc518 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_field_disjunction_big_range_small_term_query.ppl @@ -0,0 +1,4 @@ +source = big5 +| where `aws.cloudwatch.log_stream` = 'indigodagger' + or (`metrics.size` >= 1 and `metrics.size` <= 1000) +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_numeric.ppl b/integ-test/src/test/resources/big5/queries/range_numeric.ppl new file mode 100644 index 00000000000..5b5b50b7c35 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_numeric.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `metrics.size` >= 1 and `metrics.size` <= 1000 +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl new file mode 100644 index 00000000000..a3325df54ed --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_with_asc_sort.ppl @@ -0,0 +1,5 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` <= '2023-01-13 00:00:00' +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl new file mode 100644 index 00000000000..ba3a042d511 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/range_with_desc_sort.ppl @@ -0,0 +1,5 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` <= '2023-01-13 00:00:00' +| sort - `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/scroll.ppl b/integ-test/src/test/resources/big5/queries/scroll.ppl new file mode 100644 index 00000000000..6b63c414ac0 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/scroll.ppl @@ -0,0 +1,2 @@ +source = big5 +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl new file mode 100644 index 00000000000..c0ca2fcd093 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl new file mode 100644 index 00000000000..c0ca2fcd093 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_keyword_no_can_match_shortcut.ppl @@ -0,0 +1,3 @@ +source = big5 match(`process.name`, 'kernel') +| sort + `@timestamp` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl new file mode 100644 index 00000000000..eb96d2b4bab --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort + `metrics.size` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl new file mode 100644 index 00000000000..604a3c28e5c --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_asc_with_match.ppl @@ -0,0 +1,3 @@ +source = big5 match(`log.file.path`, '/var/log/messages/solarshark') +| sort + `metrics.size` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl new file mode 100644 index 00000000000..f4a4165fbfc --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc.ppl @@ -0,0 +1,3 @@ +source = big5 +| sort - `metrics.size` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl new file mode 100644 index 00000000000..607d6689230 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/sort_numeric_desc_with_match.ppl @@ -0,0 +1,3 @@ +source = big5 match(`log.file.path`, '/var/log/messages/solarshark') +| sort - `metrics.size` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/term.ppl b/integ-test/src/test/resources/big5/queries/term.ppl new file mode 100644 index 00000000000..2cbfae69eba --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/term.ppl @@ -0,0 +1,3 @@ +source = big5 +| where `log.file.path` = '/var/log/messages/birdknight' +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl new file mode 100644 index 00000000000..b33048f82a1 --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/terms_significant_1.ppl @@ -0,0 +1,5 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `aws.cloudwatch.log_stream` +| head 10 \ No newline at end of file diff --git a/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl new file mode 100644 index 00000000000..994914e3bbe --- /dev/null +++ b/integ-test/src/test/resources/big5/queries/terms_significant_2.ppl @@ -0,0 +1,5 @@ +source = big5 +| where `@timestamp` >= '2023-01-01 00:00:00' + and `@timestamp` < '2023-01-03 00:00:00' +| stats count() by `process.name` +| head 10 \ No newline at end of file