diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 1ab4fe76b77..8f09508da35 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,2 +1,2 @@ # This should match the owning team set up in https://github.com/orgs/opensearch-project/teams -* @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @mengweieric @vamsimanohar @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger @qianheng-aws @yuancu +* @ps48 @kavithacm @derek-ho @joshuali925 @dai-chen @YANG-DB @mengweieric @vamsimanohar @swiddis @penghuo @seankao-az @MaxKsyunz @Yury-Fridlyand @anirudha @forestmvey @acarbonetto @GumpacG @ykmr1224 @LantaoJin @noCharger @qianheng-aws @yuancu @RyanL1997 diff --git a/.github/workflows/integ-tests-with-security.yml b/.github/workflows/integ-tests-with-security.yml index f1d67664718..8c2ee35408a 100644 --- a/.github/workflows/integ-tests-with-security.yml +++ b/.github/workflows/integ-tests-with-security.yml @@ -9,6 +9,7 @@ on: paths: - 'integ-test/**' - '.github/workflows/integ-tests-with-security.yml' + merge_group: jobs: Get-CI-Image-Tag: diff --git a/.github/workflows/sql-test-and-build-workflow.yml b/.github/workflows/sql-test-and-build-workflow.yml index ca7c4ace56b..ba1ff29b3d7 100644 --- a/.github/workflows/sql-test-and-build-workflow.yml +++ b/.github/workflows/sql-test-and-build-workflow.yml @@ -16,6 +16,7 @@ on: - '**/*.jar' - '**/*.pom' - '.github/workflows/sql-test-and-build-workflow.yml' + merge_group: jobs: Get-CI-Image-Tag: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000000..efa9c779423 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,13 @@ +repos: + - repo: local + hooks: + - id: spotless-format + name: Spotless Format + entry: bash -c './gradlew spotlessApply && git add -u' + language: system + pass_filenames: false + - id: spotless-check + name: Spotless Post-format Check + entry: bash -c './gradlew spotlessCheck' + language: system + pass_filenames: false diff --git a/DEVELOPER_GUIDE.rst b/DEVELOPER_GUIDE.rst index 85282d67d19..7482d0675d8 100644 --- a/DEVELOPER_GUIDE.rst +++ b/DEVELOPER_GUIDE.rst @@ -173,6 +173,7 @@ Here are other files and sub-folders that you are likely to touch: - ``build.gradle``: Gradle build script. - ``docs``: documentation for developers and reference manual for users. - ``doc-test``: code that run .rst docs in ``docs`` folder by Python doctest library. +- ``language-grammar``: centralized package for ANTLR grammar files. See `Language Grammar Package`_ for details. Note that other related project code has already merged into this single repository together: @@ -441,3 +442,29 @@ with an appropriate label `backport ` is merged to main wi PR. For example, if a PR on main needs to be backported to `1.x` branch, add a label `backport 1.x` to the PR and make sure the backport workflow runs on the PR along with other checks. Once this PR is merged to main, the workflow will create a backport PR to the `1.x` branch. + +Language Grammar Package +======================== + +The ``language-grammar`` package serves as a centralized repository for all ANTLR grammar files used throughout the OpenSearch SQL project. This package contains the definitive versions of grammar files for: + +- SQL parsing (``OpenSearchSQLParser.g4``, ``OpenSearchSQLLexer.g4``) +- PPL parsing (``OpenSearchPPLParser.g4``, ``OpenSearchPPLLexer.g4``) +- Legacy SQL parsing (``OpenSearchLegacySqlParser.g4``, ``OpenSearchLegacySqlLexer.g4``) +- Spark SQL extensions (``SparkSqlBase.g4``, ``FlintSparkSqlExtensions.g4``, ``SqlBaseParser.g4``, ``SqlBaseLexer.g4``) + +Purpose +------- + +The language-grammar package enables sharing of grammar files between the main SQL repository and the Spark repository, ensuring consistency and reducing duplication. Once updated, the package automatically triggers CI to upload the new version to Maven Central for consumption by other projects. + +Updating Grammar Files +---------------------- + +When grammar files are modified in their respective modules (``sql/``, ``ppl/``, ``legacy/``, ``async-query-core/``), they must be manually copied to the ``language-grammar/src/main/antlr4/`` directory. + +**Workflow:** + +1. Modify grammar files in their source locations (e.g., ``sql/src/main/antlr/``) +2. Copy updated files to ``language-grammar/src/main/antlr4/`` +3. Commit changes to trigger automatic Maven publication via CI diff --git a/MAINTAINERS.md b/MAINTAINERS.md index c9386055306..d4acf427f3d 100644 --- a/MAINTAINERS.md +++ b/MAINTAINERS.md @@ -5,7 +5,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje ## Current Maintainers | Maintainer | GitHub ID | Affiliation | -|-------------------|-----------------------------------------------------| ----------- | +|-------------------|-----------------------------------------------------|-------------| | Eric Wei | [mengweieric](https://github.com/mengweieric) | Amazon | | Joshua Li | [joshuali925](https://github.com/joshuali925) | Amazon | | Shenoy Pratik | [ps48](https://github.com/ps48) | Amazon | @@ -23,6 +23,7 @@ This document contains a list of maintainers in this repo. See [opensearch-proje | Louis Chu | [noCharger](https://github.com/noCharger) | Amazon | | Heng Qian | [qianheng-aws](https://github.com/qianheng-aws) | Amazon | | Yuanchun Shen | [yuancu](https://github.com/yuancu) | Amazon | +| Ryan Liang | [RyanL1997](https://github.com/RyanL1997) | Amazon | | Max Ksyunz | [MaxKsyunz](https://github.com/MaxKsyunz) | Improving | | Yury Fridlyand | [Yury-Fridlyand](https://github.com/Yury-Fridlyand) | Improving | | Andrew Carbonetto | [acarbonetto](https://github.com/acarbonetto) | Improving | diff --git a/async-query-core/build.gradle b/async-query-core/build.gradle index dd88ea1fd8c..ece332519b7 100644 --- a/async-query-core/build.gradle +++ b/async-query-core/build.gradle @@ -8,7 +8,7 @@ plugins { id "io.freefair.lombok" id 'jacoco' id 'antlr' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' id 'com.gradleup.shadow' } diff --git a/build.gradle b/build.gradle index f2c6357dd18..87580cdb83a 100644 --- a/build.gradle +++ b/build.gradle @@ -81,7 +81,7 @@ plugins { id 'java-library' id "io.freefair.lombok" version "8.14" id 'jacoco' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' version '7.2.1' } // import versions defined in https://github.com/opensearch-project/OpenSearch/blob/main/buildSrc/src/main/java/org/opensearch/gradle/OpenSearchJavaPlugin.java#L94 @@ -105,10 +105,10 @@ spotless { exclude '**/build/**', '**/build-*/**', '**/gen/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/common/build.gradle b/common/build.gradle index 171e9accb38..ac67a003fab 100644 --- a/common/build.gradle +++ b/common/build.gradle @@ -25,7 +25,7 @@ plugins { id 'java-library' id "io.freefair.lombok" - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } repositories { @@ -73,11 +73,10 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// Needs https://github.com/opensearch-project/sql/issues/1893 to be addressed first -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/common/src/main/java/org/opensearch/sql/common/antlr/Parser.java b/common/src/main/java/org/opensearch/sql/common/antlr/Parser.java index 7962f53ef69..02faf4f3d5b 100644 --- a/common/src/main/java/org/opensearch/sql/common/antlr/Parser.java +++ b/common/src/main/java/org/opensearch/sql/common/antlr/Parser.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.common.antlr; import org.antlr.v4.runtime.tree.ParseTree; diff --git a/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java b/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java index 1ab1bb8976d..c1f86d55c14 100644 --- a/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptor.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.common.interceptors; diff --git a/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java b/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java index 0ade25520f5..2638cf31993 100644 --- a/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptor.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.common.interceptors; diff --git a/common/src/main/java/org/opensearch/sql/common/interceptors/URIValidatorInterceptor.java b/common/src/main/java/org/opensearch/sql/common/interceptors/URIValidatorInterceptor.java index 68e7339beb4..1a9c486c068 100644 --- a/common/src/main/java/org/opensearch/sql/common/interceptors/URIValidatorInterceptor.java +++ b/common/src/main/java/org/opensearch/sql/common/interceptors/URIValidatorInterceptor.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.common.interceptors; diff --git a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java index 069a9d18ee2..f2002e8f283 100644 --- a/common/src/main/java/org/opensearch/sql/common/setting/Settings.java +++ b/common/src/main/java/org/opensearch/sql/common/setting/Settings.java @@ -30,6 +30,7 @@ public enum Key { PATTERN_MAX_SAMPLE_COUNT("plugins.ppl.pattern.max.sample.count"), PATTERN_BUFFER_LIMIT("plugins.ppl.pattern.buffer.limit"), PPL_REX_MAX_MATCH_LIMIT("plugins.ppl.rex.max_match.limit"), + PPL_VALUES_MAX_LIMIT("plugins.ppl.values.max.limit"), PPL_SYNTAX_LEGACY_PREFERRED("plugins.ppl.syntax.legacy.preferred"), /** Enable Calcite as execution engine */ diff --git a/common/src/main/java/org/opensearch/sql/common/utils/URIValidationUtils.java b/common/src/main/java/org/opensearch/sql/common/utils/URIValidationUtils.java index c7893fc0535..19b8db61e7f 100644 --- a/common/src/main/java/org/opensearch/sql/common/utils/URIValidationUtils.java +++ b/common/src/main/java/org/opensearch/sql/common/utils/URIValidationUtils.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.common.utils; import inet.ipaddr.IPAddressString; diff --git a/common/src/test/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptorTest.java index 6c5d1bac89d..823eb6c8071 100644 --- a/common/src/test/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/interceptors/AwsSigningInterceptorTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.common.interceptors; diff --git a/common/src/test/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptorTest.java b/common/src/test/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptorTest.java index af93060fab3..993f0bf4e16 100644 --- a/common/src/test/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptorTest.java +++ b/common/src/test/java/org/opensearch/sql/common/interceptors/BasicAuthenticationInterceptorTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.common.interceptors; diff --git a/core/build.gradle b/core/build.gradle index b8630a2ef90..f7de422bae8 100644 --- a/core/build.gradle +++ b/core/build.gradle @@ -28,7 +28,7 @@ plugins { id 'jacoco' id 'info.solidsoft.pitest' version '1.9.0' id 'java-test-fixtures' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } @@ -83,10 +83,11 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") + .onlyIfContentMatches("^((?!\\(original license below\\))[\\s\\S])*\$") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java b/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java index 9c9dfa07729..e31c0b00c9c 100644 --- a/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java +++ b/core/src/main/java/org/opensearch/sql/DataSourceSchemaName.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql; diff --git a/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java index 31719d2fe37..91005c9ec37 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java +++ b/core/src/main/java/org/opensearch/sql/analysis/DataSourceSchemaIdentifierNameResolver.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.analysis; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/PatternMode.java b/core/src/main/java/org/opensearch/sql/ast/expression/PatternMode.java index 9b6cf96f641..71e1c581c7c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/PatternMode.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/PatternMode.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ast.expression; diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java index b92f30900a1..438c46e7e53 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/SpanUnit.java @@ -42,6 +42,11 @@ public enum SpanUnit { SPAN_UNITS = builder.add(SpanUnit.values()).build(); } + /** Util method to check if the unit is time unit. */ + public static boolean isTimeUnit(SpanUnit unit) { + return unit != UNKNOWN && unit != NONE; + } + /** Util method to get span unit given the unit name. */ public static SpanUnit of(String unit) { switch (unit) { diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java index c223de5dc83..d592d7691cf 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Explain.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ast.statement; diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Query.java b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java index b068fbe5268..6681d6d1d2c 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Query.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Query.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ast.statement; diff --git a/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java b/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java index d90071a0ca6..2cdf81d6dc4 100644 --- a/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java +++ b/core/src/main/java/org/opensearch/sql/ast/statement/Statement.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ast.statement; diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index faa8a8fa6f9..cf2fe0c9b2b 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -18,6 +18,7 @@ import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_MAIN; import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_NAME_SUBSEARCH; import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation; +import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall; import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild; import com.google.common.base.Strings; @@ -53,6 +54,7 @@ import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexVisitorImpl; import org.apache.calcite.rex.RexWindowBounds; +import org.apache.calcite.sql.SqlKind; import org.apache.calcite.sql.fun.SqlStdOperatorTable; import org.apache.calcite.sql.type.SqlTypeFamily; import org.apache.calcite.sql.type.SqlTypeName; @@ -80,6 +82,8 @@ import org.opensearch.sql.ast.expression.ParseMethod; import org.opensearch.sql.ast.expression.PatternMethod; import org.opensearch.sql.ast.expression.PatternMode; +import org.opensearch.sql.ast.expression.Span; +import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; import org.opensearch.sql.ast.expression.WindowFrame; import org.opensearch.sql.ast.expression.WindowFrame.FrameType; @@ -689,7 +693,19 @@ public RelNode visitPatterns(Patterns node, CalcitePlanContext context) { context.relBuilder.field(node.getAlias()), context.relBuilder.field(PatternUtils.SAMPLE_LOGS)); flattenParsedPattern(node.getAlias(), parsedNode, context, false); - context.relBuilder.projectExcept(context.relBuilder.field(PatternUtils.SAMPLE_LOGS)); + // Reorder fields for consistency with Brain's output + projectPlusOverriding( + List.of( + context.relBuilder.field(node.getAlias()), + context.relBuilder.field(PatternUtils.PATTERN_COUNT), + context.relBuilder.field(PatternUtils.TOKENS), + context.relBuilder.field(PatternUtils.SAMPLE_LOGS)), + List.of( + node.getAlias(), + PatternUtils.PATTERN_COUNT, + PatternUtils.TOKENS, + PatternUtils.SAMPLE_LOGS), + context); } else { RexNode parsedNode = PPLFuncImpTable.INSTANCE.resolve( @@ -811,6 +827,23 @@ private void projectPlusOverriding( context.relBuilder.rename(expectedRenameFields); } + private List> extractInputRefList(List aggCalls) { + return aggCalls.stream() + .map(RelBuilder.AggCall::over) + .map(RelBuilder.OverCall::toRex) + .map(node -> getRexCall(node, this::isCountField)) + .map(list -> list.isEmpty() ? null : list.getFirst()) + .map(PlanUtils::getInputRefs) + .toList(); + } + + /** Is count(FIELD) */ + private boolean isCountField(RexCall call) { + return call.isA(SqlKind.COUNT) + && call.getOperands().size() == 1 // count(FIELD) + && call.getOperands().get(0) instanceof RexInputRef; + } + /** * Resolve the aggregation with trimming unused fields to avoid bugs in {@link * org.apache.calcite.sql2rel.RelDecorrelator#decorrelateRel(Aggregate, boolean)} @@ -824,6 +857,72 @@ private Pair, List> aggregateWithTrimming( List groupExprList, List aggExprList, CalcitePlanContext context) { + Pair, List> resolved = + resolveAttributesForAggregation(groupExprList, aggExprList, context); + List resolvedGroupByList = resolved.getLeft(); + List resolvedAggCallList = resolved.getRight(); + + // `doc_count` optimization required a filter `isNotNull(RexInputRef)` for the + // `count(FIELD)` aggregation which only can be applied to single FIELD without grouping: + // + // Example 1: source=t | stats count(a) + // Before: Aggregate(count(a)) + // \- Scan t + // After: Aggregate(count(a)) + // \- Filter(isNotNull(a)) + // \- Scan t + // + // Example 2: source=t | stats count(a), count(a) + // Before: Aggregate(count(a), count(a)) + // \- Scan t + // After: Aggregate(count(a), count(a)) + // \- Filter(isNotNull(a)) + // \- Scan t + // + // Example 3: source=t | stats count(a) by b + // Before & After: Aggregate(count(a) by b) + // \- Scan t + // + // Example 4: source=t | stats count() + // Before & After: Aggregate(count()) + // \- Scan t + // + // Example 5: source=t | stats count(), count(a) + // Before & After: Aggregate(count(), count(a)) + // \- Scan t + // + // Example 6: source=t | stats count(a), count(b) + // Before & After: Aggregate(count(a), count(b)) + // \- Scan t + // + // Example 7: source=t | stats count(a+1) + // Before & After: Aggregate(count(a+1)) + // \- Scan t + if (resolvedGroupByList.isEmpty()) { + List> refsPerCount = extractInputRefList(resolvedAggCallList); + List distinctRefsOfCounts; + if (context.relBuilder.peek() instanceof org.apache.calcite.rel.core.Project project) { + List mappedInProject = + refsPerCount.stream() + .flatMap(List::stream) + .map(ref -> project.getProjects().get(ref.getIndex())) + .toList(); + if (mappedInProject.stream().allMatch(RexInputRef.class::isInstance)) { + distinctRefsOfCounts = + mappedInProject.stream().map(RexInputRef.class::cast).distinct().toList(); + } else { + distinctRefsOfCounts = List.of(); + } + } else { + distinctRefsOfCounts = refsPerCount.stream().flatMap(List::stream).distinct().toList(); + } + if (distinctRefsOfCounts.size() == 1 && refsPerCount.stream().noneMatch(List::isEmpty)) { + context.relBuilder.filter(context.relBuilder.isNotNull(distinctRefsOfCounts.getFirst())); + } + } + + // Add project before aggregate: + // // Example 1: source=t | where a > 1 | stats avg(b + 1) by c // Before: Aggregate(avg(b + 1)) // \- Filter(a > 1) @@ -834,23 +933,22 @@ private Pair, List> aggregateWithTrimming( // \- Scan t // // Example 2: source=t | where a > 1 | top b by c - // Before: Aggregate(count) - // \-Filter(a > 1) + // Before: Aggregate(count(b) by c) + // \-Filter(a > 1 && isNotNull(b)) // \- Scan t - // After: Aggregate(count) + // After: Aggregate(count(b) by c) // \- Project([c, b]) - // \- Filter(a > 1) + // \- Filter(a > 1 && isNotNull(b)) // \- Scan t - // Example 3: source=t | stats count(): no project added for count() - // Before: Aggregate(count) + // + // Example 3: source=t | stats count(): no change for count() + // Before: Aggregate(count()) // \- Scan t - // After: Aggregate(count) + // After: Aggregate(count()) // \- Scan t - Pair, List> resolved = - resolveAttributesForAggregation(groupExprList, aggExprList, context); List trimmedRefs = new ArrayList<>(); - trimmedRefs.addAll(PlanUtils.getInputRefs(resolved.getLeft())); // group-by keys first - trimmedRefs.addAll(PlanUtils.getInputRefsFromAggCall(resolved.getRight())); + trimmedRefs.addAll(PlanUtils.getInputRefs(resolvedGroupByList)); // group-by keys first + trimmedRefs.addAll(PlanUtils.getInputRefsFromAggCall(resolvedAggCallList)); context.relBuilder.project(trimmedRefs); // Re-resolve all attributes based on adding trimmed Project. @@ -893,27 +991,42 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { // The span column is always the first column in result whatever // the order of span in query is first or last one UnresolvedExpression span = node.getSpan(); - if (!Objects.isNull(span)) { + if (Objects.nonNull(span)) { groupExprList.add(span); + List timeSpanFilters = + getTimeSpanField(span).stream() + .map(f -> rexVisitor.analyze(f, context)) + .map(context.relBuilder::isNotNull) + .toList(); + if (!timeSpanFilters.isEmpty()) { + // add isNotNull filter before aggregation for time span + context.relBuilder.filter(timeSpanFilters); + } } groupExprList.addAll(node.getGroupExprList()); - Pair, List> aggregationAttributes = - aggregateWithTrimming(groupExprList, aggExprList, context); - // Add group by columns - List aliasedGroupByList = - aggregationAttributes.getLeft().stream() - .map(this::extractAliasLiteral) - .flatMap(Optional::stream) - .map(ref -> ((RexLiteral) ref).getValueAs(String.class)) - .map(context.relBuilder::field) - .map(f -> (RexNode) f) - .toList(); // add stats hint to LogicalAggregation Argument.ArgumentMap statsArgs = Argument.ArgumentMap.of(node.getArgExprList()); Boolean bucketNullable = (Boolean) statsArgs.getOrDefault(Argument.BUCKET_NULLABLE, Literal.TRUE).getValue(); - if (!bucketNullable && !aliasedGroupByList.isEmpty()) { + boolean toAddHintsOnAggregate = false; + if (!bucketNullable + && !groupExprList.isEmpty() + && !(groupExprList.size() == 1 && getTimeSpanField(span).isPresent())) { + toAddHintsOnAggregate = true; + // add isNotNull filter before aggregation for non-nullable buckets + List groupByList = + groupExprList.stream().map(expr -> rexVisitor.analyze(expr, context)).toList(); + context.relBuilder.filter( + PlanUtils.getSelectColumns(groupByList).stream() + .map(context.relBuilder::field) + .map(context.relBuilder::isNotNull) + .toList()); + } + + Pair, List> aggregationAttributes = + aggregateWithTrimming(groupExprList, aggExprList, context); + if (toAddHintsOnAggregate) { final RelHint statHits = RelHint.builder("stats_args").hintOption(Argument.BUCKET_NULLABLE, "false").build(); assert context.relBuilder.peek() instanceof LogicalAggregate @@ -930,8 +1043,6 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { return rel instanceof LogicalAggregate; }) .build()); - context.relBuilder.filter( - aliasedGroupByList.stream().map(context.relBuilder::isNotNull).toList()); } // schema reordering @@ -945,12 +1056,32 @@ public RelNode visitAggregation(Aggregation node, CalcitePlanContext context) { List aggRexList = outputFields.subList(numOfOutputFields - numOfAggList, numOfOutputFields); reordered.addAll(aggRexList); + // Add group by columns + List aliasedGroupByList = + aggregationAttributes.getLeft().stream() + .map(this::extractAliasLiteral) + .flatMap(Optional::stream) + .map(ref -> ((RexLiteral) ref).getValueAs(String.class)) + .map(context.relBuilder::field) + .map(f -> (RexNode) f) + .toList(); reordered.addAll(aliasedGroupByList); context.relBuilder.project(reordered); return context.relBuilder.peek(); } + private Optional getTimeSpanField(UnresolvedExpression expr) { + if (Objects.isNull(expr)) return Optional.empty(); + if (expr instanceof Span span && SpanUnit.isTimeUnit(span.getUnit())) { + return Optional.of(span.getField()); + } + if (expr instanceof Alias alias) { + return getTimeSpanField(alias.getDelegated()); + } + return Optional.empty(); + } + /** extract the RexLiteral of Alias from a node */ private Optional extractAliasLiteral(RexNode node) { if (node == null) { @@ -2223,7 +2354,7 @@ private void flattenParsedPattern( String originalPatternResultAlias, RexNode parsedNode, CalcitePlanContext context, - boolean flattenPatternCount) { + boolean flattenPatternAggResult) { List fattenedNodes = new ArrayList<>(); List projectNames = new ArrayList<>(); // Flatten map struct fields @@ -2239,7 +2370,7 @@ private void flattenParsedPattern( true); fattenedNodes.add(context.relBuilder.alias(patternExpr, originalPatternResultAlias)); projectNames.add(originalPatternResultAlias); - if (flattenPatternCount) { + if (flattenPatternAggResult) { RexNode patternCountExpr = context.rexBuilder.makeCast( context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.BIGINT), @@ -2265,6 +2396,24 @@ private void flattenParsedPattern( true); fattenedNodes.add(context.relBuilder.alias(tokensExpr, PatternUtils.TOKENS)); projectNames.add(PatternUtils.TOKENS); + if (flattenPatternAggResult) { + RexNode sampleLogsExpr = + context.rexBuilder.makeCast( + context + .rexBuilder + .getTypeFactory() + .createArrayType( + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), -1), + PPLFuncImpTable.INSTANCE.resolve( + context.rexBuilder, + BuiltinFunctionName.INTERNAL_ITEM, + parsedNode, + context.rexBuilder.makeLiteral(PatternUtils.SAMPLE_LOGS)), + true, + true); + fattenedNodes.add(context.relBuilder.alias(sampleLogsExpr, PatternUtils.SAMPLE_LOGS)); + projectNames.add(PatternUtils.SAMPLE_LOGS); + } projectPlusOverriding(fattenedNodes, projectNames, context); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java index 726244663fd..f15b040f98c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java +++ b/core/src/main/java/org/opensearch/sql/calcite/plan/PPLAggregateConvertRule.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.calcite.plan; import com.google.common.collect.ImmutableList; diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ListAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ListAggFunction.java index 709df157e2d..30f968fead5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ListAggFunction.java +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ListAggFunction.java @@ -21,9 +21,8 @@ *
  • Order of values in the result is non-deterministic * * - *

    Note: Similar to the TAKE function, LIST does not guarantee any specific order of values in - * the result array. The order may vary between executions and depends on the underlying query - * execution plan and optimizations. + *

    LIST does not guarantee any specific order of values in the result array. The order may vary + * between executions and depends on the underlying query execution plan and optimizations. */ public class ListAggFunction implements UserDefinedAggFunction { diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/LogPatternAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/LogPatternAggFunction.java index d4d6324cb1c..576b105bdc5 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/LogPatternAggFunction.java +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/LogPatternAggFunction.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.calcite.udf.udaf; @@ -187,7 +184,8 @@ public Object value(Object... argList) { PatternUtils.PATTERN, parseResult.toTokenOrderString(PatternUtils.WILDCARD_PREFIX), PatternUtils.PATTERN_COUNT, count, - PatternUtils.TOKENS, tokensMap); + PatternUtils.TOKENS, tokensMap, + PatternUtils.SAMPLE_LOGS, sampleLogs); }) .collect(Collectors.toList()); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ValuesAggFunction.java b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ValuesAggFunction.java new file mode 100644 index 00000000000..b5d68fe7ce3 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/calcite/udf/udaf/ValuesAggFunction.java @@ -0,0 +1,88 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.udf.udaf; + +import java.util.ArrayList; +import java.util.Set; +import java.util.TreeSet; +import org.opensearch.sql.calcite.udf.UserDefinedAggFunction; + +/** + * VALUES aggregate function implementation. Returns distinct values from a field in lexicographical + * order as a multivalue field. + * + *

    Behavior: + * + *

      + *
    • Returns unique values only (no duplicates) + *
    • Values are sorted in lexicographical order + *
    • Processes field values as strings (casts all inputs to strings) + *
    • Configurable limit via plugins.ppl.values.max.limit setting (0 = unlimited) + *
    • Supports only scalar data types (rejects STRUCT/ARRAY types) + *
    • Implementation uses TreeSet for automatic sorting and deduplication + *
    + */ +public class ValuesAggFunction + implements UserDefinedAggFunction { + + @Override + public ValuesAccumulator init() { + return new ValuesAccumulator(); + } + + @Override + public Object result(ValuesAccumulator accumulator) { + return accumulator.value(); + } + + @Override + public ValuesAccumulator add(ValuesAccumulator acc, Object... values) { + // Handle case where no values are passed + if (values == null || values.length == 0) { + return acc; + } + + Object value = values[0]; + + // Get limit from second argument (passed from AST) + int limit = 0; // Default to unlimited + if (values.length > 1 && values[1] != null) { + limit = (Integer) values[1]; + } + + // Filter out null values and check limit + if (value != null && (limit == 0 || acc.size() < limit)) { + // Convert value to string + String stringValue = String.valueOf(value); + acc.add(stringValue, limit); + } + + return acc; + } + + public static class ValuesAccumulator implements Accumulator { + private final Set values; + + public ValuesAccumulator() { + this.values = new TreeSet<>(); // TreeSet maintains sorted order and uniqueness + } + + @Override + public Object value(Object... argList) { + return new ArrayList<>(values); // Return List to match expected type + } + + public void add(String value, int limit) { + if (limit == 0 || values.size() < limit) { + values.add(value); + } + } + + public int size() { + return values.size(); + } + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java index 0d343711b3f..fd6b7c8857d 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PPLOperandTypes.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.calcite.utils; @@ -22,6 +20,45 @@ public class PPLOperandTypes { // This class is not meant to be instantiated. private PPLOperandTypes() {} + /** List of all scalar type signatures (single parameter each) */ + private static final java.util.List> + SCALAR_TYPES = + java.util.List.of( + // Numeric types + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), + // String type + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), + // Boolean type + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), + // Temporal types + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), + // Special scalar types + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), + java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY)); + + /** Helper method to create scalar types with optional integer parameter */ + private static java.util.List> + createScalarWithOptionalInteger() { + java.util.List> result = + new java.util.ArrayList<>(SCALAR_TYPES); + + // Add scalar + integer combinations + SCALAR_TYPES.forEach( + scalarType -> + result.add( + java.util.List.of( + scalarType.get(0), org.opensearch.sql.data.type.ExprCoreType.INTEGER))); + + return result; + } + public static final UDFOperandMetadata NONE = UDFOperandMetadata.wrap(OperandTypes.family()); public static final UDFOperandMetadata OPTIONAL_ANY = UDFOperandMetadata.wrap( @@ -200,25 +237,12 @@ private PPLOperandTypes() {} * booleans, datetime types, and special scalar types like IP and BINARY. Excludes complex types * like arrays, structs, and maps. */ - public static final UDFOperandMetadata ANY_SCALAR = - UDFOperandMetadata.wrapUDT( - java.util.List.of( - // Numeric types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BYTE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.SHORT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.INTEGER), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.LONG), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.FLOAT), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DOUBLE), - // String type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.STRING), - // Boolean type - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BOOLEAN), - // Temporal types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.DATE), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIME), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP), - // Special scalar types - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.IP), - java.util.List.of(org.opensearch.sql.data.type.ExprCoreType.BINARY))); + public static final UDFOperandMetadata ANY_SCALAR = UDFOperandMetadata.wrapUDT(SCALAR_TYPES); + + /** + * Operand type checker that accepts any scalar type with an optional integer argument. This is + * used for aggregation functions that take a field and an optional limit/size parameter. + */ + public static final UDFOperandMetadata ANY_SCALAR_OPTIONAL_INTEGER = + UDFOperandMetadata.wrapUDT(createScalarWithOptionalInteger()); } diff --git a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java index 80658557aa9..4d3bef062fa 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java +++ b/core/src/main/java/org/opensearch/sql/calcite/utils/PlanUtils.java @@ -15,6 +15,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.function.Predicate; import java.util.stream.Collectors; import javax.annotation.Nullable; import org.apache.calcite.plan.RelOptTable; @@ -255,6 +256,9 @@ static RelBuilder.AggCall makeAggCall( /** Get all uniq input references from a RexNode. */ static List getInputRefs(RexNode node) { + if (node == null) { + return List.of(); + } List inputRefs = new ArrayList<>(); node.accept( new RexVisitorImpl(true) { @@ -274,6 +278,26 @@ static List getInputRefs(List nodes) { return nodes.stream().flatMap(node -> getInputRefs(node).stream()).toList(); } + /** Get all uniq RexCall from RexNode with a predicate */ + static List getRexCall(RexNode node, Predicate predicate) { + List list = new ArrayList<>(); + node.accept( + new RexVisitorImpl(true) { + @Override + public Void visitCall(RexCall inputCall) { + if (predicate.test(inputCall)) { + if (!list.contains(inputCall)) { + list.add(inputCall); + } + } else { + inputCall.getOperands().forEach(call -> call.accept(this)); + } + return null; + } + }); + return list; + } + /** Get all uniq input references from a list of agg calls. */ static List getInputRefsFromAggCall(List aggCalls) { return aggCalls.stream() diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java index 5285a0c02a3..8b31a508e50 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprDateValue.java @@ -8,6 +8,7 @@ import com.google.common.base.Objects; import java.time.Instant; import java.time.LocalDate; +import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; @@ -32,7 +33,14 @@ public class ExprDateValue extends AbstractExprValue { */ public ExprDateValue(String date) { try { - this.date = LocalDate.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER); + LocalDateTime ldt; + try { + ldt = LocalDateTime.parse(date, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER); + } catch (DateTimeParseException ignored) { + ZonedDateTime zdt = ZonedDateTime.parse(date, DateTimeFormatter.ISO_DATE_TIME); + ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime(); + } + this.date = ldt.toLocalDate(); } catch (DateTimeParseException e) { throw new ExpressionEvaluationException( String.format("date:%s in unsupported format, please use 'yyyy-MM-dd'", date)); diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java index 6e441ba88d6..18cb18a7e5e 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimeValue.java @@ -10,8 +10,10 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; +import java.time.OffsetTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.util.Objects; import lombok.RequiredArgsConstructor; @@ -34,7 +36,23 @@ public class ExprTimeValue extends AbstractExprValue { */ public ExprTimeValue(String time) { try { - this.time = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER); + LocalTime lt; + try { + lt = LocalTime.parse(time, DateTimeFormatters.TIME_TIMESTAMP_FORMATTER); + } catch (DateTimeParseException ignore) { + try { + lt = + ZonedDateTime.parse(time, DateTimeFormatter.ISO_DATE_TIME) + .withZoneSameInstant(ZoneOffset.UTC) + .toLocalTime(); + } catch (DateTimeParseException ignore2) { + lt = + OffsetTime.parse(time, DateTimeFormatter.ISO_TIME) + .withOffsetSameInstant(ZoneOffset.UTC) + .toLocalTime(); + } + } + this.time = lt; } catch (DateTimeParseException e) { throw new ExpressionEvaluationException( String.format("time:%s in unsupported format, please use 'HH:mm:ss[.SSSSSSSSS]'", time)); diff --git a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java index 194a2c2420e..65d5ad0fc65 100644 --- a/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java +++ b/core/src/main/java/org/opensearch/sql/data/model/ExprTimestampValue.java @@ -13,6 +13,8 @@ import java.time.LocalDateTime; import java.time.LocalTime; import java.time.ZoneOffset; +import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import java.time.format.DateTimeParseException; import java.time.temporal.ChronoUnit; import java.util.Objects; @@ -31,17 +33,24 @@ public class ExprTimestampValue extends AbstractExprValue { /** * Constructor with timestamp string. * - * @param timestamp a date or timestamp string (does not accept time string) + * @param timestamp a date or timestamp string (does not accept time string). It accepts both ISO + * 8601 format and {@code yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]} format */ public ExprTimestampValue(String timestamp) { try { - this.timestamp = - LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER) - .toInstant(ZoneOffset.UTC); + LocalDateTime ldt; + try { + ldt = LocalDateTime.parse(timestamp, DateTimeFormatters.DATE_TIMESTAMP_FORMATTER); + } catch (DateTimeParseException ignored) { + ZonedDateTime zdt = ZonedDateTime.parse(timestamp, DateTimeFormatter.ISO_DATE_TIME); + ldt = zdt.withZoneSameInstant(ZoneOffset.UTC).toLocalDateTime(); + } + this.timestamp = ldt.toInstant(ZoneOffset.UTC); } catch (DateTimeParseException e) { throw new ExpressionEvaluationException( String.format( - "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", + "timestamp:%s in unsupported format, please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or" + + " ISO 8601 format", timestamp)); } } diff --git a/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java b/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java index 924647e3b08..1d4514331c7 100644 --- a/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java +++ b/core/src/main/java/org/opensearch/sql/datasource/model/DataSource.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.datasource.model; diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryId.java b/core/src/main/java/org/opensearch/sql/executor/QueryId.java index eea8166e2a5..3527c2472c9 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryId.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryId.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryManager.java b/core/src/main/java/org/opensearch/sql/executor/QueryManager.java index 44d6a1cd84a..95da3daad17 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryManager.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryManager.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 7afbfdf5ba2..142a2e9213b 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; @@ -265,7 +262,11 @@ public RelNode optimize(RelNode plan, CalcitePlanContext context) { private boolean isCalciteFallbackAllowed() { if (settings != null) { - return settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED); + Boolean fallback_allowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED); + if (fallback_allowed == null) { + return false; + } + return fallback_allowed; } else { return true; } diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryType.java b/core/src/main/java/org/opensearch/sql/executor/QueryType.java index 9c9ee53ea5f..5a96fbaf3e8 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryType.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryType.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java index caf23b51b14..0a81d71e5b0 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/AbstractPlan.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java index bd333806413..cdd5380df5b 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/CommandPlan.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java index 61d6f2f8e5c..67c2f65f5d1 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/ExplainPlan.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java index bc05fe50d4d..c37327f3df4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlan.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java index 46f8c77a767..5f10df92894 100644 --- a/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java +++ b/core/src/main/java/org/opensearch/sql/executor/execution/QueryPlanFactory.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java index 48975a5608d..b8d52b072a4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/DefaultMetadataLog.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.streaming; diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java index d6bb9bacd6f..e520a9c4374 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MetadataLog.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.streaming; diff --git a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java index 4e05484f15a..b6b57956aa4 100644 --- a/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java +++ b/core/src/main/java/org/opensearch/sql/executor/streaming/MicroBatchStreamingExecution.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.streaming; diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/StdDevAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/StdDevAggregator.java index d5422bc7887..87525009018 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/StdDevAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/StdDevAggregator.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.aggregation; diff --git a/core/src/main/java/org/opensearch/sql/expression/aggregation/VarianceAggregator.java b/core/src/main/java/org/opensearch/sql/expression/aggregation/VarianceAggregator.java index 920830d2667..842eb9b8b33 100644 --- a/core/src/main/java/org/opensearch/sql/expression/aggregation/VarianceAggregator.java +++ b/core/src/main/java/org/opensearch/sql/expression/aggregation/VarianceAggregator.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.aggregation; diff --git a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java index ed65a472ca3..7cfa5920603 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/BuiltinFunctionName.java @@ -207,6 +207,7 @@ public enum BuiltinFunctionName { // Multivalue aggregation function LIST(FunctionName.of("list")), + VALUES(FunctionName.of("values")), // Not always an aggregation query NESTED(FunctionName.of("nested")), // Document order aggregation functions @@ -364,6 +365,7 @@ public enum BuiltinFunctionName { .put("latest", BuiltinFunctionName.LATEST) .put("distinct_count_approx", BuiltinFunctionName.DISTINCT_COUNT_APPROX) .put("list", BuiltinFunctionName.LIST) + .put("values", BuiltinFunctionName.VALUES) .put("pattern", BuiltinFunctionName.INTERNAL_PATTERN) .put("first", BuiltinFunctionName.FIRST) .put("last", BuiltinFunctionName.LAST) diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java index 055fedaec9d..e84db28655e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLBuiltinOperators.java @@ -36,6 +36,7 @@ import org.opensearch.sql.calcite.udf.udaf.NullableSqlAvgAggFunction; import org.opensearch.sql.calcite.udf.udaf.PercentileApproxFunction; import org.opensearch.sql.calcite.udf.udaf.TakeAggFunction; +import org.opensearch.sql.calcite.udf.udaf.ValuesAggFunction; import org.opensearch.sql.calcite.utils.PPLOperandTypes; import org.opensearch.sql.calcite.utils.PPLReturnTypes; import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; @@ -450,6 +451,12 @@ public class PPLBuiltinOperators extends ReflectiveSqlOperatorTable { public static final SqlAggFunction LIST = createUserDefinedAggFunction( ListAggFunction.class, "LIST", PPLReturnTypes.STRING_ARRAY, PPLOperandTypes.ANY_SCALAR); + public static final SqlAggFunction VALUES = + createUserDefinedAggFunction( + ValuesAggFunction.class, + "VALUES", + PPLReturnTypes.STRING_ARRAY, + PPLOperandTypes.ANY_SCALAR_OPTIONAL_INTEGER); public static final SqlOperator ENHANCED_COALESCE = new EnhancedCoalesceFunction().toUDF("COALESCE"); diff --git a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java index 831ed346a5a..8752a0ae822 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/PPLFuncImpTable.java @@ -217,6 +217,7 @@ import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_DATE; import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIME; import static org.opensearch.sql.expression.function.BuiltinFunctionName.UTC_TIMESTAMP; +import static org.opensearch.sql.expression.function.BuiltinFunctionName.VALUES; import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARPOP; import static org.opensearch.sql.expression.function.BuiltinFunctionName.VARSAMP; import static org.opensearch.sql.expression.function.BuiltinFunctionName.WEEK; @@ -1120,6 +1121,7 @@ void populate() { registerOperator(TAKE, PPLBuiltinOperators.TAKE); registerOperator(INTERNAL_PATTERN, PPLBuiltinOperators.INTERNAL_PATTERN); registerOperator(LIST, PPLBuiltinOperators.LIST); + registerOperator(VALUES, PPLBuiltinOperators.VALUES); register( AVG, diff --git a/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java b/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java index b8b90bf5e3f..498901e1a8e 100644 --- a/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java +++ b/core/src/main/java/org/opensearch/sql/expression/function/TableFunctionImplementation.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.function; diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalAD.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalAD.java index 25dbd14f1af..7494e4ae20b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalAD.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalAD.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.planner.logical; import java.util.Collections; diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java index 780e0bba94a..cad550580da 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalML.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.planner.logical; import java.util.Collections; diff --git a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalMLCommons.java b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalMLCommons.java index cfc313a68d5..00a69d1d76b 100644 --- a/core/src/main/java/org/opensearch/sql/planner/logical/LogicalMLCommons.java +++ b/core/src/main/java/org/opensearch/sql/planner/logical/LogicalMLCommons.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.planner.logical; import java.util.Collections; diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java b/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java index ea117ee6dfa..8b3a4d7c963 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/SortHelper.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.planner.physical; import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java index 5542d0f0e4c..acc4719cab9 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTable.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.planner.physical.datasource; diff --git a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java index 89e21377dc3..6c931f6a335 100644 --- a/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java +++ b/core/src/main/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScan.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.planner.physical.datasource; diff --git a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java index 69d902c1d7d..5478c79c1fc 100644 --- a/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java +++ b/core/src/main/java/org/opensearch/sql/storage/DataSourceFactory.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.storage; diff --git a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java index 90bca8fe8a0..a1123d5b048 100644 --- a/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java +++ b/core/src/main/java/org/opensearch/sql/utils/MLCommonsConstants.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.utils; public class MLCommonsConstants { diff --git a/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java b/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java index 775984a528a..4021869deae 100644 --- a/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java +++ b/core/src/test/java/org/opensearch/sql/analysis/model/DataSourceSchemaIdentifierNameResolverTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.analysis.model; diff --git a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java index 4ddb147eaf8..2a2c9de63ab 100644 --- a/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java +++ b/core/src/test/java/org/opensearch/sql/common/utils/StringUtilsTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.common.utils; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java index 31a38983347..31420b382b0 100644 --- a/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java +++ b/core/src/test/java/org/opensearch/sql/data/model/DateTimeValueTest.java @@ -16,6 +16,7 @@ import java.time.LocalTime; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.time.format.DateTimeFormatter; import org.junit.jupiter.api.Test; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.expression.function.FunctionProperties; @@ -112,15 +113,21 @@ public void timeInUnsupportedFormat() { } @Test - public void timestampInUnsupportedFormat() { - Throwable exception = - assertThrows( - ExpressionEvaluationException.class, - () -> new ExprTimestampValue("2020-07-07T01:01:01Z")); + public void timestampInISO8601Format() { + ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01Z"); assertEquals( - "timestamp:2020-07-07T01:01:01Z in unsupported format, " - + "please use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", - exception.getMessage()); + LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME) + .toInstant(ZoneOffset.UTC), + timestampValue.timestampValue()); + } + + @Test + public void timestampInISO8601FormatWithTimeZone() { + ExprTimestampValue timestampValue = new ExprTimestampValue("2020-07-07T01:01:01-01:00"); + assertEquals( + LocalDateTime.parse("2020-07-07T02:01:01Z", DateTimeFormatter.ISO_DATE_TIME) + .toInstant(ZoneOffset.UTC), + timestampValue.timestampValue()); } @Test @@ -134,13 +141,11 @@ public void stringTimestampValue() { assertEquals(LocalTime.parse("19:44:00"), stringValue.timeValue()); assertEquals("\"2020-08-17 19:44:00\"", stringValue.toString()); - Throwable exception = - assertThrows( - ExpressionEvaluationException.class, - () -> new ExprStringValue("2020-07-07T01:01:01Z").timestampValue()); + ExprValue stringValueWithIsoTimestamp = new ExprStringValue("2020-07-07T01:01:01Z"); assertEquals( - "date:2020-07-07T01:01:01Z in unsupported format, " + "please use 'yyyy-MM-dd'", - exception.getMessage()); + LocalDateTime.parse("2020-07-07T01:01:01Z", DateTimeFormatter.ISO_DATE_TIME) + .toInstant(ZoneOffset.UTC), + stringValueWithIsoTimestamp.timestampValue()); } @Test @@ -221,7 +226,7 @@ public void timestampOverMaxNanoPrecision() { () -> new ExprTimestampValue("2020-07-07 01:01:01.1234567890")); assertEquals( "timestamp:2020-07-07 01:01:01.1234567890 in unsupported format, please use " - + "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", + + "'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format", exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java index 7d837c3e249..2b94e239b26 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryIdTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; diff --git a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java index c8454f90b74..727fefcff7c 100644 --- a/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/QueryServiceTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java index d9149c8da27..e074c1ebdaf 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/CommandPlanTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java index 7d025f9ad9d..2c24dbf820e 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/ExplainPlanTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java index ed0ffc305bb..dd73b26a8c3 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanFactoryTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java index 437448f2853..809cbee9483 100644 --- a/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/execution/QueryPlanTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.execution; diff --git a/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java index 17ea253e2aa..aedec9c511e 100644 --- a/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java +++ b/core/src/test/java/org/opensearch/sql/executor/streaming/DefaultMetadataLogTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor.streaming; diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java index 5d1d86189bf..2e204b9420d 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/PercentileApproxAggregatorTest.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.aggregation; diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/StdDevAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/StdDevAggregatorTest.java index ceb76815dc0..759e9591bc6 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/StdDevAggregatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/StdDevAggregatorTest.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.aggregation; diff --git a/core/src/test/java/org/opensearch/sql/expression/aggregation/VarianceAggregatorTest.java b/core/src/test/java/org/opensearch/sql/expression/aggregation/VarianceAggregatorTest.java index 64ab6474ed8..057bbe3e489 100644 --- a/core/src/test/java/org/opensearch/sql/expression/aggregation/VarianceAggregatorTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/aggregation/VarianceAggregatorTest.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.expression.aggregation; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java index 0daeb31bef7..102e3d8784c 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/StrftimeFunctionTest.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.expression.datetime; import static org.junit.jupiter.api.Assertions.assertEquals; diff --git a/core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java b/core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java index 92058f9bae1..9d27b7ee5d1 100644 --- a/core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java +++ b/core/src/test/java/org/opensearch/sql/expression/datetime/TimestampTest.java @@ -64,7 +64,8 @@ public void timestamp_one_arg_string_invalid_format(String value, String testNam () -> DSL.timestamp(functionProperties, DSL.literal(value)).valueOf()); assertEquals( String.format( - "timestamp:%s in unsupported format, please " + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]'", + "timestamp:%s in unsupported format, please " + + "use 'yyyy-MM-dd HH:mm:ss[.SSSSSSSSS]' or ISO 8601 format", value), exception.getMessage()); } diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java index 53cbd15b8ee..552f6f233eb 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableScanTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.planner.physical.datasource; diff --git a/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java index 75f21c3e52a..3f9aa2d3d7f 100644 --- a/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java +++ b/core/src/test/java/org/opensearch/sql/planner/physical/datasource/DataSourceTableTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.planner.physical.datasource; diff --git a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java index 67014b76bdc..38bdd42a9b6 100644 --- a/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java +++ b/core/src/test/java/org/opensearch/sql/storage/StorageEngineTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.storage; diff --git a/docs/category.json b/docs/category.json index 4eb954062a8..7b22ab31fc7 100644 --- a/docs/category.json +++ b/docs/category.json @@ -31,7 +31,6 @@ "user/ppl/cmd/subquery.rst", "user/ppl/general/identifiers.rst", "user/ppl/general/datatypes.rst", - "user/ppl/functions/condition.rst", "user/ppl/functions/datetime.rst", "user/ppl/functions/expressions.rst", "user/ppl/functions/ip.rst", @@ -56,6 +55,7 @@ ], "ppl_cli_calcite": [ "user/ppl/cmd/append.rst", + "user/ppl/functions/condition.rst", "user/ppl/cmd/eventstats.rst", "user/ppl/cmd/fields.rst", "user/ppl/cmd/regex.rst", diff --git a/docs/user/ppl/admin/settings.rst b/docs/user/ppl/admin/settings.rst index 61345f0fe45..389a5c24be8 100644 --- a/docs/user/ppl/admin/settings.rst +++ b/docs/user/ppl/admin/settings.rst @@ -226,3 +226,85 @@ PPL query:: } } } + +plugins.ppl.values.max.limit +============================ + +Description +----------- + +This setting controls the maximum number of unique values that the ``VALUES`` aggregation function can return. When set to 0 (the default), there is no limit on the number of unique values returned. When set to a positive integer, the function will return at most that many unique values. + +1. The default value is 0 (unlimited). +2. This setting is node scope. +3. This setting can be updated dynamically. + +The ``VALUES`` function collects all unique values from a field and returns them in lexicographical order. This setting helps manage memory usage by limiting the number of values collected. + +Example 1 +--------- + +Set the limit to 1000 unique values: + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"transient" : {"plugins.ppl.values.max.limit" : "1000"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "plugins": { + "ppl": { + "values": { + "max": { + "limit": "1000" + } + } + } + } + } + } + +Example 2 +--------- + +Reset to default (unlimited) by setting to null: + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"transient" : {"plugins.ppl.values.max.limit" : null}}' + { + "acknowledged": true, + "persistent": {}, + "transient": {} + } + +Example 3 +--------- + +Set to 0 explicitly for unlimited values: + +PPL query:: + + sh$ curl -sS -H 'Content-Type: application/json' \ + ... -X PUT localhost:9200/_plugins/_query/settings \ + ... -d '{"transient" : {"plugins.ppl.values.max.limit" : "0"}}' + { + "acknowledged": true, + "persistent": {}, + "transient": { + "plugins": { + "ppl": { + "values": { + "max": { + "limit": "0" + } + } + } + } + } + } diff --git a/docs/user/ppl/cmd/rex.rst b/docs/user/ppl/cmd/rex.rst index 05d0e3757f8..8841f0cb1d7 100644 --- a/docs/user/ppl/cmd/rex.rst +++ b/docs/user/ppl/cmd/rex.rst @@ -211,7 +211,7 @@ Feature rex parse ============================= ============ ============ Pattern Type Java Regex Java Regex Named Groups Required Yes Yes -Filtering by Match No Yes +Multiple Named Groups Yes No Multiple Matches Yes No Text Substitution Yes No Offset Tracking Yes No diff --git a/docs/user/ppl/cmd/stats.rst b/docs/user/ppl/cmd/stats.rst index 872e2787d09..e61b4120410 100644 --- a/docs/user/ppl/cmd/stats.rst +++ b/docs/user/ppl/cmd/stats.rst @@ -34,6 +34,8 @@ The following table dataSources the aggregation functions and also indicates how +----------+-------------+-------------+ | LIST | Ignore | Ignore | +----------+-------------+-------------+ +| VALUES | Ignore | Ignore | ++----------+-------------+-------------+ Syntax @@ -57,8 +59,7 @@ stats [bucket_nullable=bool] ... [by-clause] * span-expression: optional, at most one. * Syntax: span(field_expr, interval_expr) - * Description: The unit of the interval expression is the natural unit by default. If the field is a date and time type field, and the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``. - + * Description: The unit of the interval expression is the natural unit by default. **If the field is a date/time type field, the aggregation results always ignore null bucket**. And the interval is in date/time units, you will need to specify the unit in the interval expression. For example, to split the field ``age`` into buckets by 10 years, it looks like ``span(age, 10)``. And here is another example of time span, the span to split a ``timestamp`` field into hourly intervals, it looks like ``span(timestamp, 1h)``. * Available time unit: +----------------------------+ @@ -182,6 +183,10 @@ Description Usage: MAX(expr). Returns the maximum value of expr. +For non-numeric fields, values are sorted lexicographically. + +Note: Non-numeric field support requires Calcite to be enabled (see `Configuration`_ section above). Available since version 3.3.0. + Example:: os> source=accounts | stats max(age); @@ -192,6 +197,16 @@ Example:: | 36 | +----------+ +Example with text field:: + + os> source=accounts | stats max(firstname); + fetched rows / total rows = 1/1 + +----------------+ + | max(firstname) | + |----------------| + | Nanette | + +----------------+ + MIN --- @@ -200,6 +215,10 @@ Description Usage: MIN(expr). Returns the minimum value of expr. +For non-numeric fields, values are sorted lexicographically. + +Note: Non-numeric field support requires Calcite to be enabled (see `Configuration`_ section above). Available since version 3.3.0. + Example:: os> source=accounts | stats min(age); @@ -210,6 +229,16 @@ Example:: | 28 | +----------+ +Example with text field:: + + os> source=accounts | stats min(firstname); + fetched rows / total rows = 1/1 + +----------------+ + | min(firstname) | + |----------------| + | Amber | + +----------------+ + VAR_SAMP -------- @@ -577,6 +606,52 @@ Example with result field rename:: | ["Amber","Hattie","Nanette","Dale"] | +-------------------------------------+ +VALUES +------ + +Description +>>>>>>>>>>> + +Version: 3.3.0 (Calcite engine only) + +Usage: VALUES(expr). Collects all unique values from the specified expression into a sorted array. Values are converted to strings, nulls are filtered, and duplicates are removed. + +The maximum number of unique values returned is controlled by the ``plugins.ppl.values.max.limit`` setting: + +* Default value is 0, which means unlimited values are returned +* Can be configured to any positive integer to limit the number of unique values +* See the `PPL Settings <../admin/settings.rst#plugins-ppl-values-max-limit>`_ documentation for more details + +Example with string fields:: + + PPL> source=accounts | stats values(firstname); + fetched rows / total rows = 1/1 + +-------------------------------------+ + | values(firstname) | + |-------------------------------------| + | ["Amber","Dale","Hattie","Nanette"] | + +-------------------------------------+ + +Example with numeric fields (sorted as strings):: + + PPL> source=accounts | stats values(age); + fetched rows / total rows = 1/1 + +---------------------------+ + | values(age) | + |---------------------------| + | ["28","32","33","36","39"] | + +---------------------------+ + +Example with result field rename:: + + PPL> source=accounts | stats values(firstname) as unique_names; + fetched rows / total rows = 1/1 + +-------------------------------------+ + | unique_names | + |-------------------------------------| + | ["Amber","Dale","Hattie","Nanette"] | + +-------------------------------------+ + Example 1: Calculate the count of events ======================================== @@ -833,3 +908,72 @@ PPL query:: | 1 | hattiebond@netagy.com | +-----+-----------------------+ +Example 16: Collect unique values in a field using VALUES +========================================================== + +The example shows how to collect all unique firstname values, sorted lexicographically with duplicates removed. + +PPL query:: + + PPL> source=accounts | stats values(firstname); + fetched rows / total rows = 1/1 + +-------------------------------------+ + | values(firstname) | + |-------------------------------------| + | ["Amber","Dale","Hattie","Nanette"] | + +-------------------------------------+ + + +Example 17: Span on date/time field always ignore null bucket +============================================================= + +Index example data: + ++-------+--------+------------+ +| Name | DEPTNO | birthday | ++=======+========+============+ +| Alice | 1 | 2024-04-21 | ++-------+--------+------------+ +| Bob | 2 | 2025-08-21 | ++-------+--------+------------+ +| Jeff | null | 2025-04-22 | ++-------+--------+------------+ +| Adam | 2 | null | ++-------+--------+------------+ + +PPL query:: + + PPL> source=example | stats count() as cnt by span(birthday, 1y) as year; + fetched rows / total rows = 3/3 + +-----+------------+ + | cnt | year | + |-----+------------| + | 1 | 2024-01-01 | + | 2 | 2025-01-01 | + +-----+------------+ + + +PPL query:: + + PPL> source=example | stats count() as cnt by span(birthday, 1y) as year, DEPTNO; + fetched rows / total rows = 3/3 + +-----+------------+--------+ + | cnt | year | DEPTNO | + |-----+------------+--------| + | 1 | 2024-01-01 | 1 | + | 1 | 2025-01-01 | 2 | + | 1 | 2025-01-01 | null | + +-----+------------+--------+ + + +PPL query:: + + PPL> source=example | stats bucket_nullable=false count() as cnt by span(birthday, 1y) as year, DEPTNO; + fetched rows / total rows = 3/3 + +-----+------------+--------+ + | cnt | year | DEPTNO | + |-----+------------+--------| + | 1 | 2024-01-01 | 1 | + | 1 | 2025-01-01 | 2 | + +-----+------------+--------+ + diff --git a/docs/user/ppl/functions/condition.rst b/docs/user/ppl/functions/condition.rst index 94c90e6a81f..a2d10a014b3 100644 --- a/docs/user/ppl/functions/condition.rst +++ b/docs/user/ppl/functions/condition.rst @@ -97,6 +97,26 @@ Example:: | default | null | Dale | +---------+----------+-----------+ +Nested IFNULL Pattern +>>>>>>>>>>>>>>>>>>>>> + +For OpenSearch versions prior to 3.1, COALESCE-like functionality can be achieved using nested IFNULL statements. This pattern is particularly useful in observability use cases where field names may vary across different data sources. + +Usage: ifnull(field1, ifnull(field2, ifnull(field3, default_value))) + +Example:: + + os> source=accounts | eval result = ifnull(employer, ifnull(firstname, ifnull(lastname, "unknown"))) | fields result, employer, firstname, lastname + fetched rows / total rows = 4/4 + +---------+----------+-----------+----------+ + | result | employer | firstname | lastname | + |---------+----------+-----------+----------| + | Pyrami | Pyrami | Amber | Duke | + | Netagy | Netagy | Hattie | Bond | + | Quility | Quility | Nanette | Bates | + | Dale | null | Dale | Adams | + +---------+----------+-----------+----------+ + NULLIF ------ @@ -255,15 +275,28 @@ Argument type: all the supported data type. Supports mixed data types with autom Return type: determined by the least restrictive common type among all arguments, with fallback to string if no common type can be determined Behavior: + - Returns the first value that is not null and not missing (missing includes non-existent fields) - Empty strings ("") and whitespace strings (" ") are considered valid values - If all arguments are null or missing, returns null - Automatic type coercion is applied to match the determined return type - If type conversion fails, the value is converted to string representation +- For best results, use arguments of the same data type to avoid unexpected type conversions + +Performance Considerations: + +- Optimized for multiple field evaluation, more efficient than nested IFNULL patterns +- Evaluates arguments sequentially, stopping at the first non-null value +- Consider field order based on likelihood of containing values to minimize evaluation overhead + +Limitations: + +- Type coercion may result in unexpected string conversions for incompatible types +- Performance may degrade with very large numbers of arguments Example:: - PPL> source=accounts | eval result = coalesce(employer, firstname, lastname) | fields result, firstname, lastname, employer + os> source=accounts | eval result = coalesce(employer, firstname, lastname) | fields result, firstname, lastname, employer fetched rows / total rows = 4/4 +---------+-----------+----------+----------+ | result | firstname | lastname | employer | @@ -276,7 +309,7 @@ Example:: Empty String Handling Examples:: - PPL> source=accounts | eval empty_field = "" | eval result = coalesce(empty_field, firstname) | fields result, empty_field, firstname + os> source=accounts | eval empty_field = "" | eval result = coalesce(empty_field, firstname) | fields result, empty_field, firstname fetched rows / total rows = 4/4 +--------+-------------+-----------+ | result | empty_field | firstname | @@ -287,7 +320,7 @@ Empty String Handling Examples:: | | | Dale | +--------+-------------+-----------+ - PPL> source=accounts | eval result = coalesce(" ", firstname) | fields result, firstname + os> source=accounts | eval result = coalesce(" ", firstname) | fields result, firstname fetched rows / total rows = 4/4 +--------+-----------+ | result | firstname | @@ -300,20 +333,20 @@ Empty String Handling Examples:: Mixed Data Types with Auto Coercion:: - PPL> source=accounts | eval result = coalesce(employer, balance, "fallback") | fields result, employer, balance + os> source=accounts | eval result = coalesce(employer, balance, "fallback") | fields result, employer, balance fetched rows / total rows = 4/4 +---------+----------+---------+ | result | employer | balance | |---------+----------+---------| | Pyrami | Pyrami | 39225 | - | Netagy | Netagy | 32838 | - | Quility | Quility | 4180 | - | 5686 | null | 5686 | + | Netagy | Netagy | 5686 | + | Quility | Quility | 32838 | + | 4180 | null | 4180 | +---------+----------+---------+ Non-existent Field Handling:: - PPL> source=accounts | eval result = coalesce(nonexistent_field, firstname, "unknown") | fields result, firstname + os> source=accounts | eval result = coalesce(nonexistent_field, firstname, "unknown") | fields result, firstname fetched rows / total rows = 4/4 +---------+-----------+ | result | firstname | @@ -324,6 +357,7 @@ Non-existent Field Handling:: | Dale | Dale | +---------+-----------+ + ISPRESENT --------- @@ -342,7 +376,7 @@ Synonyms: `ISNOTNULL`_ Example:: - PPL> source=accounts | where ispresent(employer) | fields employer, firstname + os> source=accounts | where ispresent(employer) | fields employer, firstname fetched rows / total rows = 3/3 +----------+-----------+ | employer | firstname | @@ -368,7 +402,7 @@ Return type: BOOLEAN Example:: - PPL> source=accounts | eval temp = ifnull(employer, ' ') | eval `isblank(employer)` = isblank(employer), `isblank(temp)` = isblank(temp) | fields `isblank(temp)`, temp, `isblank(employer)`, employer + os> source=accounts | eval temp = ifnull(employer, ' ') | eval `isblank(employer)` = isblank(employer), `isblank(temp)` = isblank(temp) | fields `isblank(temp)`, temp, `isblank(employer)`, employer fetched rows / total rows = 4/4 +---------------+---------+-------------------+----------+ | isblank(temp) | temp | isblank(employer) | employer | @@ -396,7 +430,7 @@ Return type: BOOLEAN Example:: - PPL> source=accounts | eval temp = ifnull(employer, ' ') | eval `isempty(employer)` = isempty(employer), `isempty(temp)` = isempty(temp) | fields `isempty(temp)`, temp, `isempty(employer)`, employer + os> source=accounts | eval temp = ifnull(employer, ' ') | eval `isempty(employer)` = isempty(employer), `isempty(temp)` = isempty(temp) | fields `isempty(temp)`, temp, `isempty(employer)`, employer fetched rows / total rows = 4/4 +---------------+---------+-------------------+----------+ | isempty(temp) | temp | isempty(employer) | employer | @@ -447,15 +481,15 @@ Return type: BOOLEAN Example:: - PPL> source=accounts | eval now = utc_timestamp() | eval a = earliest("now", now), b = earliest("-2d@d", now) | fields a, b | head 1 + os> source=accounts | eval now = utc_timestamp() | eval a = earliest("now", now), b = earliest("-2d@d", now) | fields a, b | head 1 fetched rows / total rows = 1/1 - +-------+-------+ - | a | b | - |-------+-------| - | False | True | - +-------+-------+ + +-------+------+ + | a | b | + |-------+------| + | False | True | + +-------+------+ - PPL> source=nyc_taxi | where earliest('07/01/2014:00:30:00', timestamp) | stats COUNT() as cnt + os> source=nyc_taxi | where earliest('07/01/2014:00:30:00', timestamp) | stats COUNT() as cnt fetched rows / total rows = 1/1 +-----+ | cnt | @@ -479,15 +513,15 @@ Return type: BOOLEAN Example:: - PPL> source=accounts | eval now = utc_timestamp() | eval a = latest("now", now), b = latest("+2d@d", now) | fields a, b | head 1 + os> source=accounts | eval now = utc_timestamp() | eval a = latest("now", now), b = latest("+2d@d", now) | fields a, b | head 1 fetched rows / total rows = 1/1 - +-------+-------+ - | a | b | - |-------+-------| - | False | True | - +-------+-------+ + +------+------+ + | a | b | + |------+------| + | True | True | + +------+------+ - PPL> source=nyc_taxi | where latest('07/21/2014:04:00:00', timestamp) | stats COUNT() as cnt + os> source=nyc_taxi | where latest('07/21/2014:04:00:00', timestamp) | stats COUNT() as cnt fetched rows / total rows = 1/1 +-----+ | cnt | diff --git a/docs/user/ppl/reference/splunk_to_ppl_cheat_sheet.md b/docs/user/ppl/reference/splunk_to_ppl_cheat_sheet.md new file mode 100644 index 00000000000..031d858e8ca --- /dev/null +++ b/docs/user/ppl/reference/splunk_to_ppl_cheat_sheet.md @@ -0,0 +1,240 @@ +# Splunk to OpenSearch PPL Cheat Sheet + +This cheat sheet helps Splunk users transition to OpenSearch's PPL. It maps common Splunk Search Processing Language (SPL) commands to their PPL equivalents with examples. + +## Structure and Concepts + +| Aspect | Splunk SPL | OpenSearch PPL | Notes | +|--------|------------|---------------|-------| +| Query structure | `search terms \| command` | `search term source = index \| command` | PPL requires explicit source at the beginning | +| Index reference | `index=name*` | `source=name*` | Different command to specify data source, [PPL support refering to multiple indices](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/general/identifiers.rst#multiple-indices)| +| Raw field | Special `_raw` field | Identify a field in your OpenSearch data that contains the text content you want to work with (often `message` or `content` fields in log data) | default field configured by the index.query.default_field setting (defaults to * which searches all fields) | +| Time field | Special `_time` field | User-specified timestamp field | PPL use @timestamp by default | + + +## Command Reference + +This table provides a mapping between Splunk SPL commands and their OpenSearch PPL equivalents: + +| Splunk SPL | OpenSearch PPL | Purpose | +|------------|---------------|---------| +| append | [append](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/append.rst) | Append results from subsearch | +| appendcols | [appendcols](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/appendcol.rst) | Append columns from subsearch | +| bin | [bin](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/bin.rst) | Group numeric values into bins | +| bucket | [bin](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/bin.rst) | Group numeric values into bins | +| dedup | [dedup](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/dedup.rst) | Remove duplicate results | +| eval | [eval](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/eval.rst) | Calculate and create new fields | +| eventstats | [eventstats](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/eventstats.rst) | Calculate statistics while preserving events | +| mvexpand | [expand](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/expand.rst) | Expand multi-value fields | +| fields | [fields](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/fields.rst) | Include or exclude fields | +| fillnull | [fillnull](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/fillnull.rst) | Replace null values | +| head | [head](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/head.rst) | Retrieve the first N results | +| join | [join](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/join.rst) | Combine results from multiple sources | +| lookup | [lookup](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/lookup.rst) | Enrich data with lookups | +| rare | [rare](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/rare.rst) | Find the least common values | +| regex | [regex](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/regex.rst) | Filter with regular expression pattern | +| rename | [rename](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/rename.rst) | Rename fields in results | +| reverse | [reverse](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/reverse.rst) | Reverse the order of search results | +| rex | [rex](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/rex.rst) | Extract with regular expression pattern | +| search | [search](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/search.rst) | Basic searching of data | +| sort | [sort](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/sort.rst) | Sort results by specified fields | +| spath | [spath](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/spath.rst) | Extracting fields from structured text data | +| stats | [stats](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/stats.rst) | Statistical aggregation of data | +| subsearch | [subsearch](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/subquery.rst) | Enrich main search | +| table | [table](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/table.rst) | Select specific fields to display | +| timechart | [timechart](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/timechart.rst) | Statistical aggregation of time-series data | +| top | [top](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/top.rst) | Find the most common values | +| trendline | [trendline](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/trendline.rst) | Calculate moving averages of fields | +| where | [where](https://github.com/opensearch-project/sql/blob/main/docs/user/ppl/cmd/where.rst) | Filter results based on conditions | + + +## Example Query Conversions + +**Simple search:** +- Splunk: `error failed status=500` +- PPL: ```source=`*` error failed status=500``` + +**Aggregation:** +- Splunk: `... | stats count BY host, status | sort -count` +- PPL: `... | stats count() by host, status | sort - count` + +**Time-based query:** +- Splunk: `... earliest=-7d | timechart span=1h count BY host` +- PPL: `... | where timestamp >= date_sub(now(), INTERVAL 1 DAY) | stats count() by span(timestamp, 1h), host` + +**Complex calculation:** +- Splunk: `... | eval mb=bytes/1024/1024 | stats avg(mb) AS avg_mb BY host | where avg_mb > 100` +- PPL: `... | eval mb=bytes/1024/1024 | stats avg(mb) as avg_mb by host | where avg_mb > 100` + +## Basic Search Syntax + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Basic search | `error` | `error` | Same syntax | +| Multiple keyword search | `error failed` | `error failed` | Same syntax | +| Quoted phrases | `"error failed"` | `"error failed"` | Same syntax | +| Field value equals | `field=404` | `field=404` | Same syntax | +| Multiple values | `field IN (404, 503)` | `field in (404, 503)` | Same syntax | +| Field doesn't equal | `field!=404` | `field!=404` | Same syntax | +| Wildcard search | `field=value*` | `field=value*` | Same syntax | + +## Field Selection and Manipulation + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Select fields | `... \| fields field1, field2` | `... \| fields field1, field2` | Same syntax | +| Exclude fields | `... \| fields - field3` | `... \| fields - field3` | Same syntax | +| Rename fields | `... \| rename field1 AS new_name` | `... \| rename field1 as new_name` | Same syntax | +| Calculate field | `... \| eval new_field=field1 + field2` | `... \| eval new_field = field1 + field2` | Same syntax | + +## Filtering + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Filter results | `... \| where field > 100` | `... \| where field > 100` | Same syntax | +| Compound filter | `... \| where field1=200 OR field2=203` | `... \| where field1=200 or field2=203` | Same syntax | + + +## Aggregation + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Count | `... \| stats count` | `... \| stats count` | Same syntax | +| Count by field | `... \| stats count by field` | `... \| stats count by field` | Same syntax | +| Multiple aggregations | `... \| stats count, avg(field1) by field2` | `... \| stats count, avg(field1) by field2` | Same syntax | +| Distinct count | `... \| stats dc(field)` | `... \| stats dc(field)` | Same syntax | +| Min/Max | `... \| stats min(field), max(field)` | `... \| stats min(field), max(field)` | Same syntax | +| Percentiles | `... \| stats perc95(field)` | `... \| stats perc95(field)` | Same syntax | + +## Sorting and Limiting + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Sort ascending | `... \| sort field` | `... \| sort field` | Same syntax | +| Sort descending | `... \| sort -field` | `... \| sort -field` | Same syntax | +| Sort multiple | `... \| sort field1, -field2` | `... \| sort field1, -field2` | Same syntax | +| Limit results | `... \| head 10` | `... \| head 10` | Same syntax | +| Get last results | `... \| tail 10` | `... \| tail 10` | Same syntax | + +## Rex vs Parse + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Basic extraction | `... \| rex field=address "(?\d+) (?.+)"` | `... \| rex address "(?\d+) (?.+)"` | Same syntax | +| Field specification | `... \| rex field=address ...` | `... \| rex field=address ...` | Same syntax | +| Search and replace mode | `... \| rex field=address mode=sed "s/\d+//g"` | `... \| rex field=address mode=sed "s/\d+//g"` | Same syntax | +| Field override | `... \| rex field=address "(?
    .+)"` | `... \| rex address "(?
    .+)"` | Same syntax | +| Default field (_raw) | `... \| rex "(?\d+) (?.+)"` | Not supported | PPL does not support implicit _raw field and requires explicit field specification | + +## Time Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Relative time | `earliest=-1d latest=now()` | `earliest("-1d", @timestamp) and latest("now", @timestamp)` | PPL supports earliest() and latest() functions | +| Time extraction | `... \| eval hour=strftime(now(), "%H")` | `... \| eval hour = strftime(now(), '%H')` | Same syntax | +| Time bucket | `... \| bin _time span=5m \| stats count by _time` | `... \| stats count by span(@timestamp, 5m)` | PPL uses `span()` | + +## Dedup + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Deduplicate | `... \| dedup field1, field2` | `... \| dedup field1, field2` | Same syntax | +| Deduplicate with count | `... \| dedup 2 field1` | `... \| dedup 2 field1` | Same syntax | + +## Lookup and Joins + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Basic lookup | `... \| lookup vendors product_id` | `... \| lookup vendors product_id` | Same syntax | +| Multiple mapping fields | `... \| lookup vendors product_id, category` | `... \| lookup vendors product_id, category` | Same syntax | +| Aliased field lookup | `... \| lookup vendors product AS id` | `... \| lookup vendors product as id` | PPL uses lowercase "as" | +| Lookup with append | Not applicable | `... \| lookup vendors product_id append dept as department` | PPL-specific feature | +| Lookup with replace | Not applicable | `... \| lookup vendors product_id replace dept as department` | PPL-specific feature | +| Inner join | `... \| join type=inner vendors [search index=vendors]` | `... \| inner join vendors` | Different syntax format | +| Left join | `... \| join type=left vendors [search index=vendors]` | `... \| left join vendors` | Different syntax format | +| Join with ON clause | `... \| join type=inner left=a right=b where a.id = b.id vendors` | `... \| inner join left=a right=b ON a.id = b.id vendors` | PPL uses "ON" instead of "where" | +| Append columns | `... \| appendcols [search source=other_index \| fields id, status]` | `... \| appendcols [source=other_index \| fields id, status]` | Similar syntax | + +## Field Manipulation + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Include fields | `... \| fields field1, field2` | `... \| fields field1, field2` | Same syntax | +| Exclude fields | `... \| fields - field3` | `... \| fields - field3` | Same syntax | +| Rename fields | `... \| rename field1 as new_name` | `... \| rename field1 as new_name` | PPL uses lowercase "as" | +| Replace null values | `... \| fillnull value=0 field1, field2` | `... \| fillnull with 0 in field1, field2` | Similar syntax but different format | + +## Handling Null Values + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Basic null replacement | `... \| fillnull value=0 field1` | `... \| fillnull with 0 in field1` | Similar syntax but uses `with...in` format | +| Multiple fields | `... \| fillnull value="N/A" field1, field2, field3` | `... \| fillnull with 'N/A' in field1, field2, field3` | Similar syntax but uses `with...in` format | + +## Results Limiting + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| First N results | `... \| head 10` | `... \| head 10` | Same syntax | +| Last N results | `... \| tail 10` | `... \| tail 10` | Same syntax | +| Moving average | `... \| trendline sma5(value)` | `... \| trendline sma5(value)` | Same syntax | +| Top values | `... \| top 10 field` | `... \| top 10 field` | Same syntax | +| Rare values | `... \| rare 10 field` | `... \| rare 10 field` | Same syntax | + +## String Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| String concatenation | `... \| eval result=field1 + " " + field2` | `... \| eval result = concat(field1, ' ', field2)` | PPL requires `concat()` function | +| Substring | `... \| eval result=substr(field, 0, 5)` | `... \| eval result = substring(field, 0, 5)` | Different function name | +| String length | `... \| eval length=len(field)` | `... \| eval length = length(field)` | Different function name | +| Lowercase | `... \| eval result=lower(field)` | `... \| eval result = lower(field)` | Same syntax | +| Uppercase | `... \| eval result=upper(field)` | `... \| eval result = upper(field)` | Same syntax | +| Replace | `... \| eval result=replace(field, "pattern", "replacement")` | `... \| eval result = replace(field, 'pattern', 'replacement')` | Same syntax | +| Trim whitespace | `... \| eval result=trim(field)` | `... \| eval result = trim(field)` | Same syntax | +| Contains (wildcard) | `... \| eval result=like(field, "%pattern%")` | `... \| eval result = like(field, '%pattern%')` | Same syntax | + +## Conditional Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| If condition | `... \| eval result=if(field > 100, "High", "Low")` | `... \| eval result = if(field > 100, 'High', 'Low')` | Same syntax | +| Case statement | `... \| eval grade=case(field > 90, "A", field > 80, "B", 1==1, "C")` | `... \| eval grade = case(field > 90 then 'A', field > 80 then 'B', else 'C')` | PPL uses `then` and `else` keywords | +| NULL check | `... \| eval result=if(isnull(field), "Missing", field)` | `... \| eval result = if(isnull(field), 'Missing', field)` | Same syntax | +| Empty check | `... \| eval result=if(isnotnull(field), field, "Default")` | `... \| eval result = if(isnotnull(field), field, 'Default')` | Same syntax | +| Coalesce (first non-null) | `... \| eval result=coalesce(field1, field2, "default")` | `... \| eval result = coalesce(field1, field2, 'default')` | Same syntax | + +## Math Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Addition | `... \| eval sum=field1 + field2` | `... \| eval sum = field1 + field2` | Same syntax | +| Subtraction | `... \| eval diff=field1 - field2` | `... \| eval diff = field1 - field2` | Same syntax | +| Multiplication | `... \| eval product=field1 * field2` | `... \| eval product = field1 * field2` | Same syntax | +| Division | `... \| eval quotient=field1 / field2` | `... \| eval quotient = field1 / field2` | Same syntax | +| Modulo | `... \| eval remainder=field1 % field2` | `... \| eval remainder = field1 % field2` | Same syntax | +| Absolute value | `... \| eval result=abs(field)` | `... \| eval result = abs(field)` | Same syntax | +| Round | `... \| eval result=round(field, 2)` | `... \| eval result = round(field, 2)` | Same syntax | +| Ceiling | `... \| eval result=ceiling(field)` | `... \| eval result = ceil(field)` | Different function name | +| Floor | `... \| eval result=floor(field)` | `... \| eval result = floor(field)` | Same syntax | +| Power | `... \| eval result=pow(field, 2)` | `... \| eval result = pow(field, 2)` | Same syntax | +| Square root | `... \| eval result=sqrt(field)` | `... \| eval result = sqrt(field)` | Same syntax | + +## Date and Time Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| Current time | `... \| eval now=now()` | `... \| eval now = now()` | Same syntax | +| Format time | `... \| eval date=strftime(now(), "%Y-%m-%d")` | `... \| eval date = strftime(now(), "%Y-%m-%d")` | Same syntax | +| Extract part | `... \| eval month=date_part("month", _time)` | `... \| eval month = date_part('month', @timestamp)` | Same syntax | +| Day ago | `... \| eval yesterday=relative_time(now(), "-1d")` | `... \| eval yesterday = date_sub(now(), INTERVAL 1 DAY)` | PPL uses interval syntax | +| Day ahead | `... \| eval tomorrow=relative_time(now(), "+1d")` | `... \| eval tomorrow = date_add(now(), INTERVAL 1 DAY)` | PPL uses interval syntax | +| Time difference | `... \| eval diff=(_time2 - _time1)` | `... \| eval diff = date_diff('second', timestamp1, timestamp2)` | PPL uses function | + +## Other Functions + +| Operation | Splunk SPL | OpenSearch PPL | Notes | +|-----------|------------|---------------|-------| +| MD5 hash | Not native | `... \| eval hash = md5('string')` | PPL-specific feature | +| SHA1 hash | Not native | `... \| eval hash = sha1('string')` | PPL-specific feature | +| JSON extraction | `... \| spath input=data path=user.name output=username` | `... \| eval username = json_extract(data, '$.user.name')` | Different approach | diff --git a/integ-test/build.gradle b/integ-test/build.gradle index 4c926766674..e32e7b6c099 100644 --- a/integ-test/build.gradle +++ b/integ-test/build.gradle @@ -35,7 +35,7 @@ import java.util.stream.Collectors plugins { id "de.undercouch.download" version "5.3.0" - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } apply plugin: 'opensearch.build' @@ -709,10 +709,10 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java index 7d2d1b1c42e..b355e5f9b2b 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteBinCommandIT.java @@ -25,6 +25,7 @@ public void init() throws Exception { enableCalcite(); loadIndex(Index.ACCOUNT); loadIndex(Index.BANK); + loadIndex(Index.EVENTS_NULL); loadIndex(Index.TIME_TEST_DATA); } @@ -200,8 +201,6 @@ public void testBinOnlyWithoutAggregation() throws IOException { } @Test - @Ignore - // https://github.com/opensearch-project/sql/issues/4063 public void testBinWithTimestampAggregation() throws IOException { // Test bin operation with fields only - no aggregation JSONObject result = @@ -865,4 +864,86 @@ public void testBinFloatingPointSpanWithStatsCount() throws IOException { // Test floating point spans with stats aggregation - verify proper decimal formatting verifyDataRows(result, rows(279L, "0.0-15000.5"), rows(319L, "15000.5-30001.0")); } + + @Test + public void testStatsWithBinsOnTimeField_Count() throws IOException { + // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 + enabledOnlyWhenPushdownIsEnabled(); + + JSONObject result = + executeQuery("source=events_null | bin @timestamp bins=3 | stats count() by @timestamp"); + // TODO: @timestamp should keep date as its type, to be addressed by this issue: + // https://github.com/opensearch-project/sql/issues/4317 + verifySchema(result, schema("count()", null, "bigint"), schema("@timestamp", null, "string")); + // auto_date_histogram will choose span=5m for bins=3 + verifyDataRows(result, rows(5, "2024-07-01 00:00:00"), rows(1, "2024-07-01 00:05:00")); + + result = + executeQuery("source=events_null | bin @timestamp bins=6 | stats count() by @timestamp"); + // auto_date_histogram will choose span=1m for bins=6 + verifyDataRows( + result, + rows(1, "2024-07-01 00:00:00"), + rows(1, "2024-07-01 00:01:00"), + rows(1, "2024-07-01 00:02:00"), + rows(1, "2024-07-01 00:03:00"), + rows(1, "2024-07-01 00:04:00"), + rows(1, "2024-07-01 00:05:00")); + + result = + executeQuery("source=events_null | bin @timestamp bins=100 | stats count() by @timestamp"); + // auto_date_histogram will choose span=5s for bins=100, it will produce many empty buckets but + // we will filter them and left only 6 buckets. + verifyDataRows( + result, + rows(1, "2024-07-01 00:00:00"), + rows(1, "2024-07-01 00:01:00"), + rows(1, "2024-07-01 00:02:00"), + rows(1, "2024-07-01 00:03:00"), + rows(1, "2024-07-01 00:04:00"), + rows(1, "2024-07-01 00:05:00")); + } + + @Test + public void testStatsWithBinsOnTimeField_Avg() throws IOException { + // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 + enabledOnlyWhenPushdownIsEnabled(); + + JSONObject result = + executeQuery( + "source=events_null | bin @timestamp bins=3 | stats avg(cpu_usage) by @timestamp"); + // TODO: @timestamp should keep date as its type, to be addressed by this issue: + // https://github.com/opensearch-project/sql/issues/4317 + verifySchema( + result, schema("avg(cpu_usage)", null, "double"), schema("@timestamp", null, "string")); + // auto_date_histogram will choose span=5m for bins=3 + verifyDataRows(result, rows(44.62, "2024-07-01 00:00:00"), rows(50.0, "2024-07-01 00:05:00")); + + result = + executeQuery( + "source=events_null | bin @timestamp bins=6 | stats avg(cpu_usage) by @timestamp"); + // auto_date_histogram will choose span=1m for bins=6 + verifyDataRows( + result, + rows(45.2, "2024-07-01 00:00:00"), + rows(38.7, "2024-07-01 00:01:00"), + rows(55.3, "2024-07-01 00:02:00"), + rows(42.1, "2024-07-01 00:03:00"), + rows(41.8, "2024-07-01 00:04:00"), + rows(50.0, "2024-07-01 00:05:00")); + + result = + executeQuery( + "source=events_null | bin @timestamp bins=100 | stats avg(cpu_usage) by @timestamp"); + // auto_date_histogram will choose span=5s for bins=100, it will produce many empty buckets but + // we will filter them and left only 6 buckets. + verifyDataRows( + result, + rows(45.2, "2024-07-01 00:00:00"), + rows(38.7, "2024-07-01 00:01:00"), + rows(55.3, "2024-07-01 00:02:00"), + rows(42.1, "2024-07-01 00:03:00"), + rows(41.8, "2024-07-01 00:04:00"), + rows(50.0, "2024-07-01 00:05:00")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java index f32d5192e82..02ac75bb4b3 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExpandCommandIT.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.calcite.remote; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index c4f02502813..9af4f020eb8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -5,12 +5,12 @@ package org.opensearch.sql.calcite.remote; -import static org.junit.Assert.assertTrue; -import static org.opensearch.sql.legacy.TestUtils.*; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_LOGS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_STRINGS; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_WEBLOGS; import static org.opensearch.sql.util.MatcherUtils.assertJsonEqualsIgnoreId; import static org.opensearch.sql.util.MatcherUtils.assertYamlEqualsJsonIgnoreId; @@ -303,6 +303,23 @@ public void testExplainBinWithBins() throws IOException { explainQueryToString("source=opensearch-sql_test_index_account | bin age bins=3 | head 5")); } + @Test + public void testExplainStatsWithBinsOnTimeField() throws IOException { + // TODO: Remove this after addressing https://github.com/opensearch-project/sql/issues/4317 + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_stats_bins_on_time.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=events | bin @timestamp bins=3 | stats count() by @timestamp")); + + expected = loadExpectedPlan("explain_stats_bins_on_time2.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=events | bin @timestamp bins=3 | stats avg(cpu_usage) by @timestamp")); + } + @Test public void testExplainBinWithSpan() throws IOException { String expected = loadExpectedPlan("explain_bin_span.json"); @@ -501,6 +518,15 @@ public void testListAggregationExplain() throws IOException { "source=opensearch-sql_test_index_account | stats list(age) as age_list")); } + @Test + public void testValuesAggregationExplain() throws IOException { + String expected = loadExpectedPlan("explain_values_aggregation.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats values(age) as age_values")); + } + @Test public void testRegexExplain() throws IOException { String query = @@ -621,6 +647,159 @@ public void testPushdownLimitIntoAggregation() throws IOException { + " head 100 | head 10 from 10 ")); } + @Test + public void testExplainMaxOnStringField() throws IOException { + String expected = loadExpectedPlan("explain_max_string_field.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString("source=opensearch-sql_test_index_account | stats max(firstname)")); + } + + @Test + public void testExplainMinOnStringField() throws IOException { + String expected = loadExpectedPlan("explain_min_string_field.json"); + assertJsonEqualsIgnoreId( + expected, + explainQueryToString("source=opensearch-sql_test_index_account | stats min(firstname)")); + } + + @Test + @Override + public void testCountAggPushDownExplain() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + // should be optimized by hits.total.value + String expected = loadExpectedPlan("explain_count_agg_push1.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString("source=opensearch-sql_test_index_account | stats count() as cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push2.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(lastname) as cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push3.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats count(name) as" + + " cnt")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push4.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count() as c1, count() as c2")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push5.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(lastname) as c1," + + " count(lastname) as c2")); + + // should be optimized + expected = loadExpectedPlan("explain_count_agg_push6.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats" + + " count(lastname), count(name)")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push7.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(balance + 1) as cnt")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push8.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count() as c1, count(lastname) as" + + " c2")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push9.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | stats count(firstname), count(lastname)")); + + // should not be optimized + expected = loadExpectedPlan("explain_count_agg_push10.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + "source=opensearch-sql_test_index_account | eval name = lastname | stats" + + " count(firstname), count(name)")); + } + + @Test + public void testExplainCountsByAgg() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_agg_counts_by1.yaml"); + // case of only count(): doc_count works + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(), count() as c1 by gender", TEST_INDEX_ACCOUNT))); + + // count(FIELD) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by2.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(balance) as c1, count(balance) as c2 by gender", + TEST_INDEX_ACCOUNT))); + + // count(FIELD) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by3.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | eval account_number_alias = account_number" + + " | stats count(account_number), count(account_number_alias) as c2 by gender", + TEST_INDEX_ACCOUNT))); + + // count() + count(FIELD)): doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by4.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(), count(account_number) by gender", TEST_INDEX_ACCOUNT))); + + // count(FIELD1) + count(FIELD2)) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by5.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats count(balance), count(account_number) by gender", + TEST_INDEX_ACCOUNT))); + + // case of count(EXPRESSION) by: doc_count doesn't work + expected = loadExpectedPlan("explain_agg_counts_by6.yaml"); + assertYamlEqualsJsonIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | eval b_1 = balance + 1" + + " | stats count(b_1), count(pow(balance, 2)) as c3 by gender", + TEST_INDEX_ACCOUNT))); + } + @Test public void testExplainSortOnMetricsNoBucketNullable() throws IOException { // TODO enhancement later: https://github.com/opensearch-project/sql/issues/4282 @@ -663,4 +842,31 @@ public void testStrftimeFunctionExplain() throws IOException { String expected = loadExpectedPlan("explain_strftime_function.json"); assertJsonEqualsIgnoreId(expected, result); } + + // Script generation is not stable in v2 + @Test + public void testExplainPushDownScriptsContainingUDT() throws IOException { + assertJsonEqualsIgnoreId( + loadExpectedPlan("explain_filter_script_ip_push.json"), + explainQueryToString( + String.format( + "source=%s | where cidrmatch(host, '0.0.0.0/24') | fields host", + TEST_INDEX_WEBLOGS))); + + assertJsonEqualsIgnoreId( + loadExpectedPlan("explain_agg_script_timestamp_push.json"), + explainQueryToString( + String.format( + "source=%s | eval t = unix_timestamp(birthdate) | stats count() by t | sort t |" + + " head 3", + TEST_INDEX_BANK))); + + assertJsonEqualsIgnoreId( + loadExpectedPlan("explain_agg_script_udt_arg_push.json"), + explainQueryToString( + String.format( + "source=%s | eval t = date_add(birthdate, interval 1 day) | stats count() by" + + " span(t, 1d)", + TEST_INDEX_BANK))); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFlattenCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFlattenCommandIT.java index 9b72d67dfbd..647f11fd468 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFlattenCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFlattenCommandIT.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.calcite.remote; diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoPointFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoPointFormatsIT.java index c136d4c3e59..9b0a4882b15 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoPointFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteGeoPointFormatsIT.java @@ -5,7 +5,6 @@ package org.opensearch.sql.calcite.remote; -import java.io.IOException; import org.opensearch.sql.ppl.GeoPointFormatsIT; public class CalciteGeoPointFormatsIT extends GeoPointFormatsIT { @@ -14,17 +13,4 @@ public void init() throws Exception { super.init(); enableCalcite(); } - - @Override - public void testReadingGeoHash() throws IOException { - withFallbackEnabled( - () -> { - try { - super.testReadingGeoHash(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }, - "Need to support metadata, https://github.com/opensearch-project/sql/issues/3333"); - } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java index e8bebaf291c..c374f8bbb29 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteMultiValueStatsIT.java @@ -5,6 +5,8 @@ package org.opensearch.sql.calcite.remote; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_CALCS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NONNUMERIC; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_DATATYPE_NUMERIC; @@ -15,6 +17,7 @@ import java.io.IOException; import java.util.List; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.ppl.PPLIntegTestCase; @@ -274,4 +277,204 @@ public void testListFunctionWithArithmeticExpression() throws IOException { verifySchema(response, schema("arithmetic_list", "array")); verifyDataRows(response, rows(List.of("9", "14", "3"))); } + + // ==================== VALUES Function Tests ==================== + + @Test + public void testValuesFunctionWithBoolean() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats values(boolean_value) as bool_values", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifySchema(response, schema("bool_values", "array")); + // VALUES returns unique values sorted lexicographically + verifyDataRows(response, rows(List.of("true"))); + } + + @Test + public void testValuesFunctionWithInteger() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats values(integer_number) as int_values", + TEST_INDEX_DATATYPE_NUMERIC)); + verifySchema(response, schema("int_values", "array")); + verifyDataRows(response, rows(List.of("2"))); + } + + @Test + public void testValuesFunctionWithString() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats values(keyword_value) as keyword_values", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifySchema(response, schema("keyword_values", "array")); + verifyDataRows(response, rows(List.of("keyword"))); + } + + @Test + public void testValuesFunctionWithDuplicates() throws IOException { + // Test that VALUES deduplicates values + JSONObject response = + executeQuery( + String.format( + "source=%s | head 10 | stats values(bool0) as unique_bool_values", + TEST_INDEX_CALCS)); + verifySchema(response, schema("unique_bool_values", "array")); + // VALUES should return unique values only, sorted lexicographically + // The actual values depend on the test data - bool0 contains true/false values + assert response.has("datarows"); + // Verify that we get at most 2 unique boolean values (true/false) + assert response.getJSONArray("datarows").getJSONArray(0).getJSONArray(0).length() <= 2; + } + + @Test + public void testValuesFunctionWithNullValues() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | head 5 | stats values(int0) as int_values", TEST_INDEX_CALCS)); + verifySchema(response, schema("int_values", "array")); + // Nulls are filtered out by values function + // VALUES returns sorted unique values + verifyDataRows(response, rows(List.of("1", "7"))); + } + + @Test + public void testValuesFunctionGroupBy() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | head 5 | stats values(num0) as num_values by str0", TEST_INDEX_CALCS)); + verifySchema(response, schema("num_values", "array"), schema("str0", null, "string")); + + // Group by str0 field - should have different groups with their respective unique num0 values + // First 5 rows have: + // - FURNITURE: num0 values are 12.3, -12.3 + // - OFFICE SUPPLIES: num0 values are 15.7, -15.7, 3.5 + // VALUES returns unique values sorted lexicographically as strings + verifyDataRows( + response, + rows(List.of("-12.3", "12.3"), "FURNITURE"), + rows(List.of("-15.7", "15.7", "3.5"), "OFFICE SUPPLIES")); + } + + @Test + public void testValuesFunctionMultipleFields() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | head 3 | stats values(str2) as str_values, values(int2) as int_values", + TEST_INDEX_CALCS)); + verifySchema(response, schema("str_values", "array"), schema("int_values", "array")); + + // VALUES should return unique sorted values for each field + assert response.has("datarows"); + // Values should be unique and sorted lexicographically + verifyDataRows(response, rows(List.of("one", "three", "two"), List.of("-4", "5"))); + } + + @Test + public void testValuesFunctionWithObjectField() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats values(object_value.first) as object_field_values", + TEST_INDEX_DATATYPE_NONNUMERIC)); + verifySchema(response, schema("object_field_values", "array")); + verifyDataRows(response, rows(List.of("Dale"))); + } + + @Test + public void testValuesFunctionEmptyResult() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | where str0 = 'NONEXISTENT' | stats values(num0) as empty_values", + TEST_INDEX_CALCS)); + verifySchema(response, schema("empty_values", "array")); + + assert response.has("datarows"); + // When no records match, VALUES returns null (not an empty list) + verifyDataRows(response, rows((List) null)); + } + + @Test + public void testValuesFunctionWithUnlimitedValues() throws IOException { + // This test verifies that when the limit is set to 0 (unlimited), + // all unique values are returned + JSONObject response = + executeQuery( + String.format( + "source=%s | head 100 | stats values(int2) as all_values", TEST_INDEX_CALCS)); + verifySchema(response, schema("all_values", "array")); + + // With the default setting of 0 (unlimited), all unique values should be returned + // The actual number depends on the test data + assert response.has("datarows"); + JSONArray rows = response.getJSONArray("datarows"); + assertNotNull(rows); + assertTrue(rows.length() > 0); + } + + @Test + public void testValuesFunctionRespectsConfiguredLimit() throws IOException, InterruptedException { + // Test 1: Set limit to 3 and verify only 3 values are returned + updateClusterSettings(new ClusterSetting(TRANSIENT, "plugins.ppl.values.max.limit", "3")); + + // Wait a moment for the setting to propagate + Thread.sleep(1000); + + JSONObject response = + executeQuery( + String.format("source=%s | stats values(int2) as limited_values", TEST_INDEX_CALCS)); + verifySchema(response, schema("limited_values", "array")); + + assert response.has("datarows"); + JSONArray rows = response.getJSONArray("datarows"); + assertNotNull(rows); + assertTrue(rows.length() > 0); + + if (!rows.isNull(0)) { + JSONArray values = rows.getJSONArray(0).getJSONArray(0); + assertNotNull(values); + // With limit set to 3, should have at most 3 values + assertTrue( + "Expected at most 3 values with limit=3, but got " + values.length() + ": " + values, + values.length() <= 3); + + // Verify values are in lexicographical order + for (int i = 1; i < values.length(); i++) { + String prev = values.getString(i - 1); + String curr = values.getString(i); + assertTrue(prev.compareTo(curr) <= 0); + } + } + + // Test 2: Set limit to 0 (unlimited) and verify more values are returned + updateClusterSettings(new ClusterSetting(TRANSIENT, "plugins.ppl.values.max.limit", "0")); + + response = + executeQuery( + String.format("source=%s | stats values(int2) as unlimited_values", TEST_INDEX_CALCS)); + verifySchema(response, schema("unlimited_values", "array")); + + rows = response.getJSONArray("datarows"); + assertNotNull(rows); + + if (!rows.isNull(0)) { + JSONArray unlimitedValues = rows.getJSONArray(0).getJSONArray(0); + assertNotNull(unlimitedValues); + // With limit 0 (unlimited), should have all unique values from the dataset + // The test data has more than 3 unique values, so this should be > 3 + assertTrue( + "Expected more than 3 values with unlimited setting, but got " + unlimitedValues.length(), + unlimitedValues.length() > 3); + } + + // Reset the setting to default + updateClusterSettings(new ClusterSetting(TRANSIENT, "plugins.ppl.values.max.limit", null)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java index 27ee2d999da..2e9d62f0ace 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLAggregationIT.java @@ -518,15 +518,20 @@ public void testCountByNullableTimeSpan() throws IOException { JSONObject actual = executeQuery( String.format( - "source=%s | head 5 | stats count(datetime0), count(datetime1) by span(datetime1," - + " 15 minute) as datetime_span", + "source=%s | head 5 | stats count(datetime0), count(datetime1) by span(time1," + + " 15 minute) as time_span", TEST_INDEX_CALCS)); verifySchema( actual, - schema("datetime_span", "timestamp"), + schema("time_span", "time"), schema("count(datetime0)", "bigint"), schema("count(datetime1)", "bigint")); - verifyDataRows(actual, rows(5, 0, null)); + verifyDataRows( + actual, + rows(1, 0, "19:30:00"), + rows(1, 0, "02:00:00"), + rows(1, 0, "09:30:00"), + rows(1, 0, "22:45:00")); } @Test @@ -1176,4 +1181,69 @@ public void testMedian() throws IOException { verifySchema(actual, schema("median(balance)", "bigint")); verifyDataRows(actual, rows(32838)); } + + @Test + public void testStatsMaxOnStringField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats max(firstname)", TEST_INDEX_BANK)); + verifySchema(actual, schema("max(firstname)", "string")); + verifyDataRows(actual, rows("Virginia")); + } + + @Test + public void testStatsMinOnStringField() throws IOException { + JSONObject actual = + executeQuery(String.format("source=%s | stats min(firstname)", TEST_INDEX_BANK)); + verifySchema(actual, schema("min(firstname)", "string")); + verifyDataRows(actual, rows("Amber JOHnny")); + } + + @Test + public void testStatsCountOnFunctionsWithUDTArg() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | eval t = unix_timestamp(birthdate) | stats count() by t | sort -t", + TEST_INDEX_BANK)); + verifySchema(response, schema("count()", "bigint"), schema("t", "double")); + verifyDataRows( + response, + rows(1, 1542152000), + rows(1, 1534636800), + rows(1, 1533945600), + rows(1, 1530057600), + rows(1, 1529712000), + rows(1, 1511136000), + rows(1, 1508716800)); + } + + @Test + public void testStatsGroupByDate() throws IOException { + JSONObject resonse = + executeQuery( + String.format( + "source=%s | eval t = date_add(birthdate, interval 1 day) | stats count() by" + + " span(t, 1d)", + TEST_INDEX_BANK)); + verifySchema(resonse, schema("count()", "bigint"), schema("span(t,1d)", "timestamp")); + verifyDataRows( + resonse, + rows(1, "2017-10-24 00:00:00"), + rows(1, "2017-11-21 00:00:00"), + rows(1, "2018-06-24 00:00:00"), + rows(1, "2018-06-28 00:00:00"), + rows(1, "2018-08-12 00:00:00"), + rows(1, "2018-08-20 00:00:00"), + rows(1, "2018-11-14 00:00:00")); + } + + @Test + public void testLimitAfterAggregation() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats count() by age | sort -age | head 3", TEST_INDEX_BANK)); + verifySchema(response, schema("count()", "bigint"), schema("age", "int")); + verifyDataRows(response, rows(1, 39), rows(2, 36), rows(1, 34)); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java index dcb2684d537..c9d1c0b158c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLPatternsIT.java @@ -88,19 +88,22 @@ public void testSimplePatternAggregationMode() throws IOException { result, schema("pattern_count", "bigint"), schema("patterns_field", "string"), - schema("tokens", "struct")); + schema("tokens", "struct"), + schema("sample_logs", "array")); verifyDataRows( result, rows( - 7, "@.", + 7, ImmutableMap.of( "", ImmutableList.of("amberduke", "hattiebond", "nanettebates"), "", ImmutableList.of("pyrami", "netagy", "quility"), "", - ImmutableList.of("com", "com", "com")))); + ImmutableList.of("com", "com", "com")), + ImmutableList.of( + "amberduke@pyrami.com", "hattiebond@netagy.com", "nanettebates@quility.com"))); } @Test @@ -168,7 +171,8 @@ public void testBrainAggregationMode() throws IOException { result, schema("patterns_field", "string"), schema("pattern_count", "bigint"), - schema("tokens", "struct")); + schema("tokens", "struct"), + schema("sample_logs", "array")); verifyDataRows( result, rows( @@ -178,7 +182,10 @@ public void testBrainAggregationMode() throws IOException { "", ImmutableList.of("for", "for"), "", - ImmutableList.of("-1547954353065580372", "6996194389878584395"))), + ImmutableList.of("-1547954353065580372", "6996194389878584395")), + ImmutableList.of( + "Verification succeeded for blk_-1547954353065580372", + "Verification succeeded for blk_6996194389878584395")), rows( "BLOCK* NameSystem.addStoredBlock: blockMap updated: is added to blk_" + " size ", @@ -189,7 +196,12 @@ public void testBrainAggregationMode() throws IOException { "", ImmutableList.of("67108864", "67108864"), "", - ImmutableList.of("-7017553867379051457", "-3249711809227781266"))), + ImmutableList.of("-7017553867379051457", "-3249711809227781266")), + ImmutableList.of( + "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.31.85:50010 is added to" + + " blk_-7017553867379051457 size 67108864", + "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.107.19:50010 is added" + + " to blk_-3249711809227781266 size 67108864")), rows( " NameSystem.allocateBlock:" + " /user/root/sortrand/_temporary/_task___r__/part" @@ -209,7 +221,14 @@ public void testBrainAggregationMode() throws IOException { "", ImmutableList.of("0002", "0002"), "", - ImmutableList.of("200811092030", "200811092030"))), + ImmutableList.of("200811092030", "200811092030")), + ImmutableList.of( + "BLOCK* NameSystem.allocateBlock:" + + " /user/root/sortrand/_temporary/_task_200811092030_0002_r_000296_0/part-00296." + + " blk_-6620182933895093708", + "BLOCK* NameSystem.allocateBlock:" + + " /user/root/sortrand/_temporary/_task_200811092030_0002_r_000318_0/part-00318." + + " blk_2096692261399680562")), rows( "PacketResponder failed blk_", 2, @@ -217,7 +236,10 @@ public void testBrainAggregationMode() throws IOException { "", ImmutableList.of("for", "for"), "", - ImmutableList.of("6996194389878584395", "-1547954353065580372")))); + ImmutableList.of("6996194389878584395", "-1547954353065580372")), + ImmutableList.of( + "PacketResponder failed for blk_6996194389878584395", + "PacketResponder failed for blk_-1547954353065580372"))); } @Test @@ -229,12 +251,14 @@ public void testBrainAggregationModeWithGroupByClause() throws IOException { + " mode=aggregation max_sample_count=5" + " variable_count_threshold=2 frequency_threshold_percentage=0.2", TEST_INDEX_HDFS_LOGS)); + System.out.println(result); verifySchema( result, schema("level", "string"), schema("patterns_field", "string"), schema("pattern_count", "bigint"), - schema("tokens", "struct")); + schema("tokens", "struct"), + schema("sample_logs", "array")); verifyDataRows( result, rows( @@ -242,7 +266,10 @@ public void testBrainAggregationModeWithGroupByClause() throws IOException { "Verification succeeded for blk_", 2, ImmutableMap.of( - "", ImmutableList.of("-1547954353065580372", "6996194389878584395"))), + "", ImmutableList.of("-1547954353065580372", "6996194389878584395")), + ImmutableList.of( + "Verification succeeded for blk_-1547954353065580372", + "Verification succeeded for blk_6996194389878584395")), rows( "INFO", "BLOCK* NameSystem.addStoredBlock: blockMap updated: is added to blk_" @@ -254,7 +281,12 @@ public void testBrainAggregationModeWithGroupByClause() throws IOException { "", ImmutableList.of("67108864", "67108864"), "", - ImmutableList.of("-7017553867379051457", "-3249711809227781266"))), + ImmutableList.of("-7017553867379051457", "-3249711809227781266")), + ImmutableList.of( + "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.31.85:50010 is added to" + + " blk_-7017553867379051457 size 67108864", + "BLOCK* NameSystem.addStoredBlock: blockMap updated: 10.251.107.19:50010 is added" + + " to blk_-3249711809227781266 size 67108864")), rows( "INFO", "BLOCK* NameSystem.allocateBlock:" @@ -273,13 +305,23 @@ public void testBrainAggregationModeWithGroupByClause() throws IOException { "", ImmutableList.of("000296", "000318"), "", - ImmutableList.of("0002", "0002"))), + ImmutableList.of("0002", "0002")), + ImmutableList.of( + "BLOCK* NameSystem.allocateBlock:" + + " /user/root/sortrand/_temporary/_task_200811092030_0002_r_000296_0/part-00296." + + " blk_-6620182933895093708", + "BLOCK* NameSystem.allocateBlock:" + + " /user/root/sortrand/_temporary/_task_200811092030_0002_r_000318_0/part-00318." + + " blk_2096692261399680562")), rows( "WARN", "PacketResponder failed for blk_", 2, ImmutableMap.of( - "", ImmutableList.of("6996194389878584395", "-1547954353065580372")))); + "", ImmutableList.of("6996194389878584395", "-1547954353065580372")), + ImmutableList.of( + "PacketResponder failed for blk_6996194389878584395", + "PacketResponder failed for blk_-1547954353065580372"))); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java index 2a2836a21a6..e87bb01ced5 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/standalone/CalcitePPLDateTimeBuiltinFunctionIT.java @@ -707,6 +707,27 @@ public void testComparisonBetweenDateAndTimestamp() throws IOException { verifyDataRows(actual, rows(2)); } + @Test + public void testComparisonWithIso8601DateLiteral() { + Object[][] tests = { + {"date_optional_time = '1984-04-12T09:07:42.000Z'", 2}, + {"date_time = '1984-04-12T07:07:42-02:00'", 2}, + {"date = '1984-04-12'", 2}, + {"date = '1984-04-12T09:07:42.000Z'", 2}, + {"basic_t_time = '1984-04-12T09:07:42.000Z'", 2}, + {"basic_t_time = '10:07:42.000+01:00'", 2} + }; + for (Object[] pair : tests) { + String query = (String) pair[0]; + int result = (int) pair[1]; + JSONObject actual = + executeQuery( + String.format( + "source=%s | where %s | stats COUNT() AS cnt", TEST_INDEX_DATE_FORMATS, query)); + verifyDataRows(actual, rows(result)); + } + } + @Test public void testAddSubTime() throws IOException { JSONObject actual = diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java index bcfed3ff3da..f91125458bc 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java @@ -726,6 +726,11 @@ public enum Index { "dates", getGeopointIndexMapping(), "src/test/resources/geopoints.json"), + COMPLEX_GEO( + TestsConstants.TEST_INDEX_COMPLEX_GEO, + "complex_geo", + getComplexGeoIndexMapping(), + "src/test/resources/complex_geo.json"), STATE_COUNTRY( TestsConstants.TEST_INDEX_STATE_COUNTRY, "state_country", @@ -867,6 +872,11 @@ public enum Index { "time_data", getMappingFile("time_test_data_index_mapping.json"), "src/test/resources/time_test_data.json"), + TIME_TEST_DATA_WITH_NULL( + TestsConstants.TEST_INDEX_TIME_DATE_NULL, + "time_data_with_null", + getMappingFile("time_test_data_index_mapping.json"), + "src/test/resources/time_test_data_with_null.json"), EVENTS( "events", "events", diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java index a93cc1f1b50..aa3eb6ceeba 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestUtils.java @@ -255,6 +255,11 @@ public static String getGeopointIndexMapping() { return getMappingFile(mappingFile); } + public static String getComplexGeoIndexMapping() { + String mappingFile = "complex_geo_index_mapping.json"; + return getMappingFile(mappingFile); + } + public static String getJsonTestIndexMapping() { String mappingFile = "json_test_index_mapping.json"; return getMappingFile(mappingFile); diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java index a9f166dc466..df11332fd44 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java @@ -60,6 +60,7 @@ public class TestsConstants { public static final String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested"; public static final String TEST_INDEX_NESTED_WITH_NULLS = TEST_INDEX + "_nested_with_nulls"; public static final String TEST_INDEX_GEOPOINT = TEST_INDEX + "_geopoint"; + public static final String TEST_INDEX_COMPLEX_GEO = TEST_INDEX + "_complex_geo"; public static final String TEST_INDEX_JSON_TEST = TEST_INDEX + "_json_test"; public static final String TEST_INDEX_ALIAS = TEST_INDEX + "_alias"; public static final String TEST_INDEX_FLATTENED_VALUE = TEST_INDEX + "_flattened_value"; @@ -81,6 +82,7 @@ public class TestsConstants { public static final String TEST_INDEX_HDFS_LOGS = TEST_INDEX + "_hdfs_logs"; public static final String TEST_INDEX_LOGS = TEST_INDEX + "_logs"; public static final String TEST_INDEX_OTEL_LOGS = TEST_INDEX + "_otel_logs"; + public static final String TEST_INDEX_TIME_DATE_NULL = TEST_INDEX + "_time_date_null"; public static final String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"; public static final String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS"; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java index 9aedfaeea1f..3fa8a545622 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ExplainIT.java @@ -455,7 +455,6 @@ public void testStatsBySpan() throws IOException { @Test public void testStatsBySpanNonBucketNullable() throws IOException { - // TODO isNotNull(Span) pushdown to script, can be optimized to exist() String expected = loadExpectedPlan("explain_stats_by_span_non_bucket_nullable.json"); assertJsonEqualsIgnoreId( expected, @@ -478,6 +477,14 @@ public void testStatsByTimeSpan() throws IOException { expected, explainQueryToString( String.format("source=%s | stats count() by span(birthdate,1M)", TEST_INDEX_BANK))); + + // bucket_nullable doesn't impact by-span-time + assertJsonEqualsIgnoreId( + expected, + explainQueryToString( + String.format( + "source=%s | stats bucket_nullable=false count() by span(birthdate,1M)", + TEST_INDEX_BANK))); } @Ignore("https://github.com/opensearch-project/OpenSearch/issues/3725") diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java index d99d756e762..fe747fcdc03 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoIpFunctionsIT.java @@ -1,8 +1,6 @@ /* - * * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * */ package org.opensearch.sql.ppl; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoPointFormatsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoPointFormatsIT.java index c97a0a434d9..3f2c6ccbb62 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/GeoPointFormatsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/GeoPointFormatsIT.java @@ -5,6 +5,7 @@ package org.opensearch.sql.ppl; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_COMPLEX_GEO; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_GEOPOINT; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; @@ -14,6 +15,7 @@ import java.io.IOException; import java.util.Map; import org.apache.commons.lang3.tuple.Pair; +import org.json.JSONArray; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.sql.sql.GeopointFormatsIT; @@ -23,6 +25,7 @@ public class GeoPointFormatsIT extends PPLIntegTestCase { public void init() throws Exception { super.init(); loadIndex(Index.GEOPOINTS); + loadIndex(Index.COMPLEX_GEO); } @Test @@ -51,4 +54,160 @@ public void testReadingGeoHash() throws IOException { assertEquals(40.71, point.getLeft(), GeopointFormatsIT.TOLERANCE); assertEquals(74, point.getRight(), GeopointFormatsIT.TOLERANCE); } + + // Complex geo tests - geo points within complex types (Maps) + @Test + public void testGeoPointInSimpleMap() throws IOException { + String query = + String.format( + "search source=%s | where id = '1' | fields location", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("location", null, "struct")); + + // Verify the map contains the geo point properly converted + // Using exact precision from complex_geo.json: {"lat": 47.6062, "lon": -122.3321} + verifyDataRows( + result, + rows( + Map.of( + "name", "Seattle Office", + "point", Map.of("lat", 47.6062, "lon", -122.3321), + "city", "Seattle", + "country", "USA"))); + } + + @Test + public void testGeoPointInMapWithStringFormat() throws IOException { + String query = + String.format( + "search source=%s | where id = '2' | fields location", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("location", null, "struct")); + + // Verify the map contains geo point parsed from string format + // Using exact precision from complex_geo.json: "35.6762,139.6503" + verifyDataRows( + result, + rows( + Map.of( + "name", "Tokyo Office", + "point", Map.of("lat", 35.6762, "lon", 139.6503), + "city", "Tokyo", + "country", "Japan"))); + } + + @Test + public void testNestedMapsWithGeoPoints() throws IOException { + String query = + String.format( + "search source=%s | where id = '3' | fields nested_locations", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("nested_locations", null, "struct")); + + // Verify nested structure with multiple geo points + // Using exact precision from complex_geo.json + verifyDataRows( + result, + rows( + Map.of( + "primary", + Map.of( + "office", Map.of("lat", 37.7749, "lon", -122.4194), + "warehouse", Map.of("lat", 37.4419, "lon", -122.143)), + "secondary", + Map.of( + "branch", Map.of("lat", 37.3382, "lon", -121.8863), + "store", Map.of("lat", 37.3688, "lon", -122.0363))))); + } + + @Test + public void testNestedMapsWithStringGeoPoints() throws IOException { + String query = + String.format( + "search source=%s | where id = '4' | fields nested_locations", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("nested_locations", null, "struct")); + + // Verify nested structure with geo points in string format + // Using exact precision from complex_geo.json: "40.7128,-74.0060" etc. + verifyDataRows( + result, + rows( + Map.of( + "primary", + Map.of( + "office", Map.of("lat", 40.7128, "lon", -74.006), + "warehouse", Map.of("lat", 40.758, "lon", -73.9855)), + "secondary", + Map.of( + "branch", Map.of("lat", 40.7489, "lon", -73.968), + "store", Map.of("lat", 40.7614, "lon", -73.9776))))); + } + + @Test + public void testMultipleOfficesWithGeoPoints() throws IOException { + String query = + String.format( + "search source=%s | where id = '5' | fields multiple_offices", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("multiple_offices", null, "struct")); + + // Verify multiple offices structure + verifyDataRows( + result, + rows( + Map.of( + "headquarters", + Map.of( + "location", Map.of("lat", 51.5074, "lon", -0.1278), "address", "London HQ"), + "regional", + Map.of( + "location", + Map.of("lat", 48.8566, "lon", 2.3522), + "address", + "Paris Regional")))); + } + + @Test + public void testGeoHashInMap() throws IOException { + String query = + String.format( + "search source=%s | where id = '6' | fields location", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("location", null, "struct")); + + // Verify geo point converted from geohash "u33dc0cpke7v" + // Using tolerance for geohash conversion which has precision variations + JSONArray dataRows = (JSONArray) result.get("datarows"); + JSONObject row = (JSONObject) ((JSONArray) dataRows.get(0)).get(0); + + assertEquals("Berlin Office", row.getString("name")); + assertEquals("Berlin", row.getString("city")); + assertEquals("Germany", row.getString("country")); + + JSONObject point = row.getJSONObject("point"); + double lat = point.getDouble("lat"); + double lon = point.getDouble("lon"); + + // Expected values from geohash decoding with tolerance + assertEquals(52.52003, lat, GeopointFormatsIT.TOLERANCE); + assertEquals(13.40489, lon, GeopointFormatsIT.TOLERANCE); + } + + @Test + public void testComplexGeoAllDocumentsQuery() throws IOException { + String query = String.format("search source=%s | fields id | sort id", TEST_INDEX_COMPLEX_GEO); + + JSONObject result = executeQuery(query); + verifySchema(result, schema("id", null, "string")); + + // Verify all documents are indexed and queryable + verifyDataRows(result, rows("1"), rows("2"), rows("3"), rows("4"), rows("5"), rows("6")); + } } diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java index 0051cdb39b4..88e3a0d6522 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/InformationSchemaCommandIT.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.ppl; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/LegacyAPICompatibilityIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/LegacyAPICompatibilityIT.java index 6d2399c1118..f9ef1dc178c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/LegacyAPICompatibilityIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/LegacyAPICompatibilityIT.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.ppl; import static org.opensearch.sql.plugin.rest.RestQuerySettingsAction.SETTINGS_API_ENDPOINT; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java index fa5bacf716f..190a5d75700 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PrometheusDataSourceCommandsIT.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.ppl; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java index bf21977cbeb..88e8a85ddef 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/ShowDataSourcesCommandIT.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.ppl; diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java index 783e10b09f0..a0a65c77a83 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/StatsCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_ACCOUNT; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_BANK_WITH_NULL_VALUES; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_TIME_DATE_NULL; import static org.opensearch.sql.util.MatcherUtils.rows; import static org.opensearch.sql.util.MatcherUtils.schema; import static org.opensearch.sql.util.MatcherUtils.verifyDataRows; @@ -27,6 +28,7 @@ public void init() throws Exception { loadIndex(Index.ACCOUNT); loadIndex(Index.BANK_WITH_NULL_VALUES); loadIndex(Index.BANK); + loadIndex(Index.TIME_TEST_DATA_WITH_NULL); } @Test @@ -749,4 +751,50 @@ public void testDisableLegacyPreferred() throws IOException { rows(null, 36)); }); } + + @Test + public void testStatsBySpanTimeWithNullBucket() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | stats percentile(value, 50) as p50 by span(@timestamp, 12h) as" + + " half_day", + TEST_INDEX_TIME_DATE_NULL)); + verifySchema(response, schema("p50", null, "int"), schema("half_day", null, "timestamp")); + verifyDataRows( + response, + rows(8523, "2025-07-28 00:00:00"), + rows(8094, "2025-07-28 12:00:00"), + rows(8429, "2025-07-29 00:00:00"), + rows(8216, "2025-07-29 12:00:00"), + rows(8493, "2025-07-30 00:00:00"), + rows(8426, "2025-07-30 12:00:00"), + rows(8213, "2025-07-31 00:00:00"), + rows(8490, "2025-07-31 12:00:00")); + } + + @Test + public void testStatsByCounts() throws IOException { + JSONObject response = + executeQuery( + String.format( + "source=%s | eval b_1 = balance + 1 | stats count(), count() as c1," + + " count(account_number), count(lastname) as c2, count(balance/10)," + + " count(pow(balance, 2)) as c3, count(b_1) by gender", + TEST_INDEX_ACCOUNT)); + verifySchema( + response, + schema("count()", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c1", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(account_number)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c2", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(balance/10)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("c3", null, isCalciteEnabled() ? "bigint" : "int"), + schema("count(b_1)", null, isCalciteEnabled() ? "bigint" : "int"), + schema("gender", null, "string")); + verifyDataRows( + response, + rows(493, 493, 493, 493, 493, 493, 493, "F"), + rows(507, 507, 507, 507, 507, 507, 507, "M")); + } } diff --git a/integ-test/src/test/resources/complex_geo.json b/integ-test/src/test/resources/complex_geo.json new file mode 100644 index 00000000000..59adec87b55 --- /dev/null +++ b/integ-test/src/test/resources/complex_geo.json @@ -0,0 +1,12 @@ +{"index": {"_id": "1"}} +{"id": "1", "location": {"name": "Seattle Office", "point": {"lat": 47.6062, "lon": -122.3321}, "city": "Seattle", "country": "USA"}} +{"index": {"_id": "2"}} +{"id": "2", "location": {"name": "Tokyo Office", "point": "35.6762,139.6503", "city": "Tokyo", "country": "Japan"}} +{"index": {"_id": "3"}} +{"id": "3", "nested_locations": {"primary": {"office": {"lat": 37.7749, "lon": -122.4194}, "warehouse": {"lat": 37.4419, "lon": -122.1430}}, "secondary": {"branch": {"lat": 37.3382, "lon": -121.8863}, "store": {"lat": 37.3688, "lon": -122.0363}}}} +{"index": {"_id": "4"}} +{"id": "4", "nested_locations": {"primary": {"office": "40.7128,-74.0060", "warehouse": "40.7580,-73.9855"}, "secondary": {"branch": "40.7489,-73.9680", "store": "40.7614,-73.9776"}}} +{"index": {"_id": "5"}} +{"id": "5", "multiple_offices": {"headquarters": {"location": {"lat": 51.5074, "lon": -0.1278}, "address": "London HQ"}, "regional": {"location": {"lat": 48.8566, "lon": 2.3522}, "address": "Paris Regional"}}} +{"index": {"_id": "6"}} +{"id": "6", "location": {"name": "Berlin Office", "point": "u33dc0cpke7v", "city": "Berlin", "country": "Germany"}} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml new file mode 100644 index 00000000000..ddeefad911b --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by1.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], c1=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(gender=[$4]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], count()=[$t0], count()0=[$t0], gender=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000, PROJECT->[count(), gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml new file mode 100644 index 00000000000..000995f3c2a --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by2.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$1], c2=[$1], gender=[$0]) + LogicalAggregate(group=[{0}], c1=[COUNT($1)]) + LogicalProject(gender=[$4], balance=[$3]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..1=[{inputs}], c1=[$t0], c10=[$t0], gender=[$t1]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},c1=COUNT($1)), LIMIT->10000, PROJECT->[c1, gender]], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"c1":{"value_count":{"field":"balance"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml new file mode 100644 index 00000000000..a4dfdb25064 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by3.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(account_number)=[$1], c2=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(account_number)=[COUNT($1)], c2=[COUNT($2)]) + LogicalProject(gender=[$4], account_number=[$0], account_number_alias=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(account_number)=COUNT($1),c2=COUNT($2)), PROJECT->[count(account_number), c2, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(account_number)":{"value_count":{"field":"account_number"}},"c2":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml new file mode 100644 index 00000000000..e56eb5ad662 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by4.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], count(account_number)=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()], count(account_number)=[COUNT($1)]) + LogicalProject(gender=[$4], account_number=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT(),count(account_number)=COUNT($1)), PROJECT->[count(), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count()":{"value_count":{"field":"_index"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml new file mode 100644 index 00000000000..dc7bf3629f2 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by5.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(balance)=[$1], count(account_number)=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(balance)=[COUNT($1)], count(account_number)=[COUNT($2)]) + LogicalProject(gender=[$4], balance=[$3], account_number=[$0]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(balance)=COUNT($1),count(account_number)=COUNT($2)), PROJECT->[count(balance), count(account_number), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(balance)":{"value_count":{"field":"balance"}},"count(account_number)":{"value_count":{"field":"account_number"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml new file mode 100644 index 00000000000..a385eaf690d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_counts_by6.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count(b_1)=[$1], c3=[$2], gender=[$0]) + LogicalAggregate(group=[{0}], count(b_1)=[COUNT($1)], c3=[COUNT($2)]) + LogicalProject(gender=[$4], b_1=[+($3, 1)], $f3=[POWER($3, 2)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count(b_1)=COUNT($1),c3=COUNT($2)), PROJECT->[count(b_1), c3, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"composite_buckets":{"composite":{"size":1000,"sources":[{"gender":{"terms":{"field":"gender.keyword","missing_bucket":true,"missing_order":"first","order":"asc"}}}]},"aggregations":{"count(b_1)":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}},"c3":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBEXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJQT1dFUiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiAzLAogICAgICAibmFtZSI6ICIkMyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAt0AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AEXhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAceHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAeAAAAAHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgASfnEAfgALdAAGU1RSSU5HfnEAfgAYdAAHS2V5d29yZHEAfgAdeHQAB2FkZHJlc3NzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAAAeHQAB2JhbGFuY2VxAH4ADXQABmdlbmRlcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABGNpdHlzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AAhlbXBsb3llcnNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQABXN0YXRlc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAADYWdlcQB+AA10AAVlbWFpbHNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGxhc3RuYW1lc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4eAB4\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json new file mode 100644 index 00000000000..4d0e7f7c31d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_timestamp_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->3, PROJECT->[count(), t], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":3,\"sources\":[{\"t\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AWd7CiAgIm9wIjogewogICAgIm5hbWUiOiAiVU5JWF9USU1FU1RBTVAiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMywKICAgICAgIm5hbWUiOiAiJDMiCiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJET1VCTEUiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json new file mode 100644 index 00000000000..ad73629e888 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_script_udt_arg_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(t,1d)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')])\n LogicalFilter(condition=[IS NOT NULL($19)])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(DATE_ADD($3, 1:INTERVAL DAY)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(t,1d)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AyV7CiAgIm9wIjogewogICAgIm5hbWUiOiAiSVMgTk9UIE5VTEwiLAogICAgImtpbmQiOiAiSVNfTk9UX05VTEwiLAogICAgInN5bnRheCI6ICJQT1NURklYIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMywKICAgICAgICAgICJuYW1lIjogIiQzIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgAKdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgATeHB+cQB+AAp0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4AC3QABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB54cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACAAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABR+cQB+AAp0AAlUSU1FU1RBTVB+cQB+ABp0AAREYXRlcQB+AB9zcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAUcQB+AA9+cQB+ABp0AAdLZXl3b3JkcQB+AB94dAAEY2l0eXEAfgAPdAAIbGFzdG5hbWVxAH4AD3QAB2JhbGFuY2VxAH4ADHQACGVtcGxveWVyc3EAfgAScQB+ABhxAH4AG3EAfgAfcQB+ACN0AAVzdGF0ZXNxAH4AEnEAfgAYcQB+ABtxAH4AH3NxAH4AAAAAAAN3BAAAAAJxAH4AMHEAfgAxeHQAA2FnZX5xAH4ACnQAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AEnEAfgAYcQB+ABtxAH4AH3EAfgAjdAAEbWFsZX5xAH4ACnQAB0JPT0xFQU54eA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(t,1d)\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0BQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiU1BBTiIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAib3AiOiB7CiAgICAgICAgIm5hbWUiOiAiREFURV9BREQiLAogICAgICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICAgICAic3ludGF4IjogIkZVTkNUSU9OIgogICAgICB9LAogICAgICAib3BlcmFuZHMiOiBbCiAgICAgICAgewogICAgICAgICAgImlucHV0IjogMywKICAgICAgICAgICJuYW1lIjogIiQzIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxLAogICAgICAgICAgInR5cGUiOiB7CiAgICAgICAgICAgICJ0eXBlIjogIklOVEVSVkFMX0RBWSIsCiAgICAgICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICAgICAicHJlY2lzaW9uIjogMTAsCiAgICAgICAgICAgICJzY2FsZSI6IDYKICAgICAgICAgIH0KICAgICAgICB9CiAgICAgIF0sCiAgICAgICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInVkdCI6ICJFWFBSX1RJTUVTVEFNUCIsCiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgICAicHJlY2lzaW9uIjogLTEKICAgICAgfSwKICAgICAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICAgICAiZHluYW1pYyI6IGZhbHNlCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogImQiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMQogICAgICB9CiAgICB9CiAgXSwKICAiY2xhc3MiOiAib3JnLm9wZW5zZWFyY2guc3FsLmV4cHJlc3Npb24uZnVuY3Rpb24uVXNlckRlZmluZWRGdW5jdGlvbkJ1aWxkZXIkMSIsCiAgInR5cGUiOiB7CiAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICJwcmVjaXNpb24iOiAtMQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"value_type\":\"long\",\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json index fa5f3c8f879..7fa93a13cf4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics1.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$1], state=[$0])\n LogicalFilter(condition=[IS NOT NULL($0)])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"size\":1000,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":{\"_key\":\"asc\"}},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json index e6c9e3069fc..fb2b1104bd9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_sort_on_metrics2.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$2], gender=[$0], state=[$1])\n LogicalFilter(condition=[AND(IS NOT NULL($0), IS NOT NULL($1))])\n LogicalAggregate(group=[{0, 1}], count()=[COUNT()])\n LogicalProject(gender=[$4], state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..2=[{inputs}], count()=[$t2], gender=[$t0], state=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},{\"exists\":{\"field\":\"state\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(count()=[$2], gender=[$0], state=[$1])\n LogicalAggregate(group=[{0, 1}], count()=[COUNT()])\n LogicalProject(gender=[$4], state=[$7])\n LogicalFilter(condition=[AND(IS NOT NULL($4), IS NOT NULL($7))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->AND(IS NOT NULL($4), IS NOT NULL($7)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},count()=COUNT()), PROJECT->[count(), gender, state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"exists\":{\"field\":\"gender\",\"boost\":1.0}},{\"exists\":{\"field\":\"state\",\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json index 48fb2dcdc04..989f996b26e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_script.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum=[$2], len=[$0], gender=[$1])\n LogicalAggregate(group=[{0, 1}], sum=[SUM($2)])\n LogicalProject(len=[CHAR_LENGTH($4)], gender=[$4], $f3=[+($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], sum=[$t2], len=[$t0], gender=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"len\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQApnsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAia2luZCI6ICJDSEFSX0xFTkdUSCIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA0LAogICAgICAibmFtZSI6ICIkNCIKICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBA3sKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},sum=SUM($2)), PROJECT->[sum, len, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"len\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNCwKICAgICAgIm5hbWUiOiAiJDQiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA3LAogICAgICAibmFtZSI6ICIkNyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json index 22b65831e9d..e4b32a61e8a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_agg_with_sum_enhancement.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(sum(balance)=[$1], sum(balance + 100)=[$2], sum(balance - 100)=[$3], sum(balance * 100)=[$4], sum(balance / 100)=[$5], gender=[$0])\n LogicalAggregate(group=[{0}], sum(balance)=[SUM($1)], sum(balance + 100)=[SUM($2)], sum(balance - 100)=[SUM($3)], sum(balance * 100)=[SUM($4)], sum(balance / 100)=[SUM($5)])\n LogicalProject(gender=[$4], balance=[$7], $f6=[+($7, 100)], $f7=[-($7, 100)], $f8=[*($7, 100)], $f9=[DIVIDE($7, 100)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..3=[{inputs}], expr#4=[100], expr#5=[*($t2, $t4)], expr#6=[+($t1, $t5)], expr#7=[-($t1, $t5)], expr#8=[*($t1, $t4)], sum(balance)=[$t1], sum(balance + 100)=[$t6], sum(balance - 100)=[$t7], sum(balance * 100)=[$t8], sum(balance / 100)=[$t3], gender=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)_COUNT=COUNT($1),sum(balance / 100)=SUM($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)_COUNT\":{\"value_count\":{\"field\":\"balance\"}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBz3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJESVZJREUiLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICJ0eXBlIjogewogICAgInR5cGUiOiAiQklHSU5UIiwKICAgICJudWxsYWJsZSI6IHRydWUKICB9LAogICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAiZHluYW1pYyI6IGZhbHNlCn10AApmaWVsZFR5cGVzc3IAF2phdmEudXRpbC5MaW5rZWRIYXNoTWFwNMBOXBBswPsCAAFaAAthY2Nlc3NPcmRlcnhyABFqYXZhLnV0aWwuSGFzaE1hcAUH2sHDFmDRAwACRgAKbG9hZEZhY3RvckkACXRocmVzaG9sZHhwP0AAAAAAABh3CAAAACAAAAANdAAOYWNjb3VudF9udW1iZXJ+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AARMT05HdAAJZmlyc3RuYW1lfnEAfgALdAAGU1RSSU5HdAAHYWRkcmVzc3NyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAUeHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+AB94cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+ACEAAAAAc3EAfgAAAAAAA3cEAAAAAHh0AAliaXJ0aGRhdGVzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRlVHlwZZ4tUq4QfcqvAgABTAAHZm9ybWF0c3QAEExqYXZhL3V0aWwvTGlzdDt4cQB+ABV+cQB+AAt0AAlUSU1FU1RBTVB+cQB+ABt0AAREYXRlcQB+ACBzcQB+AAAAAAABdwQAAAAAeHQABmdlbmRlcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJ0AAdrZXl3b3Jkc3EAfgAVcQB+ABB+cQB+ABt0AAdLZXl3b3JkcQB+ACB4dAAEY2l0eXEAfgAQdAAIbGFzdG5hbWVxAH4AEHQAB2JhbGFuY2VxAH4ADXQACGVtcGxveWVyc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AAVzdGF0ZXNxAH4AE3EAfgAZcQB+ABxxAH4AIHNxAH4AAAAAAAN3BAAAAAJxAH4AMXEAfgAyeHQAA2FnZX5xAH4AC3QAB0lOVEVHRVJ0AAVlbWFpbHNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAEbWFsZX5xAH4AC3QAB0JPT0xFQU54AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},sum(balance)=SUM($1),sum(balance + 100)=SUM($2),sum(balance - 100)=SUM($3),sum(balance * 100)=SUM($4),sum(balance / 100)=SUM($5)), PROJECT->[sum(balance), sum(balance + 100), sum(balance - 100), sum(balance * 100), sum(balance / 100), gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"sum(balance)\":{\"sum\":{\"field\":\"balance\"}},\"sum(balance + 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKyIsCiAgICAia2luZCI6ICJQTFVTIiwKICAgICJzeW50YXgiOiAiQklOQVJZIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA3LAogICAgICAibmFtZSI6ICIkNyIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogMTAwLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance - 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AUB7CiAgIm9wIjogewogICAgIm5hbWUiOiAiLSIsCiAgICAia2luZCI6ICJNSU5VUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0sCiAgInR5cGUiOiB7CiAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance * 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0AQR7CiAgIm9wIjogewogICAgIm5hbWUiOiAiKiIsCiAgICAia2luZCI6ICJUSU1FUyIsCiAgICAic3ludGF4IjogIkJJTkFSWSIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogNywKICAgICAgIm5hbWUiOiAiJDciCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDEwMCwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiSU5URUdFUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgfQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgAXamF2YS51dGlsLkxpbmtlZEhhc2hNYXA0wE5cEGzA+wIAAVoAC2FjY2Vzc09yZGVyeHIAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAAGHcIAAAAIAAAAA10AA5hY2NvdW50X251bWJlcn5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQABExPTkd0AAlmaXJzdG5hbWV+cQB+AAt0AAZTVFJJTkd0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+ABR4cH5xAH4AC3QAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgAMdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AH3hwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AIQAAAABzcQB+AAAAAAADdwQAAAAAeHQACWJpcnRoZGF0ZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGVUeXBlni1SrhB9yq8CAAFMAAdmb3JtYXRzdAAQTGphdmEvdXRpbC9MaXN0O3hxAH4AFX5xAH4AC3QACVRJTUVTVEFNUH5xAH4AG3QABERhdGVxAH4AIHNxAH4AAAAAAAF3BAAAAAB4dAAGZ2VuZGVyc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABVxAH4AEH5xAH4AG3QAB0tleXdvcmRxAH4AIHh0AARjaXR5cQB+ABB0AAhsYXN0bmFtZXEAfgAQdAAHYmFsYW5jZXEAfgANdAAIZW1wbG95ZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABXN0YXRlc3EAfgATcQB+ABlxAH4AHHEAfgAgc3EAfgAAAAAAA3cEAAAAAnEAfgAxcQB+ADJ4dAADYWdlfnEAfgALdAAHSU5URUdFUnQABWVtYWlsc3EAfgATcQB+ABlxAH4AHHEAfgAgcQB+ACR0AARtYWxlfnEAfgALdAAHQk9PTEVBTngAeA==\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sum(balance / 100)\":{\"sum\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHrXsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidWR0IjogIkVYUFJfVElNRVNUQU1QIiwKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiYmlydGhkYXRlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYmFsYW5jZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiYWdlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1haWwiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCT09MRUFOIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAibWFsZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pZCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9pbmRleCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJSRUFMIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX21heHNjb3JlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiQklHSU5UIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgIm5hbWUiOiAiX3NvcnQiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJfcm91dGluZyIKICAgIH0KICBdLAogICJudWxsYWJsZSI6IHRydWUKfXQABGV4cHJ0Ac97CiAgIm9wIjogewogICAgIm5hbWUiOiAiRElWSURFIiwKICAgICJraW5kIjogIk9USEVSX0ZVTkNUSU9OIiwKICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDcsCiAgICAgICJuYW1lIjogIiQ3IgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxMDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJJR0lOVCIsCiAgICAibnVsbGFibGUiOiB0cnVlCiAgfSwKICAiZGV0ZXJtaW5pc3RpYyI6IHRydWUsCiAgImR5bmFtaWMiOiBmYWxzZQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4AC3QABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AFHhwfnEAfgALdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAx0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAfeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAhAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAVfnEAfgALdAAJVElNRVNUQU1QfnEAfgAbdAAERGF0ZXEAfgAgc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFXEAfgAQfnEAfgAbdAAHS2V5d29yZHEAfgAgeHQABGNpdHlxAH4AEHQACGxhc3RuYW1lcQB+ABB0AAdiYWxhbmNlcQB+AA10AAhlbXBsb3llcnNxAH4AE3EAfgAZcQB+ABxxAH4AIHEAfgAkdAAFc3RhdGVzcQB+ABNxAH4AGXEAfgAccQB+ACBzcQB+AAAAAAADdwQAAAACcQB+ADFxAH4AMnh0AANhZ2V+cQB+AAt0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABNxAH4AGXEAfgAccQB+ACBxAH4AJHQABG1hbGV+cQB+AAt0AAdCT09MRUFOeAB4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json index 01d4e557bcb..7a857107a13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_append_command.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalUnion(all=[true])\n LogicalProject(cnt=[$1], gender=[$0])\n LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n LogicalProject(gender=[$4], balance=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n LogicalProject(cnt=[$0], gender=[null:VARCHAR])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n EnumerableCalc(expr#0..1=[{inputs}], cnt=[$t1], gender=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableUnion(all=[true])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},cnt=COUNT($1)), PROJECT->[cnt, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[null:VARCHAR], proj#0..1=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"track_total_hits\":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json deleted file mode 100644 index 63b4c6d8ab2..00000000000 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], cnt=[COUNT()])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" - } -} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml new file mode 100644 index 00000000000..e0d0a3d0070 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push1.yaml @@ -0,0 +1,7 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml new file mode 100644 index 00000000000..26ab37adc53 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push10.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(firstname)=[COUNT($0)], count(name)=[COUNT($1)]) + LogicalProject(firstname=[$1], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(name)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml new file mode 100644 index 00000000000..6a2d697a071 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push2.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject(lastname=[$10]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml new file mode 100644 index 00000000000..71be9d237c1 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push3.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject(name=[$17]) + LogicalFilter(condition=[IS NOT NULL($10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml new file mode 100644 index 00000000000..b191f90b170 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push4.yaml @@ -0,0 +1,9 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$0], c2=[$0]) + LogicalAggregate(group=[{}], c1=[COUNT()]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml new file mode 100644 index 00000000000..689138f358d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push5.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(c1=[$0], c2=[$0]) + LogicalAggregate(group=[{}], c1=[COUNT($0)]) + LogicalProject(lastname=[$10]) + LogicalFilter(condition=[IS NOT NULL($10)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0=[{inputs}], c1=[$t0], c2=[$t0]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml new file mode 100644 index 00000000000..82260e27bbc --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push6.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(lastname)=[COUNT($0)], count(name)=[COUNT($1)]) + LogicalProject(lastname=[$10], name=[$17]) + LogicalFilter(condition=[IS NOT NULL($10)]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], _id=[$11], _index=[$12], _score=[$13], _maxscore=[$14], _sort=[$15], _routing=[$16], name=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($10), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(lastname)=COUNT($0),count(name)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","query":{"exists":{"field":"lastname","boost":1.0}},"sort":[],"track_total_hits":2147483647}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml new file mode 100644 index 00000000000..041d862fbc4 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push7.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], cnt=[COUNT($0)]) + LogicalProject($f1=[+($3, 1)]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},cnt=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"cnt":{"value_count":{"script":{"source":"{\"langType\":\"calcite\",\"script\":\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBAXsKICAib3AiOiB7CiAgICAibmFtZSI6ICIrIiwKICAgICJraW5kIjogIlBMVVMiLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJpbnB1dCI6IDMsCiAgICAgICJuYW1lIjogIiQzIgogICAgfSwKICAgIHsKICAgICAgImxpdGVyYWwiOiAxLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZQogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\"}","lang":"opensearch_compounded_script","params":{"utcTimestamp": 0}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml new file mode 100644 index 00000000000..beaac7cbe69 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push8.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], c1=[COUNT()], c2=[COUNT($0)]) + LogicalProject(lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},c1=COUNT(),c2=COUNT($0)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"c1":{"value_count":{"field":"_index"}},"c2":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml new file mode 100644 index 00000000000..b5548293b7f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_count_agg_push9.yaml @@ -0,0 +1,8 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalAggregate(group=[{}], count(firstname)=[COUNT($0)], count(lastname)=[COUNT($1)]) + LogicalProject(firstname=[$1], lastname=[$10]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},count(firstname)=COUNT($0),count(lastname)=COUNT($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"count(firstname)":{"value_count":{"field":"firstname.keyword"}},"count(lastname)":{"value_count":{"field":"lastname.keyword"}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json index e68d171db74..6afa7be030c 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], server=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(server=[$1], message=[$3], @timestamp=[$2])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], server=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"server\":{\"terms\":{\"field\":\"server\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, server], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"server\":{\"terms\":{\"field\":\"server\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"@timestamp\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json index 65c6503dbbd..98d5277e419 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_earliest_latest_custom_time.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(earliest_message=[$1], latest_message=[$2], level=[$0])\n LogicalAggregate(group=[{0}], earliest_message=[ARG_MIN($1, $2)], latest_message=[ARG_MAX($1, $2)])\n LogicalProject(level=[$4], message=[$3], created_at=[$0])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], earliest_message=[$t1], latest_message=[$t2], level=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"level\":{\"terms\":{\"field\":\"level\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_logs]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},earliest_message=ARG_MIN($1, $2),latest_message=ARG_MAX($1, $2)), PROJECT->[earliest_message, latest_message, level], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"level\":{\"terms\":{\"field\":\"level\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"earliest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"asc\"}}]}},\"latest_message\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"message\"],\"excludes\":[]},\"sort\":[{\"created_at\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json index 2bc95c47061..aa12f728621 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t0], city=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"city\",\"state\",\"age\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($8, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, city, avg_age], PROJECT->[avg_age, state, city], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json new file mode 100644 index 00000000000..8e90e2986ab --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_filter_script_ip_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]], PushDownContext=[[PROJECT->[host], SCRIPT->CIDRMATCH($0, '0.0.0.0/24':VARCHAR), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAknsKICAiZmllbGRzIjogWwogICAgewogICAgICAidWR0IjogIkVYUFJfSVAiLAogICAgICAidHlwZSI6ICJPVEhFUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogImhvc3QiCiAgICB9CiAgXSwKICAibnVsbGFibGUiOiBmYWxzZQp9dAAEZXhwcnQB9XsKICAib3AiOiB7CiAgICAibmFtZSI6ICJDSURSTUFUQ0giLAogICAgImtpbmQiOiAiT1RIRVJfRlVOQ1RJT04iLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICIwLjAuMC4wLzI0IiwKICAgICAgInR5cGUiOiB7CiAgICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICAgIm51bGxhYmxlIjogZmFsc2UsCiAgICAgICAgInByZWNpc2lvbiI6IC0xCiAgICAgIH0KICAgIH0KICBdLAogICJjbGFzcyI6ICJvcmcub3BlbnNlYXJjaC5zcWwuZXhwcmVzc2lvbi5mdW5jdGlvbi5Vc2VyRGVmaW5lZEZ1bmN0aW9uQnVpbGRlciQxIiwKICAidHlwZSI6IHsKICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgIm51bGxhYmxlIjogdHJ1ZQogIH0sCiAgImRldGVybWluaXN0aWMiOiB0cnVlLAogICJkeW5hbWljIjogZmFsc2UKfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAAXQABGhvc3R+cgApb3JnLm9wZW5zZWFyY2guc3FsLmRhdGEudHlwZS5FeHByQ29yZVR5cGUAAAAAAAAAABIAAHhyAA5qYXZhLmxhbmcuRW51bQAAAAAAAAAAEgAAeHB0AAJJUHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":1758184003318649000}},\"boost\":1.0}},\"_source\":{\"includes\":[\"host\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json index ca95fbc164c..d10259db2ef 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_first_last.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(first_name=[$1], last_name=[$2], gender=[$0])\n LogicalAggregate(group=[{0}], first_name=[FIRST($1)], last_name=[LAST($1)])\n LogicalProject(gender=[$4], firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], first_name=[$t1], last_name=[$t2], gender=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"first_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}},\"last_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},first_name=FIRST($1),last_name=LAST($1)), PROJECT->[first_name, last_name, gender], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"gender\":{\"terms\":{\"field\":\"gender.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"first_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]}}},\"last_name\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"desc\"}}]}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json index 2c5b94c3110..5fbc3abecf1 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#27:LogicalAggregate.NONE.[](input=RelSubset#26,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json index 7cc6b879da3..391568b1234 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown2.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#101:LogicalAggregate.NONE.[](input=RelSubset#100,group={0},count()=COUNT()), LIMIT->100, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":100,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, PROJECT->[count(), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":100,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json index 3432ea267a7..986459fd2f4 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown3.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json index 8ffece29593..635a33b5d6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown4.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json index 54d21661a92..e9756e74db6 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown5.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n EnumerableLimit(fetch=[100])\n EnumerableSort(sort0=[$1], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableLimit(offset=[10], fetch=[10])\n EnumerableLimit(fetch=[100])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), state]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json index a44c4af7550..2db9c47af12 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable1.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalFilter(condition=[IS NOT NULL($0)])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"size\":20,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":{\"_key\":\"asc\"}},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json index 4898344e498..40adc108671 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_limit_agg_pushdown_bucket_nullable2.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalFilter(condition=[IS NOT NULL($0)])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"size\":20,\"min_doc_count\":1,\"shard_min_doc_count\":0,\"show_term_doc_count_error\":false,\"order\":{\"_key\":\"asc\"}},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(offset=[10], fetch=[10])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[100])\n LogicalProject(count()=[$1], state=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(state=[$7])\n LogicalFilter(condition=[IS NOT NULL($7)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], state=[$t0])\n EnumerableLimit(offset=[10], fetch=[10])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->IS NOT NULL($7), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), SORT->[0 ASC FIRST], LIMIT->100, LIMIT->[10 from 10]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"state\",\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":20,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":false,\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json new file mode 100644 index 00000000000..961cdc4687f --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_max_string_field.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},max(firstname)=MAX($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"max(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"desc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json new file mode 100644 index 00000000000..41a14f5e84e --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_min_string_field.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={},min(firstname)=MIN($0))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"min(firstname)\":{\"top_hits\":{\"from\":0,\"size\":1,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"firstname\"],\"excludes\":[]},\"sort\":[{\"firstname.keyword\":{\"order\":\"asc\"}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json index 668803ec7c1..973efe47fa9 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_output.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age2=[$2])\n LogicalFilter(condition=[<=($3, 1)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[$2], _row_number_dedup_=[ROW_NUMBER() OVER (PARTITION BY $2 ORDER BY $2)])\n LogicalFilter(condition=[IS NOT NULL($2)])\n LogicalProject(avg_age=[$0], state=[$1], age2=[+($0, 2)])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n LogicalFilter(condition=[>($8, 30)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=[<=($t2, $t3)], age2=[$t1], $condition=[$t4])\n EnumerableWindow(window#0=[window(partition {1} order by [1] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[2], expr#4=[+($t2, $t3)], expr#5=[IS NOT NULL($t2)], state=[$t0], age2=[$t4], $condition=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[city, state, age], FILTER->>($2, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), SORT->[0 ASC FIRST]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"_source\":{\"includes\":[\"city\",\"state\",\"age\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableCalc(expr#0..1=[{inputs}], age2=[$t0])\n EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=[<=($t1, $t2)], proj#0..1=[{exprs}], $condition=[$t3])\n EnumerableWindow(window#0=[window(partition {0} order by [0] rows between UNBOUNDED PRECEDING and CURRENT ROW aggs [ROW_NUMBER()])])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[2], expr#2=[+($t0, $t1)], expr#3=[IS NOT NULL($t0)], age2=[$t2], $condition=[$t3])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[FILTER->>($8, 30), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[state, city, avg_age], SORT->[0 ASC FIRST], PROJECT->[avg_age, state], PROJECT->[avg_age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"range\":{\"age\":{\"from\":30,\"to\":null,\"include_lower\":false,\"include_upper\":true,\"boost\":1.0}}},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_brain_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_brain_agg_push.json index 36b6b90b7d1..c3fc80d1eb0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_brain_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_brain_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM($1, 'pattern'))], pattern_count=[SAFE_CAST(ITEM($1, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($1, 'tokens'))])\n LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n LogicalAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n LogicalProject(email=[$9], $f17=[10], $f18=[100000])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n Uncollect\n LogicalProject(patterns_field=[$cor0.patterns_field])\n LogicalValues(tuples=[[{ 0 }]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=['pattern'], expr#3=[ITEM($t1, $t2)], expr#4=[SAFE_CAST($t3)], expr#5=['pattern_count'], expr#6=[ITEM($t1, $t5)], expr#7=[SAFE_CAST($t6)], expr#8=['tokens'], expr#9=[ITEM($t1, $t8)], expr#10=[SAFE_CAST($t9)], patterns_field=[$t4], pattern_count=[$t7], tokens=[$t10])\n EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n EnumerableAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[10], expr#2=[100000], proj#0..2=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[email]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableUncollect\n EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.patterns_field], patterns_field=[$t2])\n EnumerableValues(tuples=[[{ 0 }]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM($1, 'pattern'))], pattern_count=[SAFE_CAST(ITEM($1, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($1, 'tokens'))], sample_logs=[SAFE_CAST(ITEM($1, 'sample_logs'))])\n LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n LogicalAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n LogicalProject(email=[$9], $f17=[10], $f18=[100000])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n Uncollect\n LogicalProject(patterns_field=[$cor0.patterns_field])\n LogicalValues(tuples=[[{ 0 }]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=['pattern'], expr#3=[ITEM($t1, $t2)], expr#4=[SAFE_CAST($t3)], expr#5=['pattern_count'], expr#6=[ITEM($t1, $t5)], expr#7=[SAFE_CAST($t6)], expr#8=['tokens'], expr#9=[ITEM($t1, $t8)], expr#10=[SAFE_CAST($t9)], expr#11=['sample_logs'], expr#12=[ITEM($t1, $t11)], expr#13=[SAFE_CAST($t12)], patterns_field=[$t4], pattern_count=[$t7], tokens=[$t10], sample_logs=[$t13])\n EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n EnumerableAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n EnumerableCalc(expr#0=[{inputs}], expr#1=[10], expr#2=[100000], proj#0..2=[{exprs}])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[email]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableUncollect\n EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.patterns_field], patterns_field=[$t2])\n EnumerableValues(tuples=[[{ 0 }]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json index c21c73eaf4c..2716bfe9e83 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_patterns_simple_pattern_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(pattern_count=[$1], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], pattern_count=[$t1], patterns_field=[$t6], tokens=[$t9])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t1)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], $f0=[$t6], pattern_count=[$t2], $f2=[$t9], sample_logs=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1},pattern_count=COUNT($1),sample_logs=TAKE($0, $2)), LIMIT->10000, PROJECT->[patterns_field, sample_logs, pattern_count]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"patterns_field\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"pattern_count\":{\"value_count\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQGy3sKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZ2VuZGVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiY2l0eSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImVtcGxveWVyIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAic3RhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImxhc3RuYW1lIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQBx3sKICAib3AiOiB7CiAgICAibmFtZSI6ICJSRUdFWFBfUkVQTEFDRSIsCiAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAic3ludGF4IjogIkZVTkNUSU9OIgogIH0sCiAgIm9wZXJhbmRzIjogWwogICAgewogICAgICAiaW5wdXQiOiA5LAogICAgICAibmFtZSI6ICIkOSIKICAgIH0sCiAgICB7CiAgICAgICJsaXRlcmFsIjogIlthLXpBLVowLTldKyIsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlLAogICAgICAgICJwcmVjaXNpb24iOiAtMQogICAgICB9CiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6ICI8Kj4iLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJDSEFSIiwKICAgICAgICAibnVsbGFibGUiOiBmYWxzZSwKICAgICAgICAicHJlY2lzaW9uIjogMwogICAgICB9CiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}}}},\"sample_logs\":{\"top_hits\":{\"from\":0,\"size\":10,\"version\":false,\"seq_no_primary_term\":false,\"explain\":false,\"_source\":{\"includes\":[\"email\"],\"excludes\":[]}}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json index 3c5b3b632aa..2f13054be6e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_script_push_on_text.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], address_length=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(address_length=[CHAR_LENGTH($2)])\n LogicalFilter(condition=[>(CHAR_LENGTH($2), 0)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], address_length=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[address], SCRIPT->>(CHAR_LENGTH($0), 0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), address_length], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AbN7CiAgIm9wIjogewogICAgIm5hbWUiOiAiPiIsCiAgICAia2luZCI6ICJHUkVBVEVSX1RIQU4iLAogICAgInN5bnRheCI6ICJCSU5BUlkiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJDSEFSX0xFTkdUSCIsCiAgICAgICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgICAgICJzeW50YXgiOiAiRlVOQ1RJT04iCiAgICAgIH0sCiAgICAgICJvcGVyYW5kcyI6IFsKICAgICAgICB7CiAgICAgICAgICAiaW5wdXQiOiAwLAogICAgICAgICAgIm5hbWUiOiAiJDAiCiAgICAgICAgfQogICAgICBdCiAgICB9LAogICAgewogICAgICAibGl0ZXJhbCI6IDAsCiAgICAgICJ0eXBlIjogewogICAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAgICJudWxsYWJsZSI6IGZhbHNlCiAgICAgIH0KICAgIH0KICBdCn10AApmaWVsZFR5cGVzc3IAEWphdmEudXRpbC5IYXNoTWFwBQfawcMWYNEDAAJGAApsb2FkRmFjdG9ySQAJdGhyZXNob2xkeHA/QAAAAAAADHcIAAAAEAAAAAF0AAdhZGRyZXNzc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoVGV4dFR5cGWtg6OTBOMxRAIAAUwABmZpZWxkc3QAD0xqYXZhL3V0aWwvTWFwO3hyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlwmO8ygL6BTUCAANMAAxleHByQ29yZVR5cGV0ACtMb3JnL29wZW5zZWFyY2gvc3FsL2RhdGEvdHlwZS9FeHByQ29yZVR5cGU7TAALbWFwcGluZ1R5cGV0AEhMb3JnL29wZW5zZWFyY2gvc3FsL29wZW5zZWFyY2gvZGF0YS90eXBlL09wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZTtMAApwcm9wZXJ0aWVzcQB+AAt4cH5yAClvcmcub3BlbnNlYXJjaC5zcWwuZGF0YS50eXBlLkV4cHJDb3JlVHlwZQAAAAAAAAAAEgAAeHIADmphdmEubGFuZy5FbnVtAAAAAAAAAAASAAB4cHQAB1VOS05PV05+cgBGb3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZSRNYXBwaW5nVHlwZQAAAAAAAAAAEgAAeHEAfgARdAAEVGV4dHNyADxzaGFkZWQuY29tLmdvb2dsZS5jb21tb24uY29sbGVjdC5JbW11dGFibGVNYXAkU2VyaWFsaXplZEZvcm0AAAAAAAAAAAIAAkwABGtleXN0ABJMamF2YS9sYW5nL09iamVjdDtMAAZ2YWx1ZXNxAH4AGHhwdXIAE1tMamF2YS5sYW5nLk9iamVjdDuQzlifEHMpbAIAAHhwAAAAAHVxAH4AGgAAAABzcQB+AAAAAAADdwQAAAAAeHh4\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"_source\":{\"includes\":[\"address\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"address_length\":{\"terms\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQAlnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJhZGRyZXNzIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogZmFsc2UKfXQABGV4cHJ0AKZ7CiAgIm9wIjogewogICAgIm5hbWUiOiAiQ0hBUl9MRU5HVEgiLAogICAgImtpbmQiOiAiQ0hBUl9MRU5HVEgiLAogICAgInN5bnRheCI6ICJGVU5DVElPTiIKICB9LAogICJvcGVyYW5kcyI6IFsKICAgIHsKICAgICAgImlucHV0IjogMCwKICAgICAgIm5hbWUiOiAiJDAiCiAgICB9CiAgXQp9dAAKZmllbGRUeXBlc3NyABdqYXZhLnV0aWwuTGlua2VkSGFzaE1hcDTATlwQbMD7AgABWgALYWNjZXNzT3JkZXJ4cgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAMdwgAAAAQAAAAC3QADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZXNyADpvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaFRleHRUeXBlrYOjkwTjMUQCAAFMAAZmaWVsZHN0AA9MamF2YS91dGlsL01hcDt4cgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hEYXRhVHlwZcJjvMoC+gU1AgADTAAMZXhwckNvcmVUeXBldAArTG9yZy9vcGVuc2VhcmNoL3NxbC9kYXRhL3R5cGUvRXhwckNvcmVUeXBlO0wAC21hcHBpbmdUeXBldABITG9yZy9vcGVuc2VhcmNoL3NxbC9vcGVuc2VhcmNoL2RhdGEvdHlwZS9PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGU7TAAKcHJvcGVydGllc3EAfgAReHB+cQB+AAt0AAdVTktOT1dOfnIARm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGUkTWFwcGluZ1R5cGUAAAAAAAAAABIAAHhxAH4ADHQABFRleHRzcgA8c2hhZGVkLmNvbS5nb29nbGUuY29tbW9uLmNvbGxlY3QuSW1tdXRhYmxlTWFwJFNlcmlhbGl6ZWRGb3JtAAAAAAAAAAACAAJMAARrZXlzdAASTGphdmEvbGFuZy9PYmplY3Q7TAAGdmFsdWVzcQB+ABx4cHVyABNbTGphdmEubGFuZy5PYmplY3Q7kM5YnxBzKWwCAAB4cAAAAAB1cQB+AB4AAAAAc3EAfgAAAAAAA3cEAAAAAnQAB2tleXdvcmRzcQB+ABJ+cQB+AAt0AAZTVFJJTkd+cQB+ABh0AAdLZXl3b3JkcQB+AB14dAAHYWRkcmVzc3NxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAB4dAAHYmFsYW5jZXEAfgANdAAGZ2VuZGVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAEY2l0eXNxAH4AEHEAfgAWcQB+ABlxAH4AHXNxAH4AAAAAAAN3BAAAAAJxAH4AInEAfgAjeHQACGVtcGxveWVyc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAFc3RhdGVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h0AANhZ2VxAH4ADXQABWVtYWlsc3EAfgAQcQB+ABZxAH4AGXEAfgAdc3EAfgAAAAAAA3cEAAAAAnEAfgAicQB+ACN4dAAIbGFzdG5hbWVzcQB+ABBxAH4AFnEAfgAZcQB+AB1zcQB+AAAAAAADdwQAAAACcQB+ACJxAH4AI3h4AHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json index 92ae65661e2..9fef24bdd87 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_skip_script_encoding.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(firstname=[$1], age=[$8], address=[$2])\n LogicalFilter(condition=[AND(=($2, '671 Bristol Street'), =(-($8, 2), 30))])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SCRIPT->AND(=($2, '671 Bristol Street'), =(-($8, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 2,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$2\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 8,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$8\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 30,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, address, age], SCRIPT->AND(=($1, '671 Bristol Street'), =(-($2, 2), 30)), PROJECT->[firstname, age, address], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"query\":{\"bool\":{\"must\":[{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 1,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$1\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": \\\\\\\"671 Bristol Street\\\\\\\",\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"VARCHAR\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false,\\\\n \\\\\\\"precision\\\\\\\": -1\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"{\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"=\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"EQUALS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"op\\\\\\\": {\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"-\\\\\\\",\\\\n \\\\\\\"kind\\\\\\\": \\\\\\\"MINUS\\\\\\\",\\\\n \\\\\\\"syntax\\\\\\\": \\\\\\\"BINARY\\\\\\\"\\\\n },\\\\n \\\\\\\"operands\\\\\\\": [\\\\n {\\\\n \\\\\\\"input\\\\\\\": 2,\\\\n \\\\\\\"name\\\\\\\": \\\\\\\"$2\\\\\\\"\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 2,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ],\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"BIGINT\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": true\\\\n }\\\\n },\\\\n {\\\\n \\\\\\\"literal\\\\\\\": 30,\\\\n \\\\\\\"type\\\\\\\": {\\\\n \\\\\\\"type\\\\\\\": \\\\\\\"INTEGER\\\\\\\",\\\\n \\\\\\\"nullable\\\\\\\": false\\\\n }\\\\n }\\\\n ]\\\\n}\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}}],\"adjust_pure_negative\":true,\"boost\":1.0}},\"_source\":{\"includes\":[\"firstname\",\"age\",\"address\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json index ba63d4580d4..8822d13bd70 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], dir0=[ASC-nulls-first])\n LogicalProject(avg_age=[$2], state=[$0], city=[$1])\n LogicalAggregate(group=[{0, 1}], avg_age=[AVG($2)])\n LogicalProject(state=[$7], city=[$5], age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableCalc(expr#0..2=[{inputs}], avg_age=[$t2], state=[$t0], city=[$t1])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2))], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0, 1},avg_age=AVG($2)), PROJECT->[avg_age, state, city]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}},{\"city\":{\"terms\":{\"field\":\"city.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg_age\":{\"avg\":{\"field\":\"age\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json index a6eb95580cc..204daa39ab0 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_desc_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(age=[$8], firstname=[$1])\n LogicalSort(sort0=[$8], sort1=[$1], dir0=[DESC-nulls-last], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[SORT->[{\n \"age\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"firstname.keyword\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], PROJECT->[age, firstname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\",\"firstname\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[firstname, age], SORT->[{\n \"age\" : {\n \"order\" : \"desc\",\n \"missing\" : \"_last\"\n }\n}, {\n \"firstname.keyword\" : {\n \"order\" : \"asc\",\n \"missing\" : \"_first\"\n }\n}], PROJECT->[age, firstname], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":10000,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\",\"firstname\"],\"excludes\":[]},\"sort\":[{\"age\":{\"order\":\"desc\",\"missing\":\"_last\"}},{\"firstname.keyword\":{\"order\":\"asc\",\"missing\":\"_first\"}}]}, requestedTotalSize=10000, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json index 1f14950f06c..5b355b16b0e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_sort_then_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(avg(balance)=[$1], state=[$0])\n LogicalAggregate(group=[{0}], avg(balance)=[AVG($1)])\n LogicalProject(state=[$7], balance=[$3])\n LogicalSort(sort0=[$3], sort1=[$8], dir0=[ASC-nulls-first], dir1=[ASC-nulls-first])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], avg(balance)=[$t1], state=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"state\",\"age\"],\"excludes\":[]},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[balance, state, age], AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(balance)=AVG($1)), PROJECT->[avg(balance), state], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"balance\",\"state\",\"age\"],\"excludes\":[]},\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"state\":{\"terms\":{\"field\":\"state.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]},\"aggregations\":{\"avg(balance)\":{\"avg\":{\"field\":\"balance\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml new file mode 100644 index 00000000000..cb3428897ac --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(count()=[$1], @timestamp=[$0]) + LogicalAggregate(group=[{0}], count()=[COUNT()]) + LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[0], expr#3=[>($t1, $t2)], count()=[$t1], @timestamp=[$t0], $condition=[$t3]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"count()":{"value_count":{"field":"_index"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml new file mode 100644 index 00000000000..a0080e88f90 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_bins_on_time2.yaml @@ -0,0 +1,11 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(avg(cpu_usage)=[$1], @timestamp=[$0]) + LogicalAggregate(group=[{0}], avg(cpu_usage)=[AVG($1)]) + LogicalProject(@timestamp=[WIDTH_BUCKET($1, 3, -(MAX($1) OVER (), MIN($1) OVER ()), MAX($1) OVER ())], cpu_usage=[$7]) + CalciteLogicalIndexScan(table=[[OpenSearch, events]]) + physical: | + EnumerableLimit(fetch=[10000]) + EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t1)], avg(cpu_usage)=[$t1], @timestamp=[$t0], $condition=[$t2]) + CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},avg(cpu_usage)=AVG($1))], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":0,"timeout":"1m","aggregations":{"@timestamp":{"auto_date_histogram":{"field":"@timestamp","buckets":3,"minimum_interval":null},"aggregations":{"avg(cpu_usage)":{"avg":{"field":"cpu_usage"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)]) diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json index 55a3597c8e6..01258e3fd0b 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json index 498092d47a2..779e12361b8 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_span_non_bucket_nullable.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalFilter(condition=[IS NOT NULL($0)])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[SCRIPT->IS NOT NULL(SPAN($10, 10, null:NULL)), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT())], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"script\":{\"script\":{\"source\":\"{\\\"langType\\\":\\\"calcite\\\",\\\"script\\\":\\\"rO0ABXNyABFqYXZhLnV0aWwuQ29sbFNlcleOq7Y6G6gRAwABSQADdGFneHAAAAADdwQAAAAGdAAHcm93VHlwZXQHjnsKICAiZmllbGRzIjogWwogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhY2NvdW50X251bWJlciIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImZpcnN0bmFtZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogImFkZHJlc3MiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJiaXJ0aGRhdGUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJnZW5kZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJjaXR5IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAibGFzdG5hbWUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJiYWxhbmNlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiZW1wbG95ZXIiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJzdGF0ZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIklOVEVHRVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJhZ2UiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJWQVJDSEFSIiwKICAgICAgIm51bGxhYmxlIjogdHJ1ZSwKICAgICAgInByZWNpc2lvbiI6IC0xLAogICAgICAibmFtZSI6ICJlbWFpbCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIkJPT0xFQU4iLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJtYWxlIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2lkIgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiVkFSQ0hBUiIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJwcmVjaXNpb24iOiAtMSwKICAgICAgIm5hbWUiOiAiX2luZGV4IgogICAgfSwKICAgIHsKICAgICAgInR5cGUiOiAiUkVBTCIsCiAgICAgICJudWxsYWJsZSI6IHRydWUsCiAgICAgICJuYW1lIjogIl9zY29yZSIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlJFQUwiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfbWF4c2NvcmUiCiAgICB9LAogICAgewogICAgICAidHlwZSI6ICJCSUdJTlQiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAibmFtZSI6ICJfc29ydCIKICAgIH0sCiAgICB7CiAgICAgICJ0eXBlIjogIlZBUkNIQVIiLAogICAgICAibnVsbGFibGUiOiB0cnVlLAogICAgICAicHJlY2lzaW9uIjogLTEsCiAgICAgICJuYW1lIjogIl9yb3V0aW5nIgogICAgfQogIF0sCiAgIm51bGxhYmxlIjogdHJ1ZQp9dAAEZXhwcnQDOXsKICAib3AiOiB7CiAgICAibmFtZSI6ICJJUyBOT1QgTlVMTCIsCiAgICAia2luZCI6ICJJU19OT1RfTlVMTCIsCiAgICAic3ludGF4IjogIlBPU1RGSVgiCiAgfSwKICAib3BlcmFuZHMiOiBbCiAgICB7CiAgICAgICJvcCI6IHsKICAgICAgICAibmFtZSI6ICJTUEFOIiwKICAgICAgICAia2luZCI6ICJPVEhFUl9GVU5DVElPTiIsCiAgICAgICAgInN5bnRheCI6ICJGVU5DVElPTiIKICAgICAgfSwKICAgICAgIm9wZXJhbmRzIjogWwogICAgICAgIHsKICAgICAgICAgICJpbnB1dCI6IDEwLAogICAgICAgICAgIm5hbWUiOiAiJDEwIgogICAgICAgIH0sCiAgICAgICAgewogICAgICAgICAgImxpdGVyYWwiOiAxMCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogZmFsc2UKICAgICAgICAgIH0KICAgICAgICB9LAogICAgICAgIHsKICAgICAgICAgICJsaXRlcmFsIjogbnVsbCwKICAgICAgICAgICJ0eXBlIjogewogICAgICAgICAgICAidHlwZSI6ICJOVUxMIiwKICAgICAgICAgICAgIm51bGxhYmxlIjogdHJ1ZQogICAgICAgICAgfQogICAgICAgIH0KICAgICAgXSwKICAgICAgImNsYXNzIjogIm9yZy5vcGVuc2VhcmNoLnNxbC5leHByZXNzaW9uLmZ1bmN0aW9uLlVzZXJEZWZpbmVkRnVuY3Rpb25CdWlsZGVyJDEiLAogICAgICAidHlwZSI6IHsKICAgICAgICAidHlwZSI6ICJJTlRFR0VSIiwKICAgICAgICAibnVsbGFibGUiOiB0cnVlCiAgICAgIH0sCiAgICAgICJkZXRlcm1pbmlzdGljIjogdHJ1ZSwKICAgICAgImR5bmFtaWMiOiBmYWxzZQogICAgfQogIF0KfXQACmZpZWxkVHlwZXNzcgARamF2YS51dGlsLkhhc2hNYXAFB9rBwxZg0QMAAkYACmxvYWRGYWN0b3JJAAl0aHJlc2hvbGR4cD9AAAAAAAAYdwgAAAAgAAAADXQADmFjY291bnRfbnVtYmVyfnIAKW9yZy5vcGVuc2VhcmNoLnNxbC5kYXRhLnR5cGUuRXhwckNvcmVUeXBlAAAAAAAAAAASAAB4cgAOamF2YS5sYW5nLkVudW0AAAAAAAAAABIAAHhwdAAETE9OR3QACWZpcnN0bmFtZX5xAH4ACnQABlNUUklOR3QAB2FkZHJlc3NzcgA6b3JnLm9wZW5zZWFyY2guc3FsLm9wZW5zZWFyY2guZGF0YS50eXBlLk9wZW5TZWFyY2hUZXh0VHlwZa2Do5ME4zFEAgABTAAGZmllbGRzdAAPTGphdmEvdXRpbC9NYXA7eHIAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0YVR5cGXCY7zKAvoFNQIAA0wADGV4cHJDb3JlVHlwZXQAK0xvcmcvb3BlbnNlYXJjaC9zcWwvZGF0YS90eXBlL0V4cHJDb3JlVHlwZTtMAAttYXBwaW5nVHlwZXQASExvcmcvb3BlbnNlYXJjaC9zcWwvb3BlbnNlYXJjaC9kYXRhL3R5cGUvT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlO0wACnByb3BlcnRpZXNxAH4AE3hwfnEAfgAKdAAHVU5LTk9XTn5yAEZvcmcub3BlbnNlYXJjaC5zcWwub3BlbnNlYXJjaC5kYXRhLnR5cGUuT3BlblNlYXJjaERhdGFUeXBlJE1hcHBpbmdUeXBlAAAAAAAAAAASAAB4cQB+AAt0AARUZXh0c3IAPHNoYWRlZC5jb20uZ29vZ2xlLmNvbW1vbi5jb2xsZWN0LkltbXV0YWJsZU1hcCRTZXJpYWxpemVkRm9ybQAAAAAAAAAAAgACTAAEa2V5c3QAEkxqYXZhL2xhbmcvT2JqZWN0O0wABnZhbHVlc3EAfgAeeHB1cgATW0xqYXZhLmxhbmcuT2JqZWN0O5DOWJ8QcylsAgAAeHAAAAAAdXEAfgAgAAAAAHNxAH4AAAAAAAN3BAAAAAB4dAAJYmlydGhkYXRlc3IAOm9yZy5vcGVuc2VhcmNoLnNxbC5vcGVuc2VhcmNoLmRhdGEudHlwZS5PcGVuU2VhcmNoRGF0ZVR5cGWeLVKuEH3KrwIAAUwAB2Zvcm1hdHN0ABBMamF2YS91dGlsL0xpc3Q7eHEAfgAUfnEAfgAKdAAJVElNRVNUQU1QfnEAfgAadAAERGF0ZXEAfgAfc3EAfgAAAAAAAXcEAAAAAHh0AAZnZW5kZXJzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACdAAHa2V5d29yZHNxAH4AFHEAfgAPfnEAfgAadAAHS2V5d29yZHEAfgAfeHQABGNpdHlxAH4AD3QACGxhc3RuYW1lcQB+AA90AAdiYWxhbmNlcQB+AAx0AAhlbXBsb3llcnNxAH4AEnEAfgAYcQB+ABtxAH4AH3EAfgAjdAAFc3RhdGVzcQB+ABJxAH4AGHEAfgAbcQB+AB9zcQB+AAAAAAADdwQAAAACcQB+ADBxAH4AMXh0AANhZ2V+cQB+AAp0AAdJTlRFR0VSdAAFZW1haWxzcQB+ABJxAH4AGHEAfgAbcQB+AB9xAH4AI3QABG1hbGV+cQB+AAp0AAdCT09MRUFOeHg=\\\"}\",\"lang\":\"opensearch_compounded_script\",\"params\":{\"utcTimestamp\":*}},\"boost\":1.0}},\"sort\":[],\"aggregations\":{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"interval\":10.0,\"offset\":0.0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[age], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(age,10)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"age\",\"boost\":1.0}},\"_source\":{\"includes\":[\"age\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json index 8594e2c7235..e44d3d9c613 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1m)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json index bd39fca0df6..774863863c2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_stats_by_timespan2.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]], PushDownContext=[[PROJECT->[birthdate], FILTER->IS NOT NULL($0), AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={0},count()=COUNT()), PROJECT->[count(), span(birthdate,1M)], LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"birthdate\",\"boost\":1.0}},\"_source\":{\"includes\":[\"birthdate\"],\"excludes\":[]},\"sort\":[],\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json index 790e1e3b11c..8430c670a13 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart.json @@ -1,6 +1,6 @@ { "calcite": { "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n LogicalProject(@timestamp=[$0], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], avg(cpu_usage)=[$2])\n LogicalJoin(condition=[=($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2=[$2])\n LogicalAggregate(group=[{0, 2}], agg#0=[AVG($1)])\n LogicalProject(host=[$4], cpu_usage=[$7], $f3=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[SUM($1)])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], host=[$t0], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], avg(cpu_usage)=[SUM($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[IS NOT NULL($t3)], expr#6=[IS NULL($t1)], expr#7=[null:NULL], expr#8=['OTHER'], expr#9=[CASE($t6, $t7, $t8)], expr#10=[CASE($t5, $t1, $t9)], @timestamp=[$t0], host=[$t10], avg(cpu_usage)=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], @timestamp=[$t1], host=[$t0], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[SUM($2)])\n EnumerableCalc(expr#0..3=[{inputs}], expr#4=[0], expr#5=[=($t3, $t4)], expr#6=[null:DOUBLE], expr#7=[CASE($t5, $t6, $t2)], expr#8=[/($t7, $t3)], proj#0..1=[{exprs}], $f2=[$t8])\n EnumerableAggregate(group=[{0, 2}], agg#0=[$SUM0($1)], agg#1=[COUNT($1)])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[1], expr#4=['m'], expr#5=[SPAN($t2, $t3, $t4)], proj#0..1=[{exprs}], $f2=[$t5])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host, cpu_usage], FILTER->IS NOT NULL($1), PROJECT->[host, cpu_usage, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"cpu_usage\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" } -} \ No newline at end of file +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json index 01e5362b9c2..377746b83a5 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_timechart_count.json @@ -1 +1,6 @@ -{"calcite":{"logical":"LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], count=[SUM($2)])\n LogicalUnion(all=[false])\n LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0])\n LogicalJoin(condition=[true], joinType=[inner])\n LogicalAggregate(group=[{0}])\n LogicalProject(@timestamp=[$1])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalAggregate(group=[{0}])\n LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n","physical":"EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)])\n EnumerableUnion(all=[false])\n EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2])\n EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3])\n EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[AGGREGATION->rel#:LogicalAggregate.NONE.[](input=RelSubset#,group={1})], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"$f2\":{\"date_histogram\":{\"field\":\"@timestamp\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]}}}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableAggregate(group=[{0}])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..1=[{inputs}], host=[$t0])\n EnumerableAggregate(group=[{0, 1}])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp], FILTER->IS NOT NULL($0)], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n"}} \ No newline at end of file +{ + "calcite": { + "logical": "LogicalSystemLimit(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n LogicalAggregate(group=[{0, 1}], count=[SUM($2)])\n LogicalUnion(all=[false])\n LogicalAggregate(group=[{0, 1}], actual_count=[SUM($2)])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))], count=[$2])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalProject(@timestamp=[CAST($0):TIMESTAMP(0) NOT NULL], host=[$1], count=[0])\n LogicalJoin(condition=[true], joinType=[inner])\n LogicalAggregate(group=[{0}])\n LogicalProject(@timestamp=[$1])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalAggregate(group=[{0}])\n LogicalProject($f0=[CASE(IS NOT NULL($3), $1, CASE(IS NULL($1), null:NULL, 'OTHER'))])\n LogicalJoin(condition=[IS NOT DISTINCT FROM($1, $3)], joinType=[left])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n LogicalSort(sort0=[$1], dir0=[DESC], fetch=[10])\n LogicalAggregate(group=[{1}], grand_total=[SUM($2)])\n LogicalFilter(condition=[IS NOT NULL($1)])\n LogicalProject(@timestamp=[$1], host=[$0], $f2_0=[$2])\n LogicalAggregate(group=[{0, 1}], agg#0=[COUNT()])\n LogicalProject(host=[$4], $f2=[SPAN($1, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, events]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[ASC])\n EnumerableAggregate(group=[{0, 1}], count=[$SUM0($2)])\n EnumerableUnion(all=[false])\n EnumerableAggregate(group=[{0, 1}], actual_count=[$SUM0($2)])\n EnumerableCalc(expr#0..4=[{inputs}], expr#5=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#6=[IS NOT NULL($t3)], expr#7=[IS NULL($t1)], expr#8=[null:NULL], expr#9=['OTHER'], expr#10=[CASE($t7, $t8, $t9)], expr#11=[CASE($t6, $t1, $t10)], @timestamp=[$t5], host=[$t11], count=[$t2])\n EnumerableMergeJoin(condition=[=($1, $3)], joinType=[left])\n EnumerableSort(sort0=[$1], dir0=[ASC])\n EnumerableCalc(expr#0..2=[{inputs}], @timestamp=[$t1], host=[$t0], $f2_0=[$t2])\n EnumerableAggregate(group=[{0, 1}], agg#0=[COUNT()])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($1), PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[CAST($t0):TIMESTAMP(0) NOT NULL], expr#3=[0], @timestamp=[$t2], host=[$t1], count=[$t3])\n EnumerableNestedLoopJoin(condition=[true], joinType=[inner])\n EnumerableAggregate(group=[{1}])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableAggregate(group=[{0}])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[IS NOT NULL($t1)], expr#4=[IS NULL($t0)], expr#5=[null:NULL], expr#6=['OTHER'], expr#7=[CASE($t4, $t5, $t6)], expr#8=[CASE($t3, $t0, $t7)], $f0=[$t8])\n EnumerableMergeJoin(condition=[=($0, $1)], joinType=[left])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableCalc(expr#0..1=[{inputs}], host=[$t0])\n EnumerableAggregate(group=[{0, 1}])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[1], expr#3=['m'], expr#4=[SPAN($t1, $t2, $t3)], host=[$t0], $f1=[$t4])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n EnumerableSort(sort0=[$0], dir0=[ASC])\n EnumerableLimit(fetch=[10])\n EnumerableSort(sort0=[$1], dir0=[DESC])\n EnumerableAggregate(group=[{0}], grand_total=[COUNT()])\n CalciteEnumerableIndexScan(table=[[OpenSearch, events]], PushDownContext=[[PROJECT->[@timestamp, host], FILTER->IS NOT NULL($1), PROJECT->[host, @timestamp]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"query\":{\"exists\":{\"field\":\"host\",\"boost\":1.0}},\"_source\":{\"includes\":[\"host\",\"@timestamp\"],\"excludes\":[]},\"sort\":[{\"_doc\":{\"order\":\"asc\"}}]}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_values_aggregation.json b/integ-test/src/test/resources/expectedOutput/calcite/explain_values_aggregation.json new file mode 100644 index 00000000000..1dd0bc574f0 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_values_aggregation.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], age_values=[VALUES($0)])\n LogicalProject(age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], age_values=[VALUES($0)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[age]], OpenSearchRequestBuilder(sourceBuilder={\"from\":0,\"timeout\":\"1m\",\"_source\":{\"includes\":[\"age\"],\"excludes\":[]}}, requestedTotalSize=2147483647, pageSize=null, startFrom=0)])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json new file mode 100644 index 00000000000..e0dcbaaeb8d --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_timestamp_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(sort0=[$1], dir0=[ASC-nulls-first], fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalSort(sort0=[$1], dir0=[ASC-nulls-first], fetch=[3])\n LogicalProject(count()=[$1], t=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(t=[UNIX_TIMESTAMP($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], t=[$t0])\n EnumerableLimit(fetch=[3])\n EnumerableSort(sort0=[$0], dir0=[ASC-nulls-first])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[UNIX_TIMESTAMP($t3)], t=[$t19])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json new file mode 100644 index 00000000000..39ec3279c63 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_agg_script_udt_arg_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(t,1d)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(t,1d)=[SPAN($19, 1, 'd')])\n LogicalFilter(condition=[IS NOT NULL($19)])\n LogicalProject(account_number=[$0], firstname=[$1], address=[$2], birthdate=[$3], gender=[$4], city=[$5], lastname=[$6], balance=[$7], employer=[$8], state=[$9], age=[$10], email=[$11], male=[$12], _id=[$13], _index=[$14], _score=[$15], _maxscore=[$16], _sort=[$17], _routing=[$18], t=[DATE_ADD($3, 1:INTERVAL DAY)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(t,1d)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1:INTERVAL DAY], expr#20=[DATE_ADD($t3, $t19)], expr#21=[1], expr#22=['d'], expr#23=[SPAN($t20, $t21, $t22)], expr#24=[IS NOT NULL($t20)], span(t,1d)=[$t23], $condition=[$t24])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json new file mode 100644 index 00000000000..545630e2166 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_filter_script_ip_push.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(host=[$0])\n LogicalFilter(condition=[CIDRMATCH($0, '0.0.0.0/24':VARCHAR)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..11=[{inputs}], expr#12=['0.0.0.0/24':VARCHAR], expr#13=[CIDRMATCH($t0, $t12)], host=[$t0], $condition=[$t13])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_weblogs]])\n" + } +} diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json new file mode 100644 index 00000000000..8a84763e33c --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_max_string_field.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], max(firstname)=[MAX($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], max(firstname)=[MAX($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json new file mode 100644 index 00000000000..320b519c442 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_min_string_field.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], min(firstname)=[MIN($0)])\n LogicalProject(firstname=[$1])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], min(firstname)=[MIN($1)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_brain_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_brain_agg_push.json index 6389c6dbc70..bd4d36474e2 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_brain_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_brain_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM($1, 'pattern'))], pattern_count=[SAFE_CAST(ITEM($1, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($1, 'tokens'))])\n LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n LogicalAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n LogicalProject(email=[$9], $f17=[10], $f18=[100000])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n Uncollect\n LogicalProject(patterns_field=[$cor0.patterns_field])\n LogicalValues(tuples=[[{ 0 }]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=['pattern'], expr#3=[ITEM($t1, $t2)], expr#4=[SAFE_CAST($t3)], expr#5=['pattern_count'], expr#6=[ITEM($t1, $t5)], expr#7=[SAFE_CAST($t6)], expr#8=['tokens'], expr#9=[ITEM($t1, $t8)], expr#10=[SAFE_CAST($t9)], patterns_field=[$t4], pattern_count=[$t7], tokens=[$t10])\n EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n EnumerableAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[10], expr#18=[100000], email=[$t9], $f17=[$t17], $f18=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n EnumerableUncollect\n EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.patterns_field], patterns_field=[$t2])\n EnumerableValues(tuples=[[{ 0 }]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM($1, 'pattern'))], pattern_count=[SAFE_CAST(ITEM($1, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($1, 'tokens'))], sample_logs=[SAFE_CAST(ITEM($1, 'sample_logs'))])\n LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n LogicalAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n LogicalProject(email=[$9], $f17=[10], $f18=[100000])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n Uncollect\n LogicalProject(patterns_field=[$cor0.patterns_field])\n LogicalValues(tuples=[[{ 0 }]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=['pattern'], expr#3=[ITEM($t1, $t2)], expr#4=[SAFE_CAST($t3)], expr#5=['pattern_count'], expr#6=[ITEM($t1, $t5)], expr#7=[SAFE_CAST($t6)], expr#8=['tokens'], expr#9=[ITEM($t1, $t8)], expr#10=[SAFE_CAST($t9)], expr#11=['sample_logs'], expr#12=[ITEM($t1, $t11)], expr#13=[SAFE_CAST($t12)], patterns_field=[$t4], pattern_count=[$t7], tokens=[$t10], sample_logs=[$t13])\n EnumerableCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n EnumerableAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=[10], expr#18=[100000], email=[$t9], $f17=[$t17], $f18=[$t18])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n EnumerableUncollect\n EnumerableCalc(expr#0=[{inputs}], expr#1=[$cor0], expr#2=[$t1.patterns_field], patterns_field=[$t2])\n EnumerableValues(tuples=[[{ 0 }]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json index 0f0830c05e5..73affef775a 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_patterns_simple_pattern_agg_push.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(pattern_count=[$1], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], pattern_count=[$t1], patterns_field=[$t6], tokens=[$t9])\n EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['[a-zA-Z0-9]+':VARCHAR], expr#18=['<*>'], expr#19=[REGEXP_REPLACE($t9, $t17, $t18)], expr#20=[10], email=[$t9], patterns_field=[$t19], $f18=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))], pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))], sample_logs=[$2])\n LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n LogicalProject(email=[$9], patterns_field=[REGEXP_REPLACE($9, '[a-zA-Z0-9]+':VARCHAR, '<*>')], $f18=[10])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..2=[{inputs}], expr#3=[PATTERN_PARSER($t0, $t2)], expr#4=['pattern'], expr#5=[ITEM($t3, $t4)], expr#6=[SAFE_CAST($t5)], expr#7=['tokens'], expr#8=[ITEM($t3, $t7)], expr#9=[SAFE_CAST($t8)], patterns_field=[$t6], pattern_count=[$t1], tokens=[$t9], sample_logs=[$t2])\n EnumerableAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0, $2)])\n EnumerableCalc(expr#0..16=[{inputs}], expr#17=['[a-zA-Z0-9]+':VARCHAR], expr#18=['<*>'], expr#19=[REGEXP_REPLACE($t9, $t17, $t18)], expr#20=[10], email=[$t9], patterns_field=[$t19], $f18=[$t20])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json index 8b383eea6e0..ad39983d50d 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_span_non_bucket_nullable.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalFilter(condition=[IS NOT NULL($0)])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], expr#2=[IS NOT NULL($t0)], count()=[$t1], span(age,10)=[$t0], $condition=[$t2])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], span(age,10)=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(age,10)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(age,10)=[SPAN($10, 10, null:NULL)])\n LogicalFilter(condition=[IS NOT NULL($10)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(age,10)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[10], expr#20=[null:NULL], expr#21=[SPAN($t10, $t19, $t20)], expr#22=[IS NOT NULL($t10)], span(age,10)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } } \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json index 0c08b738328..1b846e6c16e 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['m'], expr#21=[SPAN($t3, $t19, $t20)], span(birthdate,1m)=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1m)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1m)=[SPAN($3, 1, 'm')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1m)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['m'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1m)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json index 985fbc55afa..9cec9bcf190 100644 --- a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_stats_by_timespan2.json @@ -1,6 +1,6 @@ { "calcite": { - "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", - "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['M'], expr#21=[SPAN($t3, $t19, $t20)], span(birthdate,1M)=[$t21])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalProject(count()=[$1], span(birthdate,1M)=[$0])\n LogicalAggregate(group=[{0}], count()=[COUNT()])\n LogicalProject(span(birthdate,1M)=[SPAN($3, 1, 'M')])\n LogicalFilter(condition=[IS NOT NULL($3)])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableCalc(expr#0..1=[{inputs}], count()=[$t1], span(birthdate,1M)=[$t0])\n EnumerableAggregate(group=[{0}], count()=[COUNT()])\n EnumerableCalc(expr#0..18=[{inputs}], expr#19=[1], expr#20=['M'], expr#21=[SPAN($t3, $t19, $t20)], expr#22=[IS NOT NULL($t3)], span(birthdate,1M)=[$t21], $condition=[$t22])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_bank]])\n" } } diff --git a/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json new file mode 100644 index 00000000000..7fc9ba40c61 --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite_no_pushdown/explain_values_aggregation.json @@ -0,0 +1,6 @@ +{ + "calcite": { + "logical": "LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT])\n LogicalAggregate(group=[{}], age_values=[VALUES($0)])\n LogicalProject(age=[$8])\n CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n", + "physical": "EnumerableLimit(fetch=[10000])\n EnumerableAggregate(group=[{}], age_values=[VALUES($8)])\n CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]])\n" + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json index bdc1b0c7405..35c137ab172 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_span_non_bucket_nullable.json @@ -7,7 +7,7 @@ "children": [{ "name": "OpenSearchIndexScan", "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"interval\":10.0,\"offset\":0.0,\"order\":{\"_key\":\"asc\"},\"keyed\":false,\"min_doc_count\":0},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(age,10)\":{\"histogram\":{\"field\":\"age\",\"missing_bucket\":false,\"order\":\"asc\",\"interval\":10.0}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" }, "children": [] }] diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json index b2e6475b462..7d345202ce4 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan.json @@ -4,14 +4,12 @@ "description": { "fields": "[count(), span(birthdate,1m)]" }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] + "children": [{ + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1m)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"fixed_interval\":\"1m\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + }] } } diff --git a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json index 92f6218eabf..a5b9e210f09 100644 --- a/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json +++ b/integ-test/src/test/resources/expectedOutput/ppl/explain_stats_by_timespan2.json @@ -4,14 +4,12 @@ "description": { "fields": "[count(), span(birthdate,1M)]" }, - "children": [ - { - "name": "OpenSearchIndexScan", - "description": { - "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" - }, - "children": [] - } - ] + "children": [{ + "name": "OpenSearchIndexScan", + "description": { + "request": "OpenSearchQueryRequest(indexName=opensearch-sql_test_index_bank, sourceBuilder={\"from\":0,\"size\":0,\"timeout\":\"1m\",\"aggregations\":{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[{\"span(birthdate,1M)\":{\"date_histogram\":{\"field\":\"birthdate\",\"missing_bucket\":false,\"order\":\"asc\",\"calendar_interval\":\"1M\"}}}]},\"aggregations\":{\"count()\":{\"value_count\":{\"field\":\"_index\"}}}}}}, needClean=true, searchDone=false, pitId=*, cursorKeepAlive=null, searchAfter=null, searchResponse=null)" + }, + "children": [] + }] } } diff --git a/integ-test/src/test/resources/indexDefinitions/complex_geo_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/complex_geo_index_mapping.json new file mode 100644 index 00000000000..10b855f6162 --- /dev/null +++ b/integ-test/src/test/resources/indexDefinitions/complex_geo_index_mapping.json @@ -0,0 +1,73 @@ +{ + "mappings": { + "properties": { + "id": { + "type": "keyword" + }, + "location": { + "properties": { + "name": { + "type": "keyword" + }, + "point": { + "type": "geo_point" + }, + "city": { + "type": "keyword" + }, + "country": { + "type": "keyword" + } + } + }, + "nested_locations": { + "properties": { + "primary": { + "properties": { + "office": { + "type": "geo_point" + }, + "warehouse": { + "type": "geo_point" + } + } + }, + "secondary": { + "properties": { + "branch": { + "type": "geo_point" + }, + "store": { + "type": "geo_point" + } + } + } + } + }, + "multiple_offices": { + "properties": { + "headquarters": { + "properties": { + "location": { + "type": "geo_point" + }, + "address": { + "type": "text" + } + } + }, + "regional": { + "properties": { + "location": { + "type": "geo_point" + }, + "address": { + "type": "text" + } + } + } + } + } + } + } +} \ No newline at end of file diff --git a/integ-test/src/test/resources/time_test_data_with_null.json b/integ-test/src/test/resources/time_test_data_with_null.json new file mode 100644 index 00000000000..bd1f2fc1fcf --- /dev/null +++ b/integ-test/src/test/resources/time_test_data_with_null.json @@ -0,0 +1,200 @@ +{"index":{"_id":"1"}} +{"timestamp":"2025-07-28T00:15:23","value":8945,"category":"A","@timestamp":"2025-07-28T00:15:23"} +{"index":{"_id":"2"}} +{"timestamp":"2025-07-28T01:42:15","value":7623,"category":"B","@timestamp":"2025-07-28T01:42:15"} +{"index":{"_id":"3"}} +{"timestamp":"2025-07-28T02:28:45","value":9187,"category":"C","@timestamp":"2025-07-28T02:28:45"} +{"index":{"_id":"4"}} +{"timestamp":"2025-07-28T03:56:20","value":6834,"category":"A","@timestamp":"2025-07-28T03:56:20"} +{"index":{"_id":"5"}} +{"timestamp":"2025-07-28T04:33:10","value":8291,"category":"D","@timestamp":"2025-07-28T04:33:10"} +{"index":{"_id":"6"}} +{"timestamp":"2025-07-28T05:17:55","value":7456,"category":"B","@timestamp":"2025-07-28T05:17:55"} +{"index":{"_id":"7"}} +{"timestamp":"2025-07-28T06:04:40","value":9012,"category":"C","@timestamp":"2025-07-28T06:04:40"} +{"index":{"_id":"8"}} +{"timestamp":"2025-07-28T07:51:25","value":6589,"category":"A","@timestamp":"2025-07-28T07:51:25"} +{"index":{"_id":"9"}} +{"timestamp":"2025-07-28T08:38:12","value":8736,"category":"D","@timestamp":"2025-07-28T08:38:12"} +{"index":{"_id":"10"}} +{"timestamp":"2025-07-28T09:15:03","value":7198,"category":"B","@timestamp":"2025-07-28T09:15:03"} +{"index":{"_id":"11"}} +{"timestamp":"2025-07-28T10:22:48","value":8523,"category":"C","@timestamp":"2025-07-28T10:22:48"} +{"index":{"_id":"12"}} +{"timestamp":"2025-07-28T11:09:33","value":9367,"category":"A","@timestamp":"2025-07-28T11:09:33"} +{"index":{"_id":"13"}} +{"timestamp":"2025-07-28T12:56:18","value":6712,"category":"D","@timestamp":"2025-07-28T12:56:18"} +{"index":{"_id":"14"}} +{"timestamp":"2025-07-28T13:43:07","value":8094,"category":"B","@timestamp":"2025-07-28T13:43:07"} +{"index":{"_id":"15"}} +{"timestamp":"2025-07-28T14:29:52","value":7831,"category":"C","@timestamp":"2025-07-28T14:29:52"} +{"index":{"_id":"16"}} +{"timestamp":"2025-07-28T15:16:37","value":9245,"category":"A","@timestamp":"2025-07-28T15:16:37"} +{"index":{"_id":"17"}} +{"timestamp":"2025-07-28T16:03:22","value":6478,"category":"D","@timestamp":"2025-07-28T16:03:22"} +{"index":{"_id":"18"}} +{"timestamp":"2025-07-28T17:50:15","value":8652,"category":"B","@timestamp":"2025-07-28T17:50:15"} +{"index":{"_id":"19"}} +{"timestamp":"2025-07-28T18:37:08","value":7359,"category":"C","@timestamp":"2025-07-28T18:37:08"} +{"index":{"_id":"20"}} +{"timestamp":"2025-07-28T19:24:53","value":8917,"category":"A","@timestamp":"2025-07-28T19:24:53"} +{"index":{"_id":"21"}} +{"timestamp":"2025-07-28T20:11:38","value":6543,"category":"D","@timestamp":"2025-07-28T20:11:38"} +{"index":{"_id":"22"}} +{"timestamp":"2025-07-28T21:58:23","value":9103,"category":"B","@timestamp":"2025-07-28T21:58:23"} +{"index":{"_id":"23"}} +{"timestamp":"2025-07-28T22:45:16","value":7726,"category":"C","@timestamp":"2025-07-28T22:45:16"} +{"index":{"_id":"24"}} +{"timestamp":"2025-07-28T23:32:01","value":8384,"category":"A","@timestamp":"2025-07-28T23:32:01"} +{"index":{"_id":"25"}} +{"timestamp":"2025-07-29T00:18:46","value":6897,"category":"D","@timestamp":"2025-07-29T00:18:46"} +{"index":{"_id":"26"}} +{"timestamp":"2025-07-29T01:05:31","value":9521,"category":"B","@timestamp":"2025-07-29T01:05:31"} +{"index":{"_id":"27"}} +{"timestamp":"2025-07-29T02:52:24","value":7162,"category":"C","@timestamp":"2025-07-29T02:52:24"} +{"index":{"_id":"28"}} +{"timestamp":"2025-07-29T03:39:17","value":8798,"category":"A","@timestamp":"2025-07-29T03:39:17"} +{"index":{"_id":"29"}} +{"timestamp":"2025-07-29T04:26:02","value":6235,"category":"D","@timestamp":"2025-07-29T04:26:02"} +{"index":{"_id":"30"}} +{"timestamp":"2025-07-29T05:12:55","value":8961,"category":"B","@timestamp":"2025-07-29T05:12:55"} +{"index":{"_id":"31"}} +{"timestamp":"2025-07-29T06:59:40","value":7584,"category":"C","@timestamp":"2025-07-29T06:59:40"} +{"index":{"_id":"32"}} +{"timestamp":"2025-07-29T07:46:25","value":9306,"category":"A","@timestamp":"2025-07-29T07:46:25"} +{"index":{"_id":"33"}} +{"timestamp":"2025-07-29T08:33:18","value":6751,"category":"D","@timestamp":"2025-07-29T08:33:18"} +{"index":{"_id":"34"}} +{"timestamp":"2025-07-29T09:20:03","value":8429,"category":"B","@timestamp":"2025-07-29T09:20:03"} +{"index":{"_id":"35"}} +{"timestamp":"2025-07-29T10:06:48","value":7295,"category":"C","@timestamp":"2025-07-29T10:06:48"} +{"index":{"_id":"36"}} +{"timestamp":"2025-07-29T11:53:33","value":8873,"category":"A","@timestamp":"2025-07-29T11:53:33"} +{"index":{"_id":"37"}} +{"timestamp":"2025-07-29T12:40:26","value":6618,"category":"D","@timestamp":"2025-07-29T12:40:26"} +{"index":{"_id":"38"}} +{"timestamp":"2025-07-29T13:27:11","value":9094,"category":"B","@timestamp":"2025-07-29T13:27:11"} +{"index":{"_id":"39"}} +{"timestamp":"2025-07-29T14:13:56","value":7467,"category":"C","@timestamp":"2025-07-29T14:13:56"} +{"index":{"_id":"40"}} +{"timestamp":"2025-07-29T15:00:41","value":8542,"category":"A","@timestamp":"2025-07-29T15:00:41"} +{"index":{"_id":"41"}} +{"timestamp":"2025-07-29T16:47:34","value":6985,"category":"D","@timestamp":"2025-07-29T16:47:34"} +{"index":{"_id":"42"}} +{"timestamp":"2025-07-29T17:34:19","value":8216,"category":"B","@timestamp":"2025-07-29T17:34:19"} +{"index":{"_id":"43"}} +{"timestamp":"2025-07-29T18:21:04","value":7653,"category":"C","@timestamp":"2025-07-29T18:21:04"} +{"index":{"_id":"44"}} +{"timestamp":"2025-07-29T19:07:49","value":9321,"category":"A","@timestamp":"2025-07-29T19:07:49"} +{"index":{"_id":"45"}} +{"timestamp":"2025-07-29T20:54:42","value":6798,"category":"D","@timestamp":"2025-07-29T20:54:42"} +{"index":{"_id":"46"}} +{"timestamp":"2025-07-29T21:41:27","value":8574,"category":"B","@timestamp":"2025-07-29T21:41:27"} +{"index":{"_id":"47"}} +{"timestamp":"2025-07-29T22:28:12","value":7241,"category":"C","@timestamp":"2025-07-29T22:28:12"} +{"index":{"_id":"48"}} +{"timestamp":"2025-07-29T23:14:57","value":8917,"category":"A","@timestamp":"2025-07-29T23:14:57"} +{"index":{"_id":"49"}} +{"timestamp":"2025-07-30T00:01:50","value":6583,"category":"D","@timestamp":"2025-07-30T00:01:50"} +{"index":{"_id":"50"}} +{"timestamp":"2025-07-30T01:48:35","value":9105,"category":"B","@timestamp":"2025-07-30T01:48:35"} +{"index":{"_id":"51"}} +{"timestamp":"2025-07-30T02:35:20","value":7428,"category":"C","@timestamp":"2025-07-30T02:35:20"} +{"index":{"_id":"52"}} +{"timestamp":"2025-07-30T03:22:05","value":8756,"category":"A","@timestamp":"2025-07-30T03:22:05"} +{"index":{"_id":"53"}} +{"timestamp":"2025-07-30T04:08:58","value":6341,"category":"D","@timestamp":"2025-07-30T04:08:58"} +{"index":{"_id":"54"}} +{"timestamp":"2025-07-30T05:55:43","value":8912,"category":"B","@timestamp":"2025-07-30T05:55:43"} +{"index":{"_id":"55"}} +{"timestamp":"2025-07-30T06:42:28","value":7685,"category":"C","@timestamp":"2025-07-30T06:42:28"} +{"index":{"_id":"56"}} +{"timestamp":"2025-07-30T07:29:13","value":9234,"category":"A","@timestamp":"2025-07-30T07:29:13"} +{"index":{"_id":"57"}} +{"timestamp":"2025-07-30T08:16:06","value":6827,"category":"D","@timestamp":"2025-07-30T08:16:06"} +{"index":{"_id":"58"}} +{"timestamp":"2025-07-30T09:02:51","value":8493,"category":"B","@timestamp":"2025-07-30T09:02:51"} +{"index":{"_id":"59"}} +{"timestamp":"2025-07-30T10:49:36","value":7156,"category":"C","@timestamp":"2025-07-30T10:49:36"} +{"index":{"_id":"60"}} +{"timestamp":"2025-07-30T11:36:21","value":8679,"category":"A","@timestamp":"2025-07-30T11:36:21"} +{"index":{"_id":"61"}} +{"timestamp":"2025-07-30T12:23:14","value":6492,"category":"D","@timestamp":"2025-07-30T12:23:14"} +{"index":{"_id":"62"}} +{"timestamp":"2025-07-30T13:09:59","value":9018,"category":"B","@timestamp":"2025-07-30T13:09:59"} +{"index":{"_id":"63"}} +{"timestamp":"2025-07-30T14:56:44","value":7351,"category":"C","@timestamp":"2025-07-30T14:56:44"} +{"index":{"_id":"64"}} +{"timestamp":"2025-07-30T15:43:29","value":8765,"category":"A","@timestamp":"2025-07-30T15:43:29"} +{"index":{"_id":"65"}} +{"timestamp":"2025-07-30T16:30:22","value":6208,"category":"D","@timestamp":"2025-07-30T16:30:22"} +{"index":{"_id":"66"}} +{"timestamp":"2025-07-30T17:17:07","value":8941,"category":"B","@timestamp":"2025-07-30T17:17:07"} +{"index":{"_id":"67"}} +{"timestamp":"2025-07-30T18:03:52","value":7574,"category":"C","@timestamp":"2025-07-30T18:03:52"} +{"index":{"_id":"68"}} +{"timestamp":"2025-07-30T19:50:37","value":9187,"category":"A","@timestamp":"2025-07-30T19:50:37"} +{"index":{"_id":"69"}} +{"timestamp":"2025-07-30T20:37:30","value":6753,"category":"D","@timestamp":"2025-07-30T20:37:30"} +{"index":{"_id":"70"}} +{"timestamp":"2025-07-30T21:24:15","value":8426,"category":"B","@timestamp":"2025-07-30T21:24:15"} +{"index":{"_id":"71"}} +{"timestamp":"2025-07-30T22:11:00","value":7289,"category":"C","@timestamp":"2025-07-30T22:11:00"} +{"index":{"_id":"72"}} +{"timestamp":"2025-07-30T23:57:45","value":8862,"category":"A","@timestamp":"2025-07-30T23:57:45"} +{"index":{"_id":"73"}} +{"timestamp":"2025-07-31T00:44:38","value":6615,"category":"D","@timestamp":"2025-07-31T00:44:38"} +{"index":{"_id":"74"}} +{"timestamp":"2025-07-31T01:31:23","value":9091,"category":"B","@timestamp":"2025-07-31T01:31:23"} +{"index":{"_id":"75"}} +{"timestamp":"2025-07-31T02:18:08","value":7464,"category":"C","@timestamp":"2025-07-31T02:18:08"} +{"index":{"_id":"76"}} +{"timestamp":"2025-07-31T03:04:53","value":8537,"category":"A","@timestamp":"2025-07-31T03:04:53"} +{"index":{"_id":"77"}} +{"timestamp":"2025-07-31T04:51:46","value":6982,"category":"D","@timestamp":"2025-07-31T04:51:46"} +{"index":{"_id":"78"}} +{"timestamp":"2025-07-31T05:38:31","value":8213,"category":"B","@timestamp":"2025-07-31T05:38:31"} +{"index":{"_id":"79"}} +{"timestamp":"2025-07-31T06:25:16","value":7649,"category":"C","@timestamp":"2025-07-31T06:25:16"} +{"index":{"_id":"80"}} +{"timestamp":"2025-07-31T07:12:01","value":9318,"category":"A","@timestamp":"2025-07-31T07:12:01"} +{"index":{"_id":"81"}} +{"timestamp":"2025-07-31T08:58:54","value":6795,"category":"D","@timestamp":"2025-07-31T08:58:54"} +{"index":{"_id":"82"}} +{"timestamp":"2025-07-31T09:45:39","value":8571,"category":"B","@timestamp":"2025-07-31T09:45:39"} +{"index":{"_id":"83"}} +{"timestamp":"2025-07-31T10:32:24","value":7238,"category":"C","@timestamp":"2025-07-31T10:32:24"} +{"index":{"_id":"84"}} +{"timestamp":"2025-07-31T11:19:09","value":8914,"category":"A","@timestamp":"2025-07-31T11:19:09"} +{"index":{"_id":"85"}} +{"timestamp":"2025-07-31T12:06:02","value":6580,"category":"D","@timestamp":"2025-07-31T12:06:02"} +{"index":{"_id":"86"}} +{"timestamp":"2025-07-31T13:52:47","value":9102,"category":"B","@timestamp":"2025-07-31T13:52:47"} +{"index":{"_id":"87"}} +{"timestamp":"2025-07-31T14:39:32","value":7425,"category":"C","@timestamp":"2025-07-31T14:39:32"} +{"index":{"_id":"88"}} +{"timestamp":"2025-07-31T15:26:17","value":8753,"category":"A","@timestamp":"2025-07-31T15:26:17"} +{"index":{"_id":"89"}} +{"timestamp":"2025-07-31T16:13:10","value":6338,"category":"D","@timestamp":"2025-07-31T16:13:10"} +{"index":{"_id":"90"}} +{"timestamp":"2025-07-31T17:59:55","value":8909,"category":"B","@timestamp":"2025-07-31T17:59:55"} +{"index":{"_id":"91"}} +{"timestamp":"2025-07-31T18:46:40","value":7682,"category":"C","@timestamp":"2025-07-31T18:46:40"} +{"index":{"_id":"92"}} +{"timestamp":"2025-07-31T19:33:25","value":9231,"category":"A","@timestamp":"2025-07-31T19:33:25"} +{"index":{"_id":"93"}} +{"timestamp":"2025-07-31T20:20:18","value":6824,"category":"D","@timestamp":"2025-07-31T20:20:18"} +{"index":{"_id":"94"}} +{"timestamp":"2025-07-31T21:07:03","value":8490,"category":"B","@timestamp":"2025-07-31T21:07:03"} +{"index":{"_id":"95"}} +{"timestamp":"2025-07-31T22:53:48","value":7153,"category":"C","@timestamp":"2025-07-31T22:53:48"} +{"index":{"_id":"96"}} +{"timestamp":"2025-07-31T23:40:33","value":8676,"category":"A","@timestamp":"2025-07-31T23:40:33"} +{"index":{"_id":"97"}} +{"timestamp":"2025-08-01T00:27:26","value":6489,"category":"D"} +{"index":{"_id":"98"}} +{"timestamp":"2025-08-01T01:14:11","value":9015,"category":"B"} +{"index":{"_id":"99"}} +{"timestamp":"2025-08-01T02:00:56","value":7348,"category":"C"} +{"index":{"_id":"100"}} +{"timestamp":"2025-08-01T03:47:41","value":8762,"category":"A"} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4296.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4296.yml new file mode 100644 index 00000000000..fc65952cb1b --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4296.yml @@ -0,0 +1,58 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Fix bucket size wrong for limit after sort": + - skip: + features: + - headers + - allowed_warnings + - do: + bulk: + index: test + refresh: true + body: + - '{"index": {}}' + - '{ "title" : "document 1", "num": 1}' + - '{"index": {}}' + - '{ "title" : "document 2", "num": 1}' + - '{"index": {}}' + - '{ "title" : "document 3", "num": 1}' + - '{"index": {}}' + - '{ "title" : "document 4", "num": 1}' + - '{"index": {}}' + - '{ "title" : "document 5", "num": 2}' + - '{"index": {}}' + - '{ "title" : "document 6", "num": 2}' + - '{"index": {}}' + - '{ "title" : "document 7", "num": 3}' + - '{"index": {}}' + - '{ "title" : "document 8", "num": 3}' + - '{"index": {}}' + - '{ "title" : "document 9", "num": 4}' + - '{"index": {}}' + - '{ "title" : "document 10", "num": 4}' + - '{"index": {}}' + - '{ "title" : "document 11", "num": 5}' + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test | stats count() by num | sort -num | head 3 + - match: {"total": 3} diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4322.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4322.yml new file mode 100644 index 00000000000..9ac8aa67290 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4322.yml @@ -0,0 +1,69 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Fix bin with stats command exception": + - skip: + features: + - headers + - allowed_warnings + - do: + indices.create: + index: log-test + body: + mappings: + properties: + "@timestamp": + type: date + + - do: + bulk: + index: log-test + refresh: true + body: + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:15:00.000Z" }' + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:17:00.000Z" }' + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:20:00.000Z" }' + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:22:00.000Z" }' + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:25:00.000Z" }' + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-test | bin @timestamp span=5m | stats count by @timestamp + + - match: { total: 3 } + - length: { datarows: 3 } + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-test | stats count by span(@timestamp, 5m) + + - match: { total: 3 } + - length: { datarows: 3 } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4340.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4340.yml new file mode 100644 index 00000000000..eac85141363 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4340.yml @@ -0,0 +1,73 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"Fix bin with grouped stats on timestamp field": + - skip: + features: + - headers + - allowed_warnings + - do: + indices.create: + index: test-bin-stats + body: + mappings: + properties: + "@timestamp": + type: date + process: + properties: + name: + type: keyword + + - do: + bulk: + index: test-bin-stats + refresh: true + body: + - '{"index": {}}' + - '{ "@timestamp" : "2023-01-01T05:10:44.000Z", "process": { "name": "sshd" } }' + - '{"index": {}}' + - '{ "@timestamp" : "2023-01-01T07:18:55.000Z", "process": { "name": "cron" } }' + - '{"index": {}}' + - '{ "@timestamp" : "2023-01-01T04:33:24.000Z", "process": { "name": "kernel" } }' + - '{"index": {}}' + - '{ "@timestamp" : "2022-12-31T18:49:28.000Z", "process": { "name": "cron" } }' + - '{"index": {}}' + - '{ "@timestamp" : "2023-01-01T10:23:39.000Z", "process": { "name": "cron" } }' + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test-bin-stats | bin `@timestamp` span=1d | stats count() by `@timestamp` + + - match: { total: 2 } + - length: { datarows: 2 } + + - do: + allowed_warnings: + - 'Loading the fielddata on the _id field is deprecated and will be removed in future versions. If you require sorting or aggregating on this field you should also include the id in the body of your documents, and map this field as a keyword field that has [doc_values] enabled' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=test-bin-stats | bin `@timestamp` span=1d | stats count() by `@timestamp`, process.name | sort `@timestamp`, process.name + + - match: { total: 4 } + - length: { datarows: 4 } diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml new file mode 100644 index 00000000000..a3189e7588a --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/4342.yml @@ -0,0 +1,55 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : true + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"No index found with given index pattern should throw IndexNotFoundException": + - skip: + features: + - headers + - allowed_warnings + - do: + indices.create: + index: log-test + body: + mappings: + properties: + "@timestamp": + type: date + + - do: + bulk: + index: log-test + refresh: true + body: + - '{"index": {}}' + - '{ "@timestamp" : "2025-09-04T16:15:00.000Z" }' + + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-* | fields @timestamp + + - match: { total: 1 } + - length: { datarows: 1 } + + - do: + catch: missing # without the patch #4342, it throws 'bad_request' rather 'missing' + headers: + Content-Type: 'application/json' + ppl: + body: + query: source=log-abc* | fields @timestamp diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 index b7dc4b7286d..4c37be2f318 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLLexer.g4 @@ -13,9 +13,12 @@ options { caseInsensitive = true; } SEARCH: 'SEARCH'; DESCRIBE: 'DESCRIBE'; SHOW: 'SHOW'; +EXPLAIN: 'EXPLAIN'; FROM: 'FROM'; WHERE: 'WHERE'; FIELDS: 'FIELDS'; +FIELD: 'FIELD'; +TABLE: 'TABLE'; // Alias for FIELDS command RENAME: 'RENAME'; STATS: 'STATS'; EVENTSTATS: 'EVENTSTATS'; @@ -23,13 +26,14 @@ DEDUP: 'DEDUP'; SORT: 'SORT'; EVAL: 'EVAL'; HEAD: 'HEAD'; -TOP_APPROX: 'TOP_APPROX'; +BIN: 'BIN'; TOP: 'TOP'; -RARE_APPROX: 'RARE_APPROX'; RARE: 'RARE'; PARSE: 'PARSE'; -METHOD: 'METHOD'; +SPATH: 'SPATH'; REGEX: 'REGEX'; +REX: 'REX'; +SED: 'SED'; PUNCT: 'PUNCT'; GROK: 'GROK'; PATTERN: 'PATTERN'; @@ -39,10 +43,22 @@ KMEANS: 'KMEANS'; AD: 'AD'; ML: 'ML'; FILLNULL: 'FILLNULL'; -EXPAND: 'EXPAND'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; +TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; +EXPAND: 'EXPAND'; +SIMPLE_PATTERN: 'SIMPLE_PATTERN'; +BRAIN: 'BRAIN'; +VARIABLE_COUNT_THRESHOLD: 'VARIABLE_COUNT_THRESHOLD'; +FREQUENCY_THRESHOLD_PERCENTAGE: 'FREQUENCY_THRESHOLD_PERCENTAGE'; +METHOD: 'METHOD'; +MAX_SAMPLE_COUNT: 'MAX_SAMPLE_COUNT'; +MAX_MATCH: 'MAX_MATCH'; +OFFSET_FIELD: 'OFFSET_FIELD'; +BUFFER_LIMIT: 'BUFFER_LIMIT'; +LABEL: 'LABEL'; +AGGREGATION: 'AGGREGATION'; //Native JOIN KEYWORDS JOIN: 'JOIN'; @@ -56,52 +72,35 @@ CROSS: 'CROSS'; LEFT_HINT: 'HINT.LEFT'; RIGHT_HINT: 'HINT.RIGHT'; -//CORRELATION KEYWORDS -CORRELATE: 'CORRELATE'; -SELF: 'SELF'; -EXACT: 'EXACT'; -APPROXIMATE: 'APPROXIMATE'; -SCOPE: 'SCOPE'; -MAPPING: 'MAPPING'; - -//EXPLAIN KEYWORDS -EXPLAIN: 'EXPLAIN'; -FORMATTED: 'FORMATTED'; -COST: 'COST'; -CODEGEN: 'CODEGEN'; -EXTENDED: 'EXTENDED'; -SIMPLE: 'SIMPLE'; - // COMMAND ASSIST KEYWORDS AS: 'AS'; BY: 'BY'; SOURCE: 'SOURCE'; INDEX: 'INDEX'; +A: 'A'; +ASC: 'ASC'; D: 'D'; DESC: 'DESC'; DATASOURCES: 'DATASOURCES'; USING: 'USING'; WITH: 'WITH'; +SIMPLE: 'SIMPLE'; +STANDARD: 'STANDARD'; +COST: 'COST'; +EXTENDED: 'EXTENDED'; +OVERRIDE: 'OVERRIDE'; +OVERWRITE: 'OVERWRITE'; // SORT FIELD KEYWORDS -// TODO #963: Implement 'num', 'str', and 'ip' sort syntax +// TODO #3180: Fix broken sort functionality AUTO: 'AUTO'; STR: 'STR'; -IP: 'IP'; NUM: 'NUM'; -// FIELDSUMMARY keywords -FIELDSUMMARY: 'FIELDSUMMARY'; -INCLUDEFIELDS: 'INCLUDEFIELDS'; -NULLS: 'NULLS'; - -//TRENDLINE KEYWORDS +// TRENDLINE KEYWORDS SMA: 'SMA'; WMA: 'WMA'; -// APPENDCOL options -OVERRIDE: 'OVERRIDE'; - // ARGUMENT KEYWORDS KEEPEMPTY: 'KEEPEMPTY'; CONSECUTIVE: 'CONSECUTIVE'; @@ -109,6 +108,7 @@ DEDUP_SPLITVALUES: 'DEDUP_SPLITVALUES'; PARTITIONS: 'PARTITIONS'; ALLNUM: 'ALLNUM'; DELIM: 'DELIM'; +BUCKET_NULLABLE: 'BUCKET_NULLABLE'; CENTROIDS: 'CENTROIDS'; ITERATIONS: 'ITERATIONS'; DISTANCE_TYPE: 'DISTANCE_TYPE'; @@ -124,6 +124,13 @@ TIME_ZONE: 'TIME_ZONE'; TRAINING_DATA_SIZE: 'TRAINING_DATA_SIZE'; ANOMALY_SCORE_THRESHOLD: 'ANOMALY_SCORE_THRESHOLD'; APPEND: 'APPEND'; +COUNTFIELD: 'COUNTFIELD'; +SHOWCOUNT: 'SHOWCOUNT'; +LIMIT: 'LIMIT'; +USEOTHER: 'USEOTHER'; +INPUT: 'INPUT'; +OUTPUT: 'OUTPUT'; +PATH: 'PATH'; // COMPARISON FUNCTION KEYWORDS CASE: 'CASE'; @@ -131,6 +138,9 @@ ELSE: 'ELSE'; IN: 'IN'; EXISTS: 'EXISTS'; +// Geo IP eval function +GEOIP: 'GEOIP'; + // LOGICAL KEYWORDS NOT: 'NOT'; OR: 'OR'; @@ -139,6 +149,7 @@ XOR: 'XOR'; TRUE: 'TRUE'; FALSE: 'FALSE'; REGEXP: 'REGEXP'; +REGEX_MATCH: 'REGEX_MATCH'; // DATETIME, INTERVAL AND UNIT KEYWORDS CONVERT_TZ: 'CONVERT_TZ'; @@ -186,12 +197,14 @@ LONG: 'LONG'; FLOAT: 'FLOAT'; STRING: 'STRING'; BOOLEAN: 'BOOLEAN'; +IP: 'IP'; // SPECIAL CHARACTERS AND OPERATORS PIPE: '|'; COMMA: ','; DOT: '.'; EQUAL: '='; +DOUBLE_EQUAL: '=='; GREATER: '>'; LESS: '<'; NOT_GREATER: '<' '='; @@ -208,6 +221,8 @@ LT_PRTHS: '('; RT_PRTHS: ')'; LT_SQR_PRTHS: '['; RT_SQR_PRTHS: ']'; +LT_CURLY: '{'; +RT_CURLY: '}'; SINGLE_QUOTE: '\''; DOUBLE_QUOTE: '"'; BACKTICK: '`'; @@ -240,11 +255,12 @@ VAR_SAMP: 'VAR_SAMP'; VAR_POP: 'VAR_POP'; STDDEV_SAMP: 'STDDEV_SAMP'; STDDEV_POP: 'STDDEV_POP'; +PERC: 'PERC'; PERCENTILE: 'PERCENTILE'; PERCENTILE_APPROX: 'PERCENTILE_APPROX'; +EARLIEST: 'EARLIEST'; +LATEST: 'LATEST'; TAKE: 'TAKE'; -FIRST: 'FIRST'; -LAST: 'LAST'; LIST: 'LIST'; VALUES: 'VALUES'; PER_DAY: 'PER_DAY'; @@ -256,7 +272,22 @@ SPARKLINE: 'SPARKLINE'; C: 'C'; DC: 'DC'; +// SCALAR WINDOW FUNCTIONS +ROW_NUMBER: 'ROW_NUMBER'; +RANK: 'RANK'; +DENSE_RANK: 'DENSE_RANK'; +PERCENT_RANK: 'PERCENT_RANK'; +CUME_DIST: 'CUME_DIST'; +FIRST: 'FIRST'; +LAST: 'LAST'; +NTH: 'NTH'; +NTILE: 'NTILE'; + // BASIC FUNCTIONS +PLUS_FUCTION: 'ADD'; +MINUS_FUCTION: 'SUBTRACT'; +STAR_FUNCTION: 'MULTIPLY'; +DIVIDE_FUNCTION: 'DIVIDE'; ABS: 'ABS'; CBRT: 'CBRT'; CEIL: 'CEIL'; @@ -265,12 +296,13 @@ CONV: 'CONV'; CRC32: 'CRC32'; E: 'E'; EXP: 'EXP'; +EXPM1: 'EXPM1'; FLOOR: 'FLOOR'; LN: 'LN'; LOG: 'LOG'; -LOG10: 'LOG10'; -LOG2: 'LOG2'; +LOG_WITH_BASE: ([0-9]+ ('.' [0-9]+)?)? ('LOG' | 'log') [0-9]+ ('.' [0-9]+)?; MOD: 'MOD'; +MODULUS: 'MODULUS'; PI: 'PI'; POSITION: 'POSITION'; POW: 'POW'; @@ -278,9 +310,10 @@ POWER: 'POWER'; RAND: 'RAND'; ROUND: 'ROUND'; SIGN: 'SIGN'; -SIGNUM: 'SIGNUM'; SQRT: 'SQRT'; TRUNCATE: 'TRUNCATE'; +RINT: 'RINT'; +SIGNUM: 'SIGNUM'; // TRIGONOMETRIC FUNCTIONS ACOS: 'ACOS'; @@ -288,10 +321,12 @@ ASIN: 'ASIN'; ATAN: 'ATAN'; ATAN2: 'ATAN2'; COS: 'COS'; +COSH: 'COSH'; COT: 'COT'; DEGREES: 'DEGREES'; RADIANS: 'RADIANS'; SIN: 'SIN'; +SINH: 'SINH'; TAN: 'TAN'; // CRYPTOGRAPHIC FUNCTIONS @@ -306,7 +341,6 @@ CURDATE: 'CURDATE'; CURRENT_DATE: 'CURRENT_DATE'; CURRENT_TIME: 'CURRENT_TIME'; CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP'; -CURRENT_TIMEZONE: 'CURRENT_TIMEZONE'; CURTIME: 'CURTIME'; DATE: 'DATE'; DATEDIFF: 'DATEDIFF'; @@ -319,7 +353,6 @@ DAYOFWEEK: 'DAYOFWEEK'; DAYOFYEAR: 'DAYOFYEAR'; DAY_OF_MONTH: 'DAY_OF_MONTH'; DAY_OF_WEEK: 'DAY_OF_WEEK'; -DURATION: 'DURATION'; EXTRACT: 'EXTRACT'; FROM_DAYS: 'FROM_DAYS'; FROM_UNIXTIME: 'FROM_UNIXTIME'; @@ -328,7 +361,6 @@ LAST_DAY: 'LAST_DAY'; LOCALTIME: 'LOCALTIME'; LOCALTIMESTAMP: 'LOCALTIMESTAMP'; MAKEDATE: 'MAKEDATE'; -MAKE_DATE: 'MAKE_DATE'; MAKETIME: 'MAKETIME'; MONTHNAME: 'MONTHNAME'; NOW: 'NOW'; @@ -355,11 +387,6 @@ UTC_TIMESTAMP: 'UTC_TIMESTAMP'; WEEKDAY: 'WEEKDAY'; YEARWEEK: 'YEARWEEK'; -// RELATIVE TIME FUNCTIONS -RELATIVE_TIMESTAMP: 'RELATIVE_TIMESTAMP'; -EARLIEST: 'EARLIEST'; -LATEST: 'LATEST'; - // TEXT FUNCTIONS SUBSTR: 'SUBSTR'; SUBSTRING: 'SUBSTRING'; @@ -381,67 +408,45 @@ REPLACE: 'REPLACE'; REVERSE: 'REVERSE'; CAST: 'CAST'; -// JSON TEXT FUNCTIONS -JSON: 'JSON'; -JSON_OBJECT: 'JSON_OBJECT'; -JSON_ARRAY: 'JSON_ARRAY'; -JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; -TO_JSON_STRING: 'TO_JSON_STRING'; -JSON_EXTRACT: 'JSON_EXTRACT'; -JSON_DELETE : 'JSON_DELETE'; -JSON_KEYS: 'JSON_KEYS'; -JSON_VALID: 'JSON_VALID'; -JSON_APPEND: 'JSON_APPEND'; -JSON_EXTEND : 'JSON_EXTEND'; -JSON_SET: 'JSON_SET'; -//JSON_ARRAY_ALL_MATCH: 'JSON_ARRAY_ALL_MATCH'; -//JSON_ARRAY_ANY_MATCH: 'JSON_ARRAY_ANY_MATCH'; -//JSON_ARRAY_FILTER: 'JSON_ARRAY_FILTER'; -//JSON_ARRAY_MAP: 'JSON_ARRAY_MAP'; -//JSON_ARRAY_REDUCE: 'JSON_ARRAY_REDUCE'; +// BOOL FUNCTIONS +LIKE: 'LIKE'; +ISNULL: 'ISNULL'; +ISNOTNULL: 'ISNOTNULL'; +CIDRMATCH: 'CIDRMATCH'; +BETWEEN: 'BETWEEN'; +ISPRESENT: 'ISPRESENT'; +ISEMPTY: 'ISEMPTY'; +ISBLANK: 'ISBLANK'; // COLLECTION FUNCTIONS ARRAY: 'ARRAY'; ARRAY_LENGTH: 'ARRAY_LENGTH'; - -// LAMBDA FUNCTIONS -//EXISTS: 'EXISTS'; +MVJOIN: 'MVJOIN'; FORALL: 'FORALL'; FILTER: 'FILTER'; TRANSFORM: 'TRANSFORM'; REDUCE: 'REDUCE'; -// BOOL FUNCTIONS -LIKE: 'LIKE'; -ISNULL: 'ISNULL'; -ISNOTNULL: 'ISNOTNULL'; -BETWEEN: 'BETWEEN'; -CIDRMATCH: 'CIDRMATCH'; -ISPRESENT: 'ISPRESENT'; -ISEMPTY: 'ISEMPTY'; -ISBLANK: 'ISBLANK'; +// JSON FUNCTIONS +JSON_VALID: 'JSON_VALID'; +JSON: 'JSON'; +JSON_OBJECT: 'JSON_OBJECT'; +JSON_ARRAY: 'JSON_ARRAY'; +JSON_ARRAY_LENGTH: 'JSON_ARRAY_LENGTH'; +JSON_EXTRACT: 'JSON_EXTRACT'; +JSON_KEYS: 'JSON_KEYS'; +JSON_SET: 'JSON_SET'; +JSON_DELETE: 'JSON_DELETE'; +JSON_APPEND: 'JSON_APPEND'; +JSON_EXTEND: 'JSON_EXTEND'; // FLOWCONTROL FUNCTIONS IFNULL: 'IFNULL'; NULLIF: 'NULLIF'; IF: 'IF'; TYPEOF: 'TYPEOF'; - -//OTHER CONDITIONAL EXPRESSIONS COALESCE: 'COALESCE'; -//GEOLOCATION FUNCTIONS -GEOIP: 'GEOIP'; - -//GEOLOCATION PROPERTIES -COUNTRY_ISO_CODE: 'COUNTRY_ISO_CODE'; -COUNTRY_NAME: 'COUNTRY_NAME'; -CONTINENT_NAME: 'CONTINENT_NAME'; -REGION_ISO_CODE: 'REGION_ISO_CODE'; -REGION_NAME: 'REGION_NAME'; -CITY_NAME: 'CITY_NAME'; -LOCATION: 'LOCATION'; - // RELEVANCE FUNCTIONS AND PARAMETERS MATCH: 'MATCH'; MATCH_PHRASE: 'MATCH_PHRASE'; @@ -485,6 +490,11 @@ ZERO_TERMS_QUERY: 'ZERO_TERMS_QUERY'; // SPAN KEYWORDS SPAN: 'SPAN'; +BINS: 'BINS'; +MINSPAN: 'MINSPAN'; +START: 'START'; +END: 'END'; +ALIGNTIME: 'ALIGNTIME'; MS: 'MS'; S: 'S'; M: 'M'; @@ -493,6 +503,26 @@ W: 'W'; Q: 'Q'; Y: 'Y'; +// Extended timescale units +SEC: 'SEC'; +SECS: 'SECS'; +SECONDS: 'SECONDS'; +MINS: 'MINS'; +MINUTES: 'MINUTES'; +HR: 'HR'; +HRS: 'HRS'; +HOURS: 'HOURS'; +DAYS: 'DAYS'; +MON: 'MON'; +MONTHS: 'MONTHS'; +US: 'US'; +CS: 'CS'; +DS: 'DS'; + + +// PERCENTILE SHORTCUT FUNCTIONS +// Must precede ID to avoid conflicts with identifier matching +PERCENTILE_SHORTCUT: PERC(INTEGER_LITERAL | DECIMAL_LITERAL) | 'P'(INTEGER_LITERAL | DECIMAL_LITERAL); // LITERALS AND VALUES //STRING_LITERAL: DQUOTA_STRING | SQUOTA_STRING | BQUOTA_STRING; @@ -500,9 +530,10 @@ ID: ID_LITERAL; CLUSTER: CLUSTER_PREFIX_LITERAL; INTEGER_LITERAL: DEC_DIGIT+; DECIMAL_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+; +FLOAT_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ 'F'; +DOUBLE_LITERAL: (DEC_DIGIT+)? '.' DEC_DIGIT+ 'D'; fragment DATE_SUFFIX: ([\-.][*0-9]+)+; -fragment ID_LITERAL: [@*A-Z]+?[*A-Z_\-0-9]*; fragment CLUSTER_PREFIX_LITERAL: [*A-Z]+?[*A-Z_\-0-9]* COLON; ID_DATE_SUFFIX: CLUSTER_PREFIX_LITERAL? ID_LITERAL DATE_SUFFIX; DQUOTA_STRING: '"' ( '\\'. | '""' | ~('"'| '\\') )* '"'; @@ -510,6 +541,10 @@ SQUOTA_STRING: '\'' ('\\'. | '\'\'' | ~('\'' | '\\'))* '\'' BQUOTA_STRING: '`' ( '\\'. | '``' | ~('`'|'\\'))* '`'; fragment DEC_DIGIT: [0-9]; +// Identifiers cannot start with a single '_' since this an OpenSearch reserved +// metadata field. Two underscores (or more) is acceptable, such as '__field'. +fragment ID_LITERAL: ([@*A-Z_])+?[*A-Z_\-0-9]*; + LINE_COMMENT: '//' ('\\\n' | ~[\r\n])* '\r'? '\n'? -> channel(HIDDEN); BLOCK_COMMENT: '/*' .*? '*/' -> channel(HIDDEN); diff --git a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 index cae57b53181..d5cb4e3452b 100644 --- a/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 +++ b/language-grammar/src/main/antlr4/OpenSearchPPLParser.g4 @@ -3,168 +3,233 @@ * SPDX-License-Identifier: Apache-2.0 */ + parser grammar OpenSearchPPLParser; options { tokenVocab = OpenSearchPPLLexer; } + root : pplStatement? EOF ; // statement pplStatement - : dmlStatement - ; - -dmlStatement - : (explainCommand PIPE)? queryStatement + : explainStatement + | queryStatement ; queryStatement : pplCommands (PIPE commands)* ; +explainStatement + : EXPLAIN (explainMode)? queryStatement + ; + +explainMode + : SIMPLE + | STANDARD + | COST + | EXTENDED + ; + subSearch : searchCommand (PIPE commands)* ; // commands pplCommands - : searchCommand - | describeCommand + : describeCommand + | showDataSourcesCommand + | searchCommand ; commands : whereCommand - | correlateCommand - | joinCommand | fieldsCommand + | tableCommand + | joinCommand + | renameCommand | statsCommand + | eventstatsCommand | dedupCommand | sortCommand + | evalCommand | headCommand + | binCommand | topCommand | rareCommand - | evalCommand | grokCommand | parseCommand + | spathCommand | patternsCommand | lookupCommand - | renameCommand + | kmeansCommand + | adCommand + | mlCommand | fillnullCommand - | fieldsummaryCommand - | flattenCommand - | expandCommand | trendlineCommand | appendcolCommand + | appendCommand + | expandCommand + | flattenCommand + | reverseCommand + | regexCommand + | timechartCommand + | rexCommand ; commandName : SEARCH | DESCRIBE | SHOW - | AD - | ML - | KMEANS | WHERE - | CORRELATE - | JOIN | FIELDS + | TABLE + | JOIN + | RENAME | STATS | EVENTSTATS | DEDUP - | EXPLAIN | SORT + | EVAL | HEAD + | BIN | TOP - | TOP_APPROX | RARE - | RARE_APPROX - | EVAL | GROK | PARSE | PATTERNS | LOOKUP - | RENAME - | EXPAND + | KMEANS + | AD + | ML | FILLNULL - | FIELDSUMMARY + | EXPAND | FLATTEN | TRENDLINE - | APPENDCOL + | TIMECHART + | EXPLAIN + | REVERSE + | REGEX + | APPEND + | REX ; searchCommand - : (SEARCH)? fromClause # searchFrom - | (SEARCH)? fromClause logicalExpression # searchFromFilter - | (SEARCH)? logicalExpression fromClause # searchFilterFrom + : (SEARCH)? (searchExpression)* fromClause (searchExpression)* # searchFrom + ; + +searchExpression + : LT_PRTHS searchExpression RT_PRTHS # groupedExpression + | NOT searchExpression # notExpression + | searchExpression OR searchExpression # orExpression + | searchExpression AND searchExpression # andExpression + | searchTerm # termExpression + ; + +searchTerm + : searchFieldComparison # searchComparisonTerm + | searchFieldInList # searchInListTerm + | searchLiteral # searchLiteralTerm + ; + +// Unified search literal for both free text and field comparisons +searchLiteral + : numericLiteral + | booleanLiteral + | ID + | stringLiteral + | searchableKeyWord ; -fieldsummaryCommand - : FIELDSUMMARY (fieldsummaryParameter)* - ; +searchFieldComparison + : fieldExpression searchComparisonOperator searchLiteral # searchFieldCompare + ; + +searchFieldInList + : fieldExpression IN LT_PRTHS searchLiteralList RT_PRTHS # searchFieldInValues + ; + +searchLiteralList + : searchLiteral (COMMA searchLiteral)* # searchLiterals + ; + +searchComparisonOperator + : EQUAL # equals + | NOT_EQUAL # notEquals + | LESS # lessThan + | NOT_GREATER # lessOrEqual + | GREATER # greaterThan + | NOT_LESS # greaterOrEqual + ; -fieldsummaryParameter - : INCLUDEFIELDS EQUAL fieldList # fieldsummaryIncludeFields - | NULLS EQUAL booleanLiteral # fieldsummaryNulls - ; describeCommand : DESCRIBE tableSourceClause ; -explainCommand - : EXPLAIN explainMode - ; - -explainMode - : FORMATTED - | COST - | CODEGEN - | EXTENDED - | SIMPLE - ; - showDataSourcesCommand - : SHOW DATASOURCES - ; + : SHOW DATASOURCES + ; whereCommand - : WHERE logicalExpression - ; - -correlateCommand - : CORRELATE correlationType FIELDS LT_PRTHS fieldList RT_PRTHS (scopeClause)? mappingList - ; - -correlationType - : SELF - | EXACT - | APPROXIMATE - ; + : WHERE logicalExpression + ; -scopeClause - : SCOPE LT_PRTHS fieldExpression COMMA value = literalValue (unit = timespanUnit)? RT_PRTHS - ; +fieldsCommand + : FIELDS fieldsCommandBody + ; -mappingList - : MAPPING LT_PRTHS ( mappingClause (COMMA mappingClause)* ) RT_PRTHS - ; +// Table command - alias for fields command +tableCommand + : TABLE fieldsCommandBody + ; -mappingClause - : left = qualifiedName comparisonOperator right = qualifiedName # mappingCompareExpr - ; +fieldsCommandBody + : (PLUS | MINUS)? wcFieldList + ; -fieldsCommand - : FIELDS (PLUS | MINUS)? fieldList +// Wildcard field list supporting both comma-separated and space-separated fields +wcFieldList + : selectFieldExpression (COMMA? selectFieldExpression)* ; renameCommand - : RENAME renameClasue (COMMA renameClasue)* + : RENAME renameClasue (COMMA? renameClasue)* ; statsCommand - : (STATS | EVENTSTATS) (PARTITIONS EQUAL partitions = integerLiteral)? (ALLNUM EQUAL allnum = booleanLiteral)? (DELIM EQUAL delim = stringLiteral)? statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral)? + : STATS statsArgs statsAggTerm (COMMA statsAggTerm)* (statsByClause)? (dedupSplitArg)? + ; + +statsArgs + : (partitionsArg | allnumArg | delimArg | bucketNullableArg)* + ; + +partitionsArg + : PARTITIONS EQUAL partitions = integerLiteral + ; + +allnumArg + : ALLNUM EQUAL allnum = booleanLiteral + ; + +delimArg + : DELIM EQUAL delim = stringLiteral + ; + +bucketNullableArg + : BUCKET_NULLABLE EQUAL bucket_nullable = booleanLiteral + ; + +dedupSplitArg + : DEDUP_SPLITVALUES EQUAL dedupsplit = booleanLiteral + ; + +eventstatsCommand + : EVENTSTATS eventstatsAggTerm (COMMA eventstatsAggTerm)* (statsByClause)? ; dedupCommand @@ -172,7 +237,30 @@ dedupCommand ; sortCommand - : SORT sortbyClause + : SORT (count = integerLiteral)? sortbyClause (ASC | A | DESC | D)? + ; + +reverseCommand + : REVERSE + ; + +timechartCommand + : TIMECHART timechartParameter* statsFunction (BY fieldExpression)? + ; + +timechartParameter + : (spanClause | SPAN EQUAL spanLiteral) + | timechartArg + ; + +timechartArg + : LIMIT EQUAL integerLiteral + | USEOTHER EQUAL (booleanLiteral | ident) + ; + +spanLiteral + : integerLiteral timespanUnit + | stringLiteral ; evalCommand @@ -183,12 +271,42 @@ headCommand : HEAD (number = integerLiteral)? (FROM from = integerLiteral)? ; +binCommand + : BIN fieldExpression binOption* (AS alias = qualifiedName)? + ; + +binOption + : SPAN EQUAL span = spanValue + | BINS EQUAL bins = integerLiteral + | MINSPAN EQUAL minspan = literalValue (minspanUnit = timespanUnit)? + | ALIGNTIME EQUAL aligntime = aligntimeValue + | START EQUAL start = numericLiteral + | END EQUAL end = numericLiteral + ; + +aligntimeValue + : EARLIEST + | LATEST + | literalValue + ; + +spanValue + : literalValue (timespanUnit)? # numericSpanValue + | logSpanValue # logBasedSpanValue + | ident timespanUnit # extendedTimeSpanValue + | ident # identifierSpanValue + ; + +logSpanValue + : LOG_WITH_BASE # logWithBaseSpan + ; + topCommand - : (TOP | TOP_APPROX) (number = integerLiteral)? fieldList (byClause)? + : TOP (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? ; rareCommand - : (RARE | RARE_APPROX) (number = integerLiteral)? fieldList (byClause)? + : RARE (number = integerLiteral)? (COUNTFIELD EQUAL countfield = stringLiteral)? (SHOWCOUNT EQUAL showcount = booleanLiteral)? fieldList (byClause)? ; grokCommand @@ -199,20 +317,73 @@ parseCommand : PARSE (source_field = expression) (pattern = stringLiteral) ; -patternsCommand - : PATTERNS (patternsParameter)* (source_field = expression) +spathCommand + : SPATH spathParameter* ; -patternsParameter - : (NEW_FIELD EQUAL new_field = stringLiteral) - | (PATTERN EQUAL pattern = stringLiteral) +spathParameter + : (INPUT EQUAL input = expression) + | (OUTPUT EQUAL output = expression) + | ((PATH EQUAL)? path = indexablePath) ; +indexablePath + : pathElement (DOT pathElement)* + ; + +pathElement + : ident pathArrayAccess? + ; + +pathArrayAccess + : LT_CURLY (INTEGER_LITERAL)? RT_CURLY + ; +regexCommand + : REGEX regexExpr + ; + +regexExpr + : field=qualifiedName operator=(EQUAL | NOT_EQUAL) pattern=stringLiteral + ; + +rexCommand + : REX rexExpr + ; + +rexExpr + : FIELD EQUAL field=qualifiedName (rexOption)* pattern=stringLiteral (rexOption)* + ; + +rexOption + : MAX_MATCH EQUAL maxMatch=integerLiteral + | MODE EQUAL (EXTRACT | SED) + | OFFSET_FIELD EQUAL offsetField=qualifiedName + ; patternsMethod : PUNCT | REGEX ; +patternsCommand + : PATTERNS (source_field = expression) (statsByClause)? (METHOD EQUAL method = patternMethod)? (MODE EQUAL pattern_mode = patternMode)? (MAX_SAMPLE_COUNT EQUAL max_sample_count = integerLiteral)? (BUFFER_LIMIT EQUAL buffer_limit = integerLiteral)? (NEW_FIELD EQUAL new_field = stringLiteral)? (patternsParameter)* + ; + +patternsParameter + : (PATTERN EQUAL pattern = stringLiteral) + | (VARIABLE_COUNT_THRESHOLD EQUAL variable_count_threshold = integerLiteral) + | (FREQUENCY_THRESHOLD_PERCENTAGE EQUAL frequency_threshold_percentage = decimalLiteral) + ; + +patternMethod + : SIMPLE_PATTERN + | BRAIN + ; + +patternMode + : LABEL + | AGGREGATION + ; + // lookup lookupCommand : LOOKUP tableSource lookupMappingList ((APPEND | REPLACE) outputCandidateList)? @@ -235,36 +406,28 @@ lookupPair ; fillnullCommand - : FILLNULL (fillNullWithTheSameValue - | fillNullWithFieldVariousValues) + : FILLNULL fillNullWith + | FILLNULL fillNullUsing ; -fillNullWithTheSameValue - : WITH nullReplacement = valueExpression IN nullableFieldList = fieldList +fillNullWith + : WITH replacement = valueExpression (IN fieldList)? ; -fillNullWithFieldVariousValues - : USING nullableReplacementExpression (COMMA nullableReplacementExpression)* +fillNullUsing + : USING replacementPair (COMMA replacementPair)* ; -nullableReplacementExpression - : nullableField = fieldExpression EQUAL nullableReplacement = valueExpression +replacementPair + : fieldExpression EQUAL replacement = valueExpression ; -expandCommand - : EXPAND fieldExpression (AS alias = qualifiedName)? - ; - -flattenCommand - : FLATTEN fieldExpression (AS alias = identifierSeq)? - ; - trendlineCommand : TRENDLINE (SORT sortField)? trendlineClause (trendlineClause)* ; trendlineClause - : trendlineType LT_PRTHS numberOfDataPoints = INTEGER_LITERAL COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? + : trendlineType LT_PRTHS numberOfDataPoints = integerLiteral COMMA field = fieldExpression RT_PRTHS (AS alias = qualifiedName)? ; trendlineType @@ -272,10 +435,22 @@ trendlineType | WMA ; +expandCommand + : EXPAND fieldExpression (AS alias = qualifiedName)? + ; + +flattenCommand + : FLATTEN fieldExpression (AS aliases = identifierSeq)? + ; + appendcolCommand : APPENDCOL (OVERRIDE EQUAL override = booleanLiteral)? LT_SQR_PRTHS commands (PIPE commands)* RT_SQR_PRTHS ; +appendCommand + : APPEND LT_SQR_PRTHS searchCommand? (PIPE commands)* RT_SQR_PRTHS + ; + kmeansCommand : KMEANS (kmeansParameter)* ; @@ -317,6 +492,10 @@ mlArg fromClause : SOURCE EQUAL tableOrSubqueryClause | INDEX EQUAL tableOrSubqueryClause + | SOURCE EQUAL tableFunction + | INDEX EQUAL tableFunction + | SOURCE EQUAL dynamicSourceClause + | INDEX EQUAL dynamicSourceClause ; tableOrSubqueryClause @@ -324,36 +503,64 @@ tableOrSubqueryClause | tableSourceClause ; -// One tableSourceClause will generate one Relation node with/without one alias -// even if the relation contains more than one table sources. -// These table sources in one relation will be readed one by one in OpenSearch. -// But it may have different behaivours in different execution backends. -// For example, a Spark UnresovledRelation node only accepts one data source. tableSourceClause : tableSource (COMMA tableSource)* (AS alias = qualifiedName)? ; +dynamicSourceClause + : LT_SQR_PRTHS sourceReferences (COMMA sourceFilterArgs)? RT_SQR_PRTHS + ; + +sourceReferences + : sourceReference (COMMA sourceReference)* + ; + +sourceReference + : (CLUSTER)? wcQualifiedName + ; + +sourceFilterArgs + : sourceFilterArg (COMMA sourceFilterArg)* + ; + +sourceFilterArg + : ident EQUAL literalValue + | ident IN valueList + ; + // join joinCommand - : (joinType) JOIN sideAlias joinHintList? joinCriteria? right = tableOrSubqueryClause + : JOIN (joinOption)* (fieldList)? right = tableOrSubqueryClause + | sqlLikeJoinType? JOIN (joinOption)* sideAlias joinHintList? joinCriteria right = tableOrSubqueryClause ; -joinType - : INNER? +sqlLikeJoinType + : INNER | CROSS - | LEFT OUTER? + | (LEFT OUTER? | OUTER) | RIGHT OUTER? | FULL OUTER? | LEFT? SEMI | LEFT? ANTI ; +joinType + : INNER + | CROSS + | OUTER + | LEFT + | RIGHT + | FULL + | SEMI + | ANTI + ; + sideAlias : (LEFT EQUAL leftAlias = qualifiedName)? COMMA? (RIGHT EQUAL rightAlias = qualifiedName)? ; joinCriteria - : ON logicalExpression + : (ON | WHERE) logicalExpression ; joinHintList @@ -365,8 +572,14 @@ hintPair | rightHintKey = RIGHT_HINT DOT ID EQUAL rightHintValue = ident #rightHint ; +joinOption + : OVERWRITE EQUAL booleanLiteral # overwriteOption + | TYPE EQUAL joinType # typeOption + | MAX EQUAL integerLiteral # maxOption + ; + renameClasue - : orignalField = wcFieldExpression AS renamedField = wcFieldExpression + : orignalField = renameFieldExpression AS renamedField = renameFieldExpression ; byClause @@ -377,6 +590,7 @@ statsByClause : BY fieldList | BY bySpanClause | BY bySpanClause COMMA fieldList + | BY fieldList COMMA bySpanClause ; bySpanClause @@ -392,12 +606,34 @@ sortbyClause ; evalClause - : fieldExpression EQUAL expression - | geoipCommand + : fieldExpression EQUAL logicalExpression ; -geoipCommand - : fieldExpression EQUAL GEOIP LT_PRTHS ipAddress = functionArg (COMMA properties = geoIpPropertyList)? RT_PRTHS +eventstatsAggTerm + : windowFunction (AS alias = wcFieldExpression)? + ; + +windowFunction + : windowFunctionName LT_PRTHS functionArgs RT_PRTHS + ; + +windowFunctionName + : statsFunctionName + | scalarWindowFunctionName + ; + +scalarWindowFunctionName + : ROW_NUMBER + | RANK + | DENSE_RANK + | PERCENT_RANK + | CUME_DIST + | FIRST + | LAST + | NTH + | NTILE + | DISTINCT_COUNT + | DC ; // aggregation terms @@ -407,10 +643,13 @@ statsAggTerm // aggregation functions statsFunction - : statsFunctionName LT_PRTHS valueExpression RT_PRTHS # statsFunctionCall - | COUNT LT_PRTHS RT_PRTHS # countAllFunctionCall - | (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall - | percentileFunctionName = (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS valueExpression COMMA percent = integerLiteral RT_PRTHS # percentileFunctionCall + : (COUNT | C) LT_PRTHS evalExpression RT_PRTHS # countEvalFunctionCall + | (COUNT | C) (LT_PRTHS RT_PRTHS)? # countAllFunctionCall + | PERCENTILE_SHORTCUT LT_PRTHS valueExpression RT_PRTHS # percentileShortcutFunctionCall + | (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall + | takeAggFunction # takeAggFunctionCall + | percentileApproxFunction # percentileApproxFunctionCall + | statsFunctionName LT_PRTHS functionArgs RT_PRTHS # statsFunctionCall ; statsFunctionName @@ -419,72 +658,89 @@ statsFunctionName | SUM | MIN | MAX + | VAR_SAMP + | VAR_POP | STDDEV_SAMP | STDDEV_POP + | PERCENTILE + | PERCENTILE_APPROX + | MEDIAN + | LIST + | FIRST + | EARLIEST + | LATEST + | LAST ; -// expressions -expression - : logicalExpression - | valueExpression +takeAggFunction + : TAKE LT_PRTHS fieldExpression (COMMA size = integerLiteral)? RT_PRTHS + ; + +percentileApproxFunction + : (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS aggField = valueExpression + COMMA percent = numericLiteral (COMMA compression = numericLiteral)? RT_PRTHS ; +numericLiteral + : integerLiteral + | decimalLiteral + | doubleLiteral + | floatLiteral + ; + +// predicates logicalExpression : NOT logicalExpression # logicalNot - | LT_PRTHS logicalExpression RT_PRTHS # parentheticLogicalExpr - | comparisonExpression # comparsion - | left = logicalExpression (AND)? right = logicalExpression # logicalAnd - | left = logicalExpression OR right = logicalExpression # logicalOr + | left = logicalExpression AND right = logicalExpression # logicalAnd | left = logicalExpression XOR right = logicalExpression # logicalXor - | booleanExpression # booleanExpr - ; - -comparisonExpression - : left = valueExpression comparisonOperator right = valueExpression # compareExpr - | valueExpression NOT? IN valueList # inExpr - | expr1 = functionArg NOT? BETWEEN expr2 = functionArg AND expr3 = functionArg # between + | left = logicalExpression OR right = logicalExpression # logicalOr + | expression # logicalExpr ; -valueExpressionList - : valueExpression - | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS +expression + : valueExpression # valueExpr + | relevanceExpression # relevanceExpr + | left = expression comparisonOperator right = expression # compareExpr + | expression NOT? IN valueList # inExpr + | expression NOT? BETWEEN expression AND expression # between ; valueExpression - : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic - | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic - | primaryExpression # valueExpressionDefault - | positionFunction # positionFunctionCall - | caseFunction # caseExpr - | timestampFunction # timestampFunctionCall - | LT_PRTHS valueExpression RT_PRTHS # parentheticValueExpr - | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr - | ident ARROW expression # lambda - | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW expression # lambda - ; - -primaryExpression + : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic + | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic + | literalValue # literalValueExpr + | functionCall # functionCallExpr + | lambda # lambdaExpr + | LT_SQR_PRTHS subSearch RT_SQR_PRTHS # scalarSubqueryExpr + | valueExpression NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr + | LT_PRTHS valueExpression (COMMA valueExpression)* RT_PRTHS NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr + | EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr + | fieldExpression # fieldExpr + | LT_PRTHS logicalExpression RT_PRTHS # nestedValueExpr + ; + +evalExpression + : EVAL LT_PRTHS logicalExpression RT_PRTHS + ; + +functionCall : evalFunctionCall - | fieldExpression - | literalValue | dataTypeFunctionCall + | positionFunctionCall + | caseFunctionCall + | timestampFunctionCall + | extractFunctionCall + | getFormatFunctionCall ; -positionFunction +positionFunctionCall : positionFunctionName LT_PRTHS functionArg IN functionArg RT_PRTHS ; -booleanExpression - : booleanFunctionCall # booleanFunctionCallExpr - | valueExpressionList NOT? IN LT_SQR_PRTHS subSearch RT_SQR_PRTHS # inSubqueryExpr - | EXISTS LT_SQR_PRTHS subSearch RT_SQR_PRTHS # existsSubqueryExpr - | cidrMatchFunctionCall # cidrFunctionCallExpr +caseFunctionCall + : CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS ; - caseFunction - : CASE LT_PRTHS logicalExpression COMMA valueExpression (COMMA logicalExpression COMMA valueExpression)* (ELSE valueExpression)? RT_PRTHS - ; - relevanceExpression : singleFieldRelevanceFunction | multiFieldRelevanceFunction @@ -497,7 +753,7 @@ singleFieldRelevanceFunction // Field is a list of columns multiFieldRelevanceFunction - : multiFieldRelevanceFunctionName LT_PRTHS LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS + : multiFieldRelevanceFunctionName LT_PRTHS (LT_SQR_PRTHS field = relevanceFieldAndWeight (COMMA field = relevanceFieldAndWeight)* RT_SQR_PRTHS COMMA)? query = relevanceQuery (COMMA relevanceArg)* RT_PRTHS ; // tables @@ -507,16 +763,12 @@ tableSource ; tableFunction - : qualifiedName LT_PRTHS functionArgs RT_PRTHS + : qualifiedName LT_PRTHS namedFunctionArgs RT_PRTHS ; // fields fieldList - : fieldExpression (COMMA fieldExpression)* - ; - -wcFieldList - : wcFieldExpression (COMMA wcFieldExpression)* + : fieldExpression ((COMMA)? fieldExpression)* ; sortField @@ -525,8 +777,6 @@ sortField sortFieldExpression : fieldExpression - - // TODO #963: Implement 'num', 'str', and 'ip' sort syntax | AUTO LT_PRTHS fieldExpression RT_PRTHS | STR LT_PRTHS fieldExpression RT_PRTHS | IP LT_PRTHS fieldExpression RT_PRTHS @@ -541,6 +791,16 @@ wcFieldExpression : wcQualifiedName ; +selectFieldExpression + : wcQualifiedName + | STAR + ; + +renameFieldExpression + : wcQualifiedName + | STAR + ; + // functions evalFunctionCall : evalFunctionName LT_PRTHS functionArgs RT_PRTHS @@ -548,16 +808,7 @@ evalFunctionCall // cast function dataTypeFunctionCall - : CAST LT_PRTHS expression AS convertedDataType RT_PRTHS - ; - -// boolean functions -booleanFunctionCall - : conditionFunctionBase LT_PRTHS functionArgs RT_PRTHS - ; - -cidrMatchFunctionCall - : CIDRMATCH LT_PRTHS ipAddress = functionArg COMMA cidrBlock = functionArg RT_PRTHS + : CAST LT_PRTHS logicalExpression AS convertedDataType RT_PRTHS ; convertedDataType @@ -571,28 +822,48 @@ convertedDataType | typeName = FLOAT | typeName = STRING | typeName = BOOLEAN + | typeName = IP + | typeName = JSON ; evalFunctionName : mathematicalFunctionName | dateTimeFunctionName | textFunctionName - | conditionFunctionBase + | conditionFunctionName + | flowControlFunctionName | systemFunctionName | positionFunctionName - | coalesceFunctionName | cryptographicFunctionName | jsonFunctionName + | geoipFunctionName | collectionFunctionName - | lambdaFunctionName ; functionArgs : (functionArg (COMMA functionArg)*)? ; +namedFunctionArgs + : (namedFunctionArg (COMMA namedFunctionArg)*)? + ; + functionArg - : (ident EQUAL)? valueExpression + : functionArgExpression + ; + +namedFunctionArg + : (ident EQUAL)? functionArgExpression + ; + +functionArgExpression + : lambda + | logicalExpression + ; + +lambda + : ident ARROW logicalExpression + | LT_PRTHS ident (COMMA ident)+ RT_PRTHS ARROW logicalExpression ; relevanceArg @@ -644,6 +915,8 @@ relevanceFieldAndWeight relevanceFieldWeight : integerLiteral | decimalLiteral + | doubleLiteral + | floatLiteral ; relevanceField @@ -662,6 +935,10 @@ relevanceArgValue mathematicalFunctionName : ABS + | PLUS_FUCTION + | MINUS_FUCTION + | STAR_FUNCTION + | DIVIDE_FUNCTION | CBRT | CEIL | CEILING @@ -669,37 +946,72 @@ mathematicalFunctionName | CRC32 | E | EXP + | EXPM1 | FLOOR | LN | LOG - | LOG10 - | LOG2 + | LOG_WITH_BASE | MOD + | MODULUS | PI | POW | POWER | RAND | ROUND | SIGN - | SIGNUM | SQRT | TRUNCATE + | RINT + | SIGNUM + | SUM + | AVG | trigonometricFunctionName ; +geoipFunctionName + : GEOIP + ; + +collectionFunctionName + : ARRAY + | ARRAY_LENGTH + | MVJOIN + | FORALL + | EXISTS + | FILTER + | TRANSFORM + | REDUCE + ; + + trigonometricFunctionName : ACOS | ASIN | ATAN | ATAN2 | COS + | COSH | COT | DEGREES | RADIANS | SIN + | SINH | TAN ; +jsonFunctionName + : JSON + | JSON_OBJECT + | JSON_ARRAY + | JSON_ARRAY_LENGTH + | JSON_EXTRACT + | JSON_KEYS + | JSON_SET + | JSON_DELETE + | JSON_APPEND + | JSON_EXTEND + ; + cryptographicFunctionName : MD5 | SHA1 @@ -714,7 +1026,6 @@ dateTimeFunctionName | CURRENT_DATE | CURRENT_TIME | CURRENT_TIMESTAMP - | CURRENT_TIMEZONE | CURTIME | DATE | DATEDIFF @@ -738,7 +1049,6 @@ dateTimeFunctionName | LOCALTIME | LOCALTIMESTAMP | MAKEDATE - | MAKE_DATE | MAKETIME | MICROSECOND | MINUTE @@ -774,16 +1084,9 @@ dateTimeFunctionName | WEEK_OF_YEAR | YEAR | YEARWEEK - | relativeTimeFunctionName ; -relativeTimeFunctionName - : RELATIVE_TIMESTAMP - | EARLIEST - | LATEST - ; - -getFormatFunction +getFormatFunctionCall : GET_FORMAT LT_PRTHS getFormatType COMMA functionArg RT_PRTHS ; @@ -794,7 +1097,7 @@ getFormatType | TIMESTAMP ; -extractFunction +extractFunctionCall : EXTRACT LT_PRTHS datetimePart FROM functionArg RT_PRTHS ; @@ -829,7 +1132,7 @@ datetimePart | complexDateTimePart ; -timestampFunction +timestampFunctionCall : timestampFunctionName LT_PRTHS simpleDateTimePart COMMA firstArg = functionArg COMMA secondArg = functionArg RT_PRTHS ; @@ -839,19 +1142,26 @@ timestampFunctionName ; // condition function return boolean value -conditionFunctionBase +conditionFunctionName : LIKE - | IF | ISNULL | ISNOTNULL - | IFNULL - | NULLIF - | ISPRESENT + | CIDRMATCH + | REGEX_MATCH | JSON_VALID - | EARLIEST - | LATEST + | ISPRESENT | ISEMPTY | ISBLANK + | EARLIEST + | LATEST + ; + +// flow control function return non-boolean value +flowControlFunctionName + : IF + | IFNULL + | NULLIF + | COALESCE ; systemFunctionName @@ -876,75 +1186,23 @@ textFunctionName | LOCATE | REPLACE | REVERSE - | ISEMPTY - | ISBLANK - ; - -jsonFunctionName - : JSON - | JSON_OBJECT - | JSON_ARRAY - | JSON_ARRAY_LENGTH - | TO_JSON_STRING - | JSON_EXTRACT - | JSON_DELETE - | JSON_APPEND - | JSON_KEYS - | JSON_VALID - | JSON_EXTEND - | JSON_SET -// | JSON_ARRAY_ALL_MATCH -// | JSON_ARRAY_ANY_MATCH -// | JSON_ARRAY_FILTER -// | JSON_ARRAY_MAP -// | JSON_ARRAY_REDUCE ; -collectionFunctionName - : ARRAY - | ARRAY_LENGTH - ; - -lambdaFunctionName - : FORALL - | EXISTS - | FILTER - | TRANSFORM - | REDUCE - ; - positionFunctionName : POSITION ; -coalesceFunctionName - : COALESCE - ; - -geoIpPropertyList - : geoIpProperty (COMMA geoIpProperty)* - ; - -geoIpProperty - : COUNTRY_ISO_CODE - | COUNTRY_NAME - | CONTINENT_NAME - | REGION_ISO_CODE - | REGION_NAME - | CITY_NAME - | TIME_ZONE - | LOCATION - ; - // operators comparisonOperator : EQUAL + | DOUBLE_EQUAL | NOT_EQUAL | LESS | NOT_LESS | GREATER | NOT_GREATER | REGEXP + | LIKE ; singleFieldRelevanceFunctionName @@ -962,12 +1220,14 @@ multiFieldRelevanceFunctionName // literals and values literalValue - : stringLiteral + : intervalLiteral + | stringLiteral | integerLiteral | decimalLiteral + | doubleLiteral + | floatLiteral | booleanLiteral | datetimeLiteral //#datetime - | intervalLiteral ; intervalLiteral @@ -987,6 +1247,14 @@ decimalLiteral : (PLUS | MINUS)? DECIMAL_LITERAL ; +doubleLiteral + : (PLUS | MINUS)? DOUBLE_LITERAL + ; + +floatLiteral + : (PLUS | MINUS)? FLOAT_LITERAL + ; + booleanLiteral : TRUE | FALSE @@ -1052,6 +1320,20 @@ timespanUnit | MONTH | QUARTER | YEAR + | SEC + | SECS + | SECONDS + | MINS + | MINUTES + | HR + | HRS + | HOURS + | DAYS + | MON + | MONTHS + | US + | CS + | DS ; valueList @@ -1062,11 +1344,6 @@ qualifiedName : ident (DOT ident)* # identsAsQualifiedName ; -identifierSeq - : qualifiedName (COMMA qualifiedName)* # identsAsQualifiedNameSeq - | LT_PRTHS qualifiedName (COMMA qualifiedName)* RT_PRTHS # identsAsQualifiedNameSeq - ; - tableQualifiedName : tableIdent (DOT ident)* # identsAsTableQualifiedName ; @@ -1075,6 +1352,11 @@ wcQualifiedName : wildcard (DOT wildcard)* # identsAsWildcardQualifiedName ; +identifierSeq + : qualifiedName (COMMA qualifiedName)* # identsAsQualifiedNameSeq + | LT_PRTHS qualifiedName (COMMA qualifiedName)* RT_PRTHS # identsAsQualifiedNameSeq + ; + ident : (DOT)? ID | BACKTICK ident BACKTICK @@ -1094,40 +1376,49 @@ wildcard ; keywordsCanBeId + : searchableKeyWord + | IN + ; + +searchableKeyWord : D // OD SQL and ODBC special | timespanUnit | SPAN | evalFunctionName + | jsonFunctionName | relevanceArgName | intervalUnit - | dateTimeFunctionName - | textFunctionName - | jsonFunctionName - | mathematicalFunctionName - | positionFunctionName - | cryptographicFunctionName + | trendlineType | singleFieldRelevanceFunctionName | multiFieldRelevanceFunctionName | commandName - | comparisonOperator + | collectionFunctionName + | REGEX | explainMode - | correlationType - | geoIpProperty + | REGEXP // commands assist keywords - | GEOIP - | OVERRIDE + | CASE + | ELSE | ARROW - | IN + | BETWEEN + | EXISTS | SOURCE | INDEX + | A + | ASC | DESC | DATASOURCES | FROM | PATTERN | NEW_FIELD - | SCOPE - | MAPPING + | METHOD + | VARIABLE_COUNT_THRESHOLD + | FREQUENCY_THRESHOLD_PERCENTAGE + | MAX_SAMPLE_COUNT + | BUFFER_LIMIT | WITH + | REGEX + | PUNCT | USING | CAST | GET_FORMAT @@ -1135,8 +1426,12 @@ keywordsCanBeId | INTERVAL | PLUS | MINUS - | INCLUDEFIELDS - | NULLS + | OVERRIDE + // SORT FIELD KEYWORDS + | AUTO + | STR + | IP + | NUM // ARGUMENT KEYWORDS | KEEPEMPTY | CONSECUTIVE @@ -1144,6 +1439,7 @@ keywordsCanBeId | PARTITIONS | ALLNUM | DELIM + | BUCKET_NULLABLE | CENTROIDS | ITERATIONS | DISTANCE_TYPE @@ -1158,12 +1454,17 @@ keywordsCanBeId | TIME_ZONE | TRAINING_DATA_SIZE | ANOMALY_SCORE_THRESHOLD - // AGGREGATIONS + | COUNTFIELD + | SHOWCOUNT + | PATH + | INPUT + | OUTPUT + + // AGGREGATIONS AND WINDOW | statsFunctionName + | windowFunctionName | DISTINCT_COUNT | DISTINCT_COUNT_APPROX - | PERCENTILE - | PERCENTILE_APPROX | ESTDC | ESTDC_ERROR | MEAN @@ -1176,8 +1477,6 @@ keywordsCanBeId | VAR_SAMP | VAR_POP | TAKE - | FIRST - | LAST | LIST | VALUES | PER_DAY @@ -1197,12 +1496,7 @@ keywordsCanBeId | FULL | SEMI | ANTI - | BETWEEN - | CIDRMATCH - | trendlineType - // SORT FIELD KEYWORDS - | AUTO - | STR - | IP - | NUM + | LEFT_HINT + | RIGHT_HINT + | PERCENTILE_SHORTCUT ; diff --git a/legacy/build.gradle b/legacy/build.gradle index 94489b3de20..7e78d5e6e13 100644 --- a/legacy/build.gradle +++ b/legacy/build.gradle @@ -26,7 +26,7 @@ plugins { id 'java' id 'io.freefair.lombok' id 'antlr' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } generateGrammarSource { @@ -61,10 +61,10 @@ spotless { exclude '**/build/**', '**/build-*/**', '**/gen/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java index 33d3daeec61..c1691de3c21 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/Paginate.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.legacy.query.planner.physical.node; import java.util.Arrays; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java index 639f7187105..368a24a8bf1 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/query/planner/physical/node/pointInTime/PointInTime.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.legacy.query.planner.physical.node.pointInTime; import static org.opensearch.sql.opensearch.storage.OpenSearchIndex.METADATA_FIELD_ID; diff --git a/legacy/src/main/java/org/opensearch/sql/legacy/utils/MetricUtils.java b/legacy/src/main/java/org/opensearch/sql/legacy/utils/MetricUtils.java index b7a56bde4f4..76df4b2e11d 100644 --- a/legacy/src/main/java/org/opensearch/sql/legacy/utils/MetricUtils.java +++ b/legacy/src/main/java/org/opensearch/sql/legacy/utils/MetricUtils.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.legacy.utils; import lombok.experimental.UtilityClass; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java index 237e13c7a1e..de51bd448e6 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/executor/format/PrettyFormatRestExecutorTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.legacy.executor.format; import static org.junit.Assert.assertFalse; diff --git a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java index f65b65ede6a..e1e0d45cff4 100644 --- a/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java +++ b/legacy/src/test/java/org/opensearch/sql/legacy/pit/PointInTimeHandlerImplTest.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.legacy.pit; import static org.junit.Assert.*; diff --git a/opensearch/build.gradle b/opensearch/build.gradle index f41eac0a0fd..27aa81b0b67 100644 --- a/opensearch/build.gradle +++ b/opensearch/build.gradle @@ -27,7 +27,7 @@ plugins { id "io.freefair.lombok" id 'jacoco' id 'info.solidsoft.pitest' version '1.9.0' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } dependencies { @@ -64,10 +64,11 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") + .onlyIfContentMatches("^((?!\\(original license below\\))[\\s\\S])*\$") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/MLClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/MLClient.java index f395fa85d63..ac884ea5abd 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/MLClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/MLClient.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.opensearch.client; import org.opensearch.ml.client.MachineLearningNodeClient; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java index db09074da47..0261bc98120 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchClient.java @@ -76,6 +76,13 @@ public interface OpenSearchClient { */ Map meta(); + /** + * Force to clean up resources related to the search request. + * + * @param request search request + */ + void forceCleanup(OpenSearchRequest request); + /** * Clean up resources related to the search request, for example scroll context. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java index 9f3f114e676..152fe499d2e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClient.java @@ -89,6 +89,9 @@ public Map getIndexMappings(String... indexExpression) { try { GetMappingsResponse mappingsResponse = client.admin().indices().prepareGetMappings(indexExpression).setLocal(true).get(); + if (mappingsResponse.mappings().isEmpty()) { + throw new IndexNotFoundException(indexExpression[0]); + } return mappingsResponse.mappings().entrySet().stream() .collect( Collectors.toUnmodifiableMap( @@ -171,6 +174,27 @@ public Map meta() { client.settings().get("plugins.sql.pagination.api", "true")); } + @Override + public void forceCleanup(OpenSearchRequest request) { + if (request instanceof OpenSearchScrollRequest) { + request.forceClean( + scrollId -> { + try { + client.prepareClearScroll().addScrollId(scrollId).get(); + } catch (Exception e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.forceClean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } + } + @Override public void cleanup(OpenSearchRequest request) { if (request instanceof OpenSearchScrollRequest) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java index a65aa1997a8..87b171707bb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/client/OpenSearchRestClient.java @@ -29,6 +29,7 @@ import org.opensearch.client.indices.GetMappingsResponse; import org.opensearch.cluster.metadata.AliasMetadata; import org.opensearch.common.settings.Settings; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.sql.opensearch.mapping.IndexMapping; import org.opensearch.sql.opensearch.request.OpenSearchRequest; import org.opensearch.sql.opensearch.request.OpenSearchScrollRequest; @@ -71,6 +72,9 @@ public Map getIndexMappings(String... indexExpression) { GetMappingsRequest request = new GetMappingsRequest().indices(indexExpression); try { GetMappingsResponse response = client.indices().getMapping(request, RequestOptions.DEFAULT); + if (response.mappings().isEmpty()) { + throw new IndexNotFoundException(indexExpression[0]); + } return response.mappings().entrySet().stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> new IndexMapping(e.getValue()))); } catch (IOException e) { @@ -180,6 +184,29 @@ public Map meta() { } } + @Override + public void forceCleanup(OpenSearchRequest request) { + if (request instanceof OpenSearchScrollRequest) { + request.forceClean( + scrollId -> { + try { + ClearScrollRequest clearRequest = new ClearScrollRequest(); + clearRequest.addScrollId(scrollId); + client.clearScroll(clearRequest, RequestOptions.DEFAULT); + } catch (IOException e) { + throw new IllegalStateException( + "Failed to clean up resources for search request " + request, e); + } + }); + } else { + request.forceClean( + pitId -> { + DeletePitRequest deletePitRequest = new DeletePitRequest(pitId); + deletePit(deletePitRequest); + }); + } + } + @Override public void cleanup(OpenSearchRequest request) { if (request instanceof OpenSearchScrollRequest) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index d813ffdb962..dc4a4bcaed7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -338,6 +338,11 @@ private static ExprValue createOpenSearchDateType(Content value, ExprType type) return parseDateTimeString(value.stringValue(), dt); } + if (value.objectValue() instanceof ZonedDateTime) { + ZonedDateTime zonedDateTime = (ZonedDateTime) value.objectValue(); + return new ExprTimestampValue(zonedDateTime.withZoneSameLocal(ZoneOffset.UTC).toInstant()); + } + return new ExprTimestampValue((Instant) value.objectValue()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java index 2aa10ef8859..76218d8295d 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManager.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.opensearch.executor; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java index eeb6c671283..f5f9969b8fc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchAggregateIndexScanRule.java @@ -2,15 +2,24 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.opensearch.planner.physical; +import static org.opensearch.sql.expression.function.PPLBuiltinOperators.WIDTH_BUCKET; + +import java.util.List; import java.util.function.Predicate; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; +import org.apache.calcite.rel.AbstractRelNode; import org.apache.calcite.rel.logical.LogicalAggregate; import org.apache.calcite.rel.logical.LogicalProject; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexCall; import org.apache.calcite.sql.SqlKind; import org.immutables.value.Value; +import org.opensearch.sql.calcite.type.ExprSqlType; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.ExprUDT; import org.opensearch.sql.opensearch.storage.scan.CalciteLogicalIndexScan; /** Planner rule that push a {@link LogicalAggregate} down to {@link CalciteLogicalIndexScan} */ @@ -29,6 +38,13 @@ public void onMatch(RelOptRuleCall call) { final LogicalAggregate aggregate = call.rel(0); final LogicalProject project = call.rel(1); final CalciteLogicalIndexScan scan = call.rel(2); + + // For multiple group-by, we currently have to use CompositeAggregationBuilder while it + // doesn't support auto_date_histogram referring to bin command with parameter bins + if (aggregate.getGroupSet().length() > 1 && Config.containsWidthBucketFuncOnDate(project)) { + return; + } + apply(call, aggregate, project, scan); } else if (call.rels.length == 2) { // case of count() without group-by @@ -48,9 +64,9 @@ protected void apply( LogicalAggregate aggregate, LogicalProject project, CalciteLogicalIndexScan scan) { - CalciteLogicalIndexScan newScan = scan.pushDownAggregate(aggregate, project); - if (newScan != null) { - call.transformTo(newScan); + AbstractRelNode newRelNode = scan.pushDownAggregate(aggregate, project); + if (newRelNode != null) { + call.transformTo(newRelNode); } } @@ -68,9 +84,13 @@ public interface Config extends RelRule.Config { b1 -> b1.operand(LogicalProject.class) .predicate( - // Don't push down aggregate on window function + // Support push down aggregate with project that: + // 1. No RexOver and no duplicate projection + // 2. Contains width_bucket function on date field referring + // to bin command with parameter bins Predicate.not(OpenSearchIndexScanRule::containsRexOver) - .and(OpenSearchIndexScanRule::distinctProjectList)) + .and(OpenSearchIndexScanRule::distinctProjectList) + .or(Config::containsWidthBucketFuncOnDate)) .oneInput( b2 -> b2.operand(CalciteLogicalIndexScan.class) @@ -108,5 +128,20 @@ public interface Config extends RelRule.Config { default OpenSearchAggregateIndexScanRule toRule() { return new OpenSearchAggregateIndexScanRule(this); } + + static boolean containsWidthBucketFuncOnDate(LogicalProject project) { + return project.getProjects().stream() + .anyMatch( + expr -> + expr instanceof RexCall rexCall + && rexCall.getOperator().equals(WIDTH_BUCKET) + && dateRelatedType(rexCall.getOperands().getFirst().getType())); + } + + static boolean dateRelatedType(RelDataType type) { + return type instanceof ExprSqlType exprSqlType + && List.of(ExprUDT.EXPR_DATE, ExprUDT.EXPR_TIME, ExprUDT.EXPR_TIMESTAMP) + .contains(exprSqlType.getUdt()); + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java index a070e3ef1ba..a51fa365905 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchDedupPushdownRule.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.opensearch.planner.physical; import static org.opensearch.sql.calcite.utils.PlanUtils.ROW_NUMBER_COLUMN_FOR_DEDUP; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalOperator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalOperator.java index b52c1dc3f86..32aba3b5ac4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalOperator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalOperator.java @@ -1,8 +1,6 @@ /* - * * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * */ package org.opensearch.sql.opensearch.planner.physical; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalProcessor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalProcessor.java index 85834efe539..c61665f2729 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalProcessor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchEvalProcessor.java @@ -1,8 +1,6 @@ /* - * * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * */ package org.opensearch.sql.opensearch.planner.physical; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java index 4ee5148b826..306617ae2cf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchFilterIndexScanRule.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.opensearch.planner.physical; import java.util.function.Predicate; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java index 50f67485cca..24abb3c3bc9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchIndexScanRule.java @@ -15,6 +15,7 @@ import org.apache.calcite.rex.RexCall; import org.apache.calcite.rex.RexNode; import org.apache.calcite.rex.RexOver; +import org.apache.calcite.util.Pair; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan; @@ -41,8 +42,10 @@ static boolean isLimitPushed(AbstractCalciteIndexScan scan) { // e.g. Project($0, $0, +($0,1)). We cannot push down the Aggregate for this corner case. // TODO: Simplify the Project where there is RexCall by adding a new rule. static boolean distinctProjectList(LogicalProject project) { - Set rexSet = new HashSet<>(); - return project.getProjects().stream().allMatch(rexSet::add); + // Change to Set> to resolve + // https://github.com/opensearch-project/sql/issues/4347 + Set> rexSet = new HashSet<>(); + return project.getNamedProjects().stream().allMatch(rexSet::add); } static boolean containsRexOver(LogicalProject project) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java index 2724c419e33..b468d482570 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchProjectIndexScanRule.java @@ -2,12 +2,14 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.opensearch.planner.physical; import static java.util.Objects.requireNonNull; import java.util.ArrayList; import java.util.List; +import java.util.Objects; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelOptTable; import org.apache.calcite.plan.RelRule; @@ -52,7 +54,7 @@ protected void apply(RelOptRuleCall call, LogicalProject project, CalciteLogical // TODO: support script pushdown for project instead of only reference // https://github.com/opensearch-project/sql/issues/3387 - final List selectedColumns = new ArrayList<>(); + final SelectedColumns selectedColumns = new SelectedColumns(); final RexVisitorImpl visitor = new RexVisitorImpl(true) { @Override @@ -65,19 +67,39 @@ public Void visitInputRef(RexInputRef inputRef) { }; visitor.visitEach(project.getProjects()); // Only do push down when an actual projection happens - if (!selectedColumns.isEmpty() && selectedColumns.size() != scan.getRowType().getFieldCount()) { + if (!selectedColumns.isEmpty() + && !selectedColumns.isIdentity(scan.getRowType().getFieldCount())) { Mapping mapping = Mappings.target(selectedColumns, scan.getRowType().getFieldCount()); CalciteLogicalIndexScan newScan = scan.pushDownProject(selectedColumns); - final List newProjectRexNodes = RexUtil.apply(mapping, project.getProjects()); + if (newScan != null) { + final List newProjectRexNodes = RexUtil.apply(mapping, project.getProjects()); - if (RexUtil.isIdentity(newProjectRexNodes, newScan.getRowType())) { - call.transformTo(newScan); - } else { - call.transformTo(call.builder().push(newScan).project(newProjectRexNodes).build()); + if (RexUtil.isIdentity(newProjectRexNodes, newScan.getRowType())) { + call.transformTo(newScan); + } else { + call.transformTo(call.builder().push(newScan).project(newProjectRexNodes).build()); + } } } } + static final class SelectedColumns extends ArrayList { + private boolean isSequential = true; + private Integer current = 0; + + @Override + public boolean add(Integer integer) { + if (isSequential && !Objects.equals(integer, current++)) { + isSequential = false; + } + return super.add(integer); + } + + public boolean isIdentity(Integer size) { + return isSequential && size == size(); + } + } + /** Rule configuration. */ @Value.Immutable public interface Config extends RelRule.Config { @@ -88,11 +110,7 @@ public interface Config extends RelRule.Config { .withOperandSupplier( b0 -> b0.operand(LogicalProject.class) - .oneInput( - b1 -> - b1.operand(CalciteLogicalIndexScan.class) - .predicate(OpenSearchIndexScanRule::noAggregatePushed) - .noInputs())); + .oneInput(b1 -> b1.operand(CalciteLogicalIndexScan.class).noInputs())); @Override default OpenSearchProjectIndexScanRule toRule() { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java index 36909772c52..c36a410dccf 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/OpenSearchRelevanceFunctionPushdownRule.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.opensearch.planner.physical; import static org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils.MULTI_FIELDS_RELEVANCE_FUNCTION_SET; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java index cfe737cc386..dfec6754908 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/planner/physical/SortProjectExprTransposeRule.java @@ -10,6 +10,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.Predicate; import org.apache.calcite.plan.RelOptRuleCall; import org.apache.calcite.plan.RelRule; import org.apache.calcite.rel.RelCollation; @@ -129,8 +130,9 @@ public interface Config extends RelRule.Config { .oneInput( b1 -> b1.operand(LogicalProject.class) - .predicate(OpenSearchIndexScanRule::projectContainsExpr) - .predicate(p -> !p.containsOver()) + .predicate( + Predicate.not(LogicalProject::containsOver) + .and(OpenSearchIndexScanRule::projectContainsExpr)) .anyInputs())); @Override diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java index 79624310cd7..065375dd230 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/AggregateAnalyzer.java @@ -24,12 +24,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.opensearch.sql.opensearch.request; import static java.util.Objects.requireNonNull; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.TIME; import static org.opensearch.sql.data.type.ExprCoreType.TIMESTAMP; +import static org.opensearch.sql.expression.function.PPLBuiltinOperators.WIDTH_BUCKET; import com.google.common.base.Throwables; import com.google.common.collect.ImmutableList; @@ -58,23 +60,28 @@ import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder; import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; +import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; import org.opensearch.search.aggregations.bucket.missing.MissingOrder; import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.aggregations.metrics.ExtendedStats; import org.opensearch.search.aggregations.metrics.PercentilesAggregationBuilder; import org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.opensearch.search.aggregations.support.ValueType; import org.opensearch.search.aggregations.support.ValuesSourceAggregationBuilder; import org.opensearch.search.sort.SortOrder; import org.opensearch.sql.ast.expression.Argument; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.PredicateAnalyzer.NamedFieldExpression; import org.opensearch.sql.opensearch.response.agg.ArgMaxMinParser; import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; +import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; @@ -127,6 +134,7 @@ static class AggregateBuilderHelper { final RelDataType rowType; final Map fieldTypes; final RelOptCluster cluster; + final boolean bucketNullable; > T build(RexNode node, T aggBuilder) { return build(node, aggBuilder::field, aggBuilder::script); @@ -186,7 +194,8 @@ public static Pair, OpenSearchAggregationResponseParser .findFirst() .orElseGet(() -> "true")); List groupList = aggregate.getGroupSet().asList(); - AggregateBuilderHelper helper = new AggregateBuilderHelper(rowType, fieldTypes, cluster); + AggregateBuilderHelper helper = + new AggregateBuilderHelper(rowType, fieldTypes, cluster, bucketNullable); List aggFieldNames = outputFields.subList(groupList.size(), outputFields.size()); // Process all aggregate calls Pair> builderAndParser = @@ -194,27 +203,77 @@ public static Pair, OpenSearchAggregationResponseParser Builder metricBuilder = builderAndParser.getLeft(); List metricParserList = builderAndParser.getRight(); + // both count() and count(FIELD) can apply doc_count optimization in non-bucket aggregation, + // but only count() can apply doc_count optimization in bucket aggregation. + boolean countAllOnly = !aggregate.getGroupSet().isEmpty(); + Pair, Builder> pair = + removeCountAggregationBuilders(metricBuilder, countAllOnly); + List removedCountAggBuilders = pair.getLeft(); + Builder newMetricBuilder = pair.getRight(); + + boolean removedCountAggBuildersHaveSomeField = + removedCountAggBuilders.stream() + .map(ValuesSourceAggregationBuilder::fieldName) + .distinct() + .count() + == 1; + boolean allCountAggRemoved = + removedCountAggBuilders.size() == metricBuilder.getAggregatorFactories().size(); if (aggregate.getGroupSet().isEmpty()) { - return Pair.of( - ImmutableList.copyOf(metricBuilder.getAggregatorFactories()), - new NoBucketAggregationParser(metricParserList)); - } else if (aggregate.getGroupSet().length() == 1 && !bucketNullable) { - // one bucket, use values source bucket builder for getting better performance - // TODO for multiple buckets, use MultiTermsAggregationBuilder + if (allCountAggRemoved && removedCountAggBuildersHaveSomeField) { + // The optimization must require all count aggregations are removed, + // and they have only one field name + List countAggNameList = + removedCountAggBuilders.stream() + .map(ValuesSourceAggregationBuilder::getName) + .toList(); + return Pair.of( + ImmutableList.copyOf(newMetricBuilder.getAggregatorFactories()), + new CountAsTotalHitsParser(countAggNameList)); + } else { + return Pair.of( + ImmutableList.copyOf(metricBuilder.getAggregatorFactories()), + new NoBucketAggregationParser(metricParserList)); + } + } else if (aggregate.getGroupSet().length() == 1 + && isAutoDateSpan(project.getProjects().get(groupList.getFirst()))) { + RexCall rexCall = (RexCall) project.getProjects().get(groupList.getFirst()); + String bucketName = project.getRowType().getFieldList().get(groupList.getFirst()).getName(); + RexInputRef rexInputRef = (RexInputRef) rexCall.getOperands().getFirst(); + RexLiteral valueLiteral = (RexLiteral) rexCall.getOperands().get(1); ValuesSourceAggregationBuilder bucketBuilder = - createBucketAggregation(groupList.getFirst(), project, helper); + new AutoDateHistogramAggregationBuilder(bucketName) + .field(helper.inferNamedField(rexInputRef).getRootName()) + .setNumBuckets(requireNonNull(valueLiteral.getValueAs(Integer.class))); return Pair.of( Collections.singletonList(bucketBuilder.subAggregations(metricBuilder)), new BucketAggregationParser(metricParserList)); } else { List> buckets = createCompositeBuckets(groupList, project, helper); + AggregationBuilder aggregationBuilder = + AggregationBuilders.composite("composite_buckets", buckets) + .size(AGGREGATION_BUCKET_SIZE); + + // For bucket aggregation, no count() aggregator or not all aggregators are count(), + // fallback to original ValueCountAggregation. + if (removedCountAggBuilders.isEmpty() + || removedCountAggBuilders.size() != metricBuilder.getAggregatorFactories().size()) { + aggregationBuilder.subAggregations(metricBuilder); + return Pair.of( + Collections.singletonList(aggregationBuilder), + new CompositeAggregationParser(metricParserList)); + } + // No need to register sub-factories if no aggregator factories left after removing all + // ValueCountAggregationBuilder. + if (!newMetricBuilder.getAggregatorFactories().isEmpty()) { + aggregationBuilder.subAggregations(newMetricBuilder); + } + List countAggNameList = + removedCountAggBuilders.stream().map(ValuesSourceAggregationBuilder::getName).toList(); return Pair.of( - Collections.singletonList( - AggregationBuilders.composite("composite_buckets", buckets) - .subAggregations(metricBuilder) - .size(AGGREGATION_BUCKET_SIZE)), - new CompositeAggregationParser(metricParserList)); + Collections.singletonList(aggregationBuilder), + new CompositeAggregationParser(metricParserList, countAggNameList)); } } catch (Throwable e) { Throwables.throwIfInstanceOf(e, UnsupportedOperationException.class); @@ -222,11 +281,35 @@ public static Pair, OpenSearchAggregationResponseParser } } + /** + * Remove all ValueCountAggregationBuilder from metric builder, and return the removed + * ValueCountAggregationBuilder list. + * + * @param metricBuilder metrics builder + * @param countAllOnly remove count() only, or count(FIELD) will be removed. + * @return a pair of removed ValueCountAggregationBuilder and updated metric builder + */ + private static Pair, Builder> removeCountAggregationBuilders( + Builder metricBuilder, boolean countAllOnly) { + List countAggregatorFactories = + metricBuilder.getAggregatorFactories().stream() + .filter(ValueCountAggregationBuilder.class::isInstance) + .map(ValueCountAggregationBuilder.class::cast) + .filter(vc -> vc.script() == null) + .filter(vc -> !countAllOnly || vc.fieldName().equals("_index")) + .toList(); + List copy = new ArrayList<>(metricBuilder.getAggregatorFactories()); + copy.removeAll(countAggregatorFactories); + Builder newMetricBuilder = new AggregatorFactories.Builder(); + copy.forEach(newMetricBuilder::addAggregator); + return Pair.of(countAggregatorFactories, newMetricBuilder); + } + private static Pair> processAggregateCalls( List aggFieldNames, List aggCalls, Project project, - AggregateBuilderHelper helper) + AggregateAnalyzer.AggregateBuilderHelper helper) throws PredicateAnalyzer.ExpressionNotAnalyzableException { Builder metricBuilder = new AggregatorFactories.Builder(); List metricParserList = new ArrayList<>(); @@ -298,12 +381,46 @@ private static Pair createRegularAggregation( helper.build( !args.isEmpty() ? args.getFirst() : null, AggregationBuilders.count(aggFieldName)), new SingleValueParser(aggFieldName)); - case MIN -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.min(aggFieldName)), - new SingleValueParser(aggFieldName)); - case MAX -> Pair.of( - helper.build(args.getFirst(), AggregationBuilders.max(aggFieldName)), - new SingleValueParser(aggFieldName)); + case MIN -> { + String fieldName = helper.inferNamedField(args.getFirst()).getRootName(); + ExprType fieldType = helper.fieldTypes.get(fieldName); + + if (supportsMaxMinAggregation(fieldType)) { + yield Pair.of( + helper.build(args.getFirst(), AggregationBuilders.min(aggFieldName)), + new SingleValueParser(aggFieldName)); + } else { + yield Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .size(1) + .from(0) + .sort( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery(), + SortOrder.ASC), + new TopHitsParser(aggFieldName, true)); + } + } + case MAX -> { + String fieldName = helper.inferNamedField(args.getFirst()).getRootName(); + ExprType fieldType = helper.fieldTypes.get(fieldName); + + if (supportsMaxMinAggregation(fieldType)) { + yield Pair.of( + helper.build(args.getFirst(), AggregationBuilders.max(aggFieldName)), + new SingleValueParser(aggFieldName)); + } else { + yield Pair.of( + AggregationBuilders.topHits(aggFieldName) + .fetchSource(helper.inferNamedField(args.getFirst()).getRootName(), null) + .size(1) + .from(0) + .sort( + helper.inferNamedField(args.getFirst()).getReferenceForTermQuery(), + SortOrder.DESC), + new TopHitsParser(aggFieldName, true)); + } + } case VAR_SAMP -> Pair.of( helper.build(args.getFirst(), AggregationBuilders.extendedStats(aggFieldName)), new StatsParser(ExtendedStats::getVarianceSampling, aggFieldName)); @@ -378,11 +495,23 @@ yield switch (functionName) { String.format("Unsupported push-down aggregator %s", aggCall.getAggregation())); }; } - default -> throw new AggregateAnalyzerException( + default -> throw new AggregateAnalyzer.AggregateAnalyzerException( String.format("unsupported aggregator %s", aggCall.getAggregation())); }; } + private static boolean supportsMaxMinAggregation(ExprType fieldType) { + ExprType coreType = + (fieldType instanceof OpenSearchDataType) + ? ((OpenSearchDataType) fieldType).getExprType() + : fieldType; + + return ExprCoreType.numberTypes().contains(coreType) + || coreType == ExprCoreType.DATE + || coreType == ExprCoreType.TIME + || coreType == ExprCoreType.TIMESTAMP; + } + private static ValuesSourceAggregationBuilder createBucketAggregation( Integer group, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { return createBucket(group, project, helper); @@ -396,6 +525,12 @@ private static List> createCompositeBuckets( return resultBuilder.build(); } + private static boolean isAutoDateSpan(RexNode rex) { + return rex instanceof RexCall rexCall + && rexCall.getKind() == SqlKind.OTHER_FUNCTION + && rexCall.getOperator().equals(WIDTH_BUCKET); + } + private static ValuesSourceAggregationBuilder createBucket( Integer groupIndex, Project project, AggregateBuilderHelper helper) { RexNode rex = project.getProjects().get(groupIndex); @@ -418,7 +553,7 @@ private static ValuesSourceAggregationBuilder createBucket( } private static CompositeValuesSourceBuilder createCompositeBucket( - Integer groupIndex, Project project, AggregateBuilderHelper helper) { + Integer groupIndex, Project project, AggregateAnalyzer.AggregateBuilderHelper helper) { RexNode rex = project.getProjects().get(groupIndex); String bucketName = project.getRowType().getFieldList().get(groupIndex).getName(); if (rex instanceof RexCall rexCall @@ -433,7 +568,12 @@ private static CompositeValuesSourceBuilder createCompositeBucket( helper.inferNamedField(rexInputRef).getRootName(), valueLiteral.getValueAs(Double.class), SpanUnit.of(unitLiteral.getValueAs(String.class)), - MissingOrder.FIRST); + MissingOrder.FIRST, + helper.bucketNullable); + } else if (isAutoDateSpan(rex)) { + // Defense check. We've already prevented this case in OpenSearchAggregateIndexScanRule. + throw new UnsupportedOperationException( + "auto_date_histogram is not supported in composite agg."); } else { return createTermsSourceBuilder(bucketName, rex, helper); } @@ -441,13 +581,12 @@ private static CompositeValuesSourceBuilder createCompositeBucket( private static CompositeValuesSourceBuilder createTermsSourceBuilder( String bucketName, RexNode group, AggregateBuilderHelper helper) { - CompositeValuesSourceBuilder sourceBuilder = - helper.build( - group, - new TermsValuesSourceBuilder(bucketName) - .missingBucket(true) - .missingOrder(MissingOrder.FIRST) - .order(SortOrder.ASC)); + TermsValuesSourceBuilder termsBuilder = + new TermsValuesSourceBuilder(bucketName).order(SortOrder.ASC); + if (helper.bucketNullable) { + termsBuilder.missingBucket(true).missingOrder(MissingOrder.FIRST); + } + CompositeValuesSourceBuilder sourceBuilder = helper.build(group, termsBuilder); // Time types values are converted to LONG in ExpressionAggregationScript::execute if (List.of(TIMESTAMP, TIME, DATE) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java index 77ceff1d564..c6a3e6197db 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchQueryRequest.java @@ -119,6 +119,14 @@ public OpenSearchQueryRequest( this.pitId = pitId; } + /** true if the request is a count aggregation request. */ + public boolean isCountAggRequest() { + return !searchDone + && sourceBuilder.size() == 0 + && sourceBuilder.trackTotalHitsUpTo() != null // only set in v3 + && sourceBuilder.trackTotalHitsUpTo() == Integer.MAX_VALUE; + } + /** * Constructs OpenSearchQueryRequest from serialized representation. * @@ -163,14 +171,18 @@ public OpenSearchResponse search( if (this.pitId == null) { // When SearchRequest doesn't contain PitId, fetch single page request if (searchDone) { - return new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + return new OpenSearchResponse( + SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { + // get the value before set searchDone = true + boolean isCountAggRequest = isCountAggRequest(); searchDone = true; return new OpenSearchResponse( searchAction.apply( new SearchRequest().indices(indexName.getIndexNames()).source(sourceBuilder)), exprValueFactory, - includes); + includes, + isCountAggRequest); } } else { // Search with PIT instead of scroll API @@ -181,7 +193,9 @@ public OpenSearchResponse search( public OpenSearchResponse searchWithPIT(Function searchAction) { OpenSearchResponse openSearchResponse; if (searchDone) { - openSearchResponse = new OpenSearchResponse(SearchHits.empty(), exprValueFactory, includes); + openSearchResponse = + new OpenSearchResponse( + SearchHits.empty(), exprValueFactory, includes, isCountAggRequest()); } else { this.sourceBuilder.pointInTimeBuilder(new PointInTimeBuilder(this.pitId)); this.sourceBuilder.timeout(cursorKeepAlive); @@ -200,7 +214,9 @@ public OpenSearchResponse searchWithPIT(Function new SearchRequest().indices(indexName.getIndexNames()).source(this.sourceBuilder); this.searchResponse = searchAction.apply(searchRequest); - openSearchResponse = new OpenSearchResponse(this.searchResponse, exprValueFactory, includes); + openSearchResponse = + new OpenSearchResponse( + this.searchResponse, exprValueFactory, includes, isCountAggRequest()); needClean = openSearchResponse.isEmpty(); searchDone = openSearchResponse.isEmpty(); @@ -226,6 +242,18 @@ public void clean(Consumer cleanAction) { } } + @Override + public void forceClean(Consumer cleanAction) { + try { + if (this.pitId != null) { + cleanAction.accept(this.pitId); + searchDone = true; + } + } finally { + this.pitId = null; + } + } + @Override public boolean hasAnotherBatch() { if (this.pitId != null) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java index f775d552962..93d07a83843 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequest.java @@ -43,6 +43,9 @@ OpenSearchResponse search( */ void clean(Consumer cleanAction); + /** Force clean the request. */ + void forceClean(Consumer cleanAction); + /** * Get the OpenSearchExprValueFactory. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java index b9beec5e1f5..9cc9bafc531 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchRequestBuilder.java @@ -48,6 +48,7 @@ import org.opensearch.sql.opensearch.client.OpenSearchClient; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.value.OpenSearchExprValueFactory; +import org.opensearch.sql.opensearch.response.agg.CountAsTotalHitsParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; /** OpenSearch search request builder. */ @@ -199,6 +200,9 @@ public void pushDownAggregation( if (sourceBuilder.sorts() != null) { sourceBuilder.sorts().clear(); } + if (aggregationBuilder.getRight() instanceof CountAsTotalHitsParser) { + sourceBuilder.trackTotalHits(true); + } } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java index d793b53fcac..be5c1f0baf9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/OpenSearchScrollRequest.java @@ -102,7 +102,7 @@ public OpenSearchResponse search( openSearchResponse = searchAction.apply(initialSearchRequest); } - var response = new OpenSearchResponse(openSearchResponse, exprValueFactory, includes); + var response = new OpenSearchResponse(openSearchResponse, exprValueFactory, includes, false); needClean = response.isEmpty(); if (!needClean) { setScrollId(openSearchResponse.getScrollId()); @@ -123,6 +123,12 @@ public void clean(Consumer cleanAction) { } } + @Override + public void forceClean(Consumer cleanAction) { + throw new UnsupportedOperationException( + "Force clean is unsupported in OpenSearchScrollRequest"); + } + /** * Is scroll started which means pages after first is being requested. * diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java index 0c5ccdf580c..ecfe9006036 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/request/PredicateAnalyzer.java @@ -24,6 +24,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.opensearch.sql.opensearch.request; import static com.google.common.base.Preconditions.checkArgument; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 83e7fd7721c..4f3e37ac4e6 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -48,6 +48,8 @@ public class OpenSearchResponse implements Iterable { /** List of requested include fields. */ private final List includes; + private final boolean isCountAgg; + /** OpenSearchExprValueFactory used to build ExprValue from search result. */ @EqualsAndHashCode.Exclude private final OpenSearchExprValueFactory exprValueFactory; @@ -56,19 +58,33 @@ public OpenSearchResponse( SearchResponse searchResponse, OpenSearchExprValueFactory exprValueFactory, List includes) { + this(searchResponse, exprValueFactory, includes, false); + } + + /** Constructor of OpenSearchResponse. */ + public OpenSearchResponse( + SearchResponse searchResponse, + OpenSearchExprValueFactory exprValueFactory, + List includes, + boolean isCountAgg) { this.hits = searchResponse.getHits(); this.aggregations = searchResponse.getAggregations(); this.exprValueFactory = exprValueFactory; this.includes = includes; + this.isCountAgg = isCountAgg; } /** Constructor of OpenSearchResponse with SearchHits. */ public OpenSearchResponse( - SearchHits hits, OpenSearchExprValueFactory exprValueFactory, List includes) { + SearchHits hits, + OpenSearchExprValueFactory exprValueFactory, + List includes, + boolean isCountAgg) { this.hits = hits; this.aggregations = null; this.exprValueFactory = exprValueFactory; this.includes = includes; + this.isCountAgg = isCountAgg; } /** @@ -78,20 +94,31 @@ public OpenSearchResponse( * @return true for empty */ public boolean isEmpty() { - return (hits.getHits() == null) || (hits.getHits().length == 0) && aggregations == null; + return (hits.getHits() == null) + || (((hits.getHits().length == 0) && aggregations == null) + && (!isCountAgg + || hits.getTotalHits() == null)); // check total hits if is count aggregation } public boolean isAggregationResponse() { return aggregations != null; } + public boolean isCountResponse() { + return isCountAgg; + } + + public int getHitsSize() { + return hits.getHits() == null ? 0 : hits.getHits().length; + } + /** * Make response iterable without need to return internal data structure explicitly. * * @return search hit iterator */ public Iterator iterator() { - if (isAggregationResponse()) { + if (isAggregationResponse() || isCountAgg) { return handleAggregationResponse(); } else { return Arrays.stream(hits.getHits()) @@ -191,7 +218,11 @@ private void addMetaDataFieldsToBuilder( * @return Parsed and built return values from response. */ private Iterator handleAggregationResponse() { - return exprValueFactory.getParser().parse(aggregations).stream() + List> res = + isCountAgg + ? exprValueFactory.getParser().parse(hits) + : exprValueFactory.getParser().parse(aggregations); + return res.stream() .map( entry -> { ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java index 149480b39ab..f43743d2e28 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/BucketAggregationParser.java @@ -11,6 +11,7 @@ import java.util.Map; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregation; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.MultiBucketsAggregation; @@ -44,4 +45,10 @@ private Map parse(MultiBucketsAggregation.Bucket bucket, String resultMap.putAll(metricsParser.parse(bucket.getAggregations())); return resultMap; } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "BucketAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java index 3f36ffa0f8e..57941311d44 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CompositeAggregationParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; @@ -20,6 +12,7 @@ import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.Getter; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.search.aggregations.bucket.composite.CompositeAggregation; @@ -33,6 +26,8 @@ public class CompositeAggregationParser implements OpenSearchAggregationResponseParser { private final MetricParserHelper metricsParser; + // countAggNameList dedicated the list of count aggregations which are filled by doc_count + private List countAggNameList = List.of(); public CompositeAggregationParser(MetricParser... metricParserList) { metricsParser = new MetricParserHelper(Arrays.asList(metricParserList)); @@ -42,6 +37,13 @@ public CompositeAggregationParser(List metricParserList) { metricsParser = new MetricParserHelper(metricParserList); } + /** CompositeAggregationParser with count aggregation name list, used in v3 */ + public CompositeAggregationParser( + List metricParserList, List countAggNameList) { + metricsParser = new MetricParserHelper(metricParserList); + this.countAggNameList = countAggNameList; + } + @Override public List> parse(Aggregations aggregations) { return ((CompositeAggregation) aggregations.asList().get(0)) @@ -52,6 +54,13 @@ private Map parse(CompositeAggregation.Bucket bucket) { Map resultMap = new HashMap<>(); resultMap.putAll(bucket.getKey()); resultMap.putAll(metricsParser.parse(bucket.getAggregations())); + countAggNameList.forEach(name -> resultMap.put(name, bucket.getDocCount())); return resultMap; } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "CompositeAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java new file mode 100644 index 00000000000..b30e588f725 --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/CountAsTotalHitsParser.java @@ -0,0 +1,40 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.response.agg; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import org.opensearch.search.SearchHits; +import org.opensearch.search.aggregations.Aggregations; + +@Getter +@EqualsAndHashCode +public class CountAsTotalHitsParser implements OpenSearchAggregationResponseParser { + + // countAggNameList dedicated the list of count aggregations which are filled by hits.total.value + private final List countAggNameList; + + public CountAsTotalHitsParser(List countAggNameList) { + this.countAggNameList = countAggNameList; + } + + @Override + public List> parse(Aggregations aggregations) { + throw new UnsupportedOperationException( + "CountAsTotalHitsParser doesn't support parse(Aggregations)"); + } + + @Override + public List> parse(SearchHits hits) { + Map resultMap = new HashMap<>(); + countAggNameList.forEach(name -> resultMap.put(name, hits.getTotalHits().value())); + return Collections.singletonList(resultMap); + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/FilterParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/FilterParser.java index 406f2797849..95571df30a7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/FilterParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/FilterParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParser.java index 0f8f8e284b7..04781d2a94a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java index 527748c8077..8886668abb0 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/MetricParserHelper.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; @@ -31,10 +23,20 @@ public class MetricParserHelper { private final Map metricParserMap; + // countAggNameList dedicated the list of count aggregations which are filled by doc_count + private final List countAggNameList; public MetricParserHelper(List metricParserList) { metricParserMap = metricParserList.stream().collect(Collectors.toMap(MetricParser::getName, m -> m)); + this.countAggNameList = List.of(); + } + + /** MetricParserHelper with count aggregation name list, used in v3 */ + public MetricParserHelper(List metricParserList, List countAggNameList) { + metricParserMap = + metricParserList.stream().collect(Collectors.toMap(MetricParser::getName, m -> m)); + this.countAggNameList = countAggNameList; } /** @@ -54,6 +56,7 @@ public Map parse(Aggregations aggregations) { "couldn't parse field %s in aggregation response", aggregation.getName())); } } + // countAggNameList.forEach(name -> resultMap.put(name, bucket.getDocCount())); return resultMap; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java index c0a45cba5dc..df8dcdd4ce9 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/NoBucketAggregationParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; @@ -18,6 +10,7 @@ import java.util.List; import java.util.Map; import lombok.Getter; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; /** No Bucket Aggregation Parser which include only metric parsers. */ @@ -38,4 +31,10 @@ public NoBucketAggregationParser(List metricParserList) { public List> parse(Aggregations aggregations) { return Collections.singletonList(metricsParser.parse(aggregations)); } + + @Override + public List> parse(SearchHits hits) { + throw new UnsupportedOperationException( + "NoBucketAggregationParser doesn't support parse(SearchHits)"); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java index 0c15d72eb60..564209336d4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/OpenSearchAggregationResponseParser.java @@ -1,20 +1,13 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; import java.util.List; import java.util.Map; +import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; /** OpenSearch Aggregation Response Parser. */ @@ -27,4 +20,6 @@ public interface OpenSearchAggregationResponseParser { * @return aggregation result. */ List> parse(Aggregations aggregations); + + List> parse(SearchHits hit); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/PercentilesParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/PercentilesParser.java index 86ed735b4a6..c9d78a94418 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/PercentilesParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/PercentilesParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SinglePercentileParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SinglePercentileParser.java index 94a70302af3..9665d863fc7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SinglePercentileParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SinglePercentileParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java index 1492fedfc2e..5487f9ca605 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/SingleValueParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java index 82a2f8648f1..ccda391c5eb 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/StatsParser.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java index 9ce46c6de6e..cd05dd8a2b7 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/agg/Utils.java @@ -1,14 +1,6 @@ /* - * Licensed under the Apache License, Version 2.0 (the "License"). - * You may not use this file except in compliance with the License. - * A copy of the License is located at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * or in the "license" file accompanying this file. This file is distributed - * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either - * express or implied. See the License for the specific language governing - * permissions and limitations under the License. + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.opensearch.response.agg; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index 53da492ba4d..14f5b099d9f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -114,6 +114,14 @@ public class OpenSearchSettings extends Settings { Setting.Property.NodeScope, Setting.Property.Dynamic); + public static final Setting PPL_VALUES_MAX_LIMIT_SETTING = + Setting.intSetting( + Key.PPL_VALUES_MAX_LIMIT.getKeyValue(), + 0, + 0, + Setting.Property.NodeScope, + Setting.Property.Dynamic); + public static final Setting CALCITE_ENGINE_ENABLED_SETTING = Setting.boolSetting( Key.CALCITE_ENGINE_ENABLED.getKeyValue(), @@ -361,6 +369,12 @@ public OpenSearchSettings(ClusterSettings clusterSettings) { Key.PPL_REX_MAX_MATCH_LIMIT, PPL_REX_MAX_MATCH_LIMIT_SETTING, new Updater(Key.PPL_REX_MAX_MATCH_LIMIT)); + register( + settingBuilder, + clusterSettings, + Key.PPL_VALUES_MAX_LIMIT, + PPL_VALUES_MAX_LIMIT_SETTING, + new Updater(Key.PPL_VALUES_MAX_LIMIT)); register( settingBuilder, clusterSettings, @@ -574,6 +588,7 @@ public static List> pluginSettings() { .add(DEFAULT_PATTERN_MAX_SAMPLE_COUNT_SETTING) .add(DEFAULT_PATTERN_BUFFER_LIMIT_SETTING) .add(PPL_REX_MAX_MATCH_LIMIT_SETTING) + .add(PPL_VALUES_MAX_LIMIT_SETTING) .add(QUERY_MEMORY_LIMIT_SETTING) .add(QUERY_SIZE_LIMIT_SETTING) .add(METRICS_ROLLING_WINDOW_SETTING) diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java index 91235be54c5..aaefef5b099 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/AbstractCalciteIndexScan.java @@ -7,7 +7,6 @@ import static java.util.Objects.requireNonNull; import static org.opensearch.sql.common.setting.Settings.Key.CALCITE_PUSHDOWN_ROWCOUNT_ESTIMATION_FACTOR; -import static org.opensearch.sql.opensearch.request.AggregateAnalyzer.AGGREGATION_BUCKET_SIZE; import java.util.ArrayDeque; import java.util.ArrayList; @@ -191,6 +190,10 @@ public boolean add(PushDownAction pushDownAction) { } return super.add(pushDownAction); } + + public boolean containsDigest(Object digest) { + return this.stream().anyMatch(action -> action.digest.equals(digest)); + } } protected abstract AbstractCalciteIndexScan buildScan( @@ -286,7 +289,8 @@ public AbstractCalciteIndexScan pushDownSort(List collations) Object digest; if (pushDownContext.isAggregatePushed) { // Push down the sort into the aggregation bucket - this.pushDownContext.aggPushDownAction.pushDownSortIntoAggBucket(collations); + this.pushDownContext.aggPushDownAction.pushDownSortIntoAggBucket( + collations, getRowType().getFieldNames()); action = requestBuilder -> {}; digest = collations; } else { @@ -373,19 +377,24 @@ public String toString() { } } + // TODO: shall we do deep copy for this action since it's mutable? public static class AggPushDownAction implements AbstractAction { private Pair, OpenSearchAggregationResponseParser> aggregationBuilder; private final Map extendedTypeMapping; @Getter private final boolean isScriptPushed; + // Record the output field names of all buckets as the sequence of buckets + private List bucketNames; public AggPushDownAction( Pair, OpenSearchAggregationResponseParser> aggregationBuilder, - Map extendedTypeMapping) { + Map extendedTypeMapping, + List bucketNames) { this.aggregationBuilder = aggregationBuilder; this.extendedTypeMapping = extendedTypeMapping; this.isScriptPushed = aggregationBuilder.getLeft().stream().anyMatch(this::isScriptAggBuilder); + this.bucketNames = bucketNames; } private boolean isScriptAggBuilder(AggregationBuilder aggBuilder) { @@ -399,34 +408,54 @@ public void apply(OpenSearchRequestBuilder requestBuilder) { requestBuilder.pushTypeMapping(extendedTypeMapping); } - public void pushDownSortIntoAggBucket(List collations) { + public void pushDownSortIntoAggBucket( + List collations, List fieldNames) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return; AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); - List selected = new ArrayList<>(collations.size()); + List selected = new ArrayList<>(collations.size()); if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { // It will always use a single CompositeAggregationBuilder for the aggregation with GroupBy // See {@link AggregateAnalyzer} List> buckets = compositeAggBuilder.sources(); List> newBuckets = new ArrayList<>(buckets.size()); + List newBucketNames = new ArrayList<>(buckets.size()); // Have to put the collation required buckets first, then the rest of buckets. collations.forEach( collation -> { - CompositeValuesSourceBuilder bucket = buckets.get(collation.getFieldIndex()); + /* + Must find the bucket by field name because: + 1. The sequence of buckets may have changed after sort push-down. + 2. The schema of scan operator may be inconsistent with the sequence of buckets + after project push-down. + */ + String bucketName = fieldNames.get(collation.getFieldIndex()); + CompositeValuesSourceBuilder bucket = buckets.get(bucketNames.indexOf(bucketName)); Direction direction = collation.getDirection(); NullDirection nullDirection = collation.nullDirection; SortOrder order = Direction.DESCENDING.equals(direction) ? SortOrder.DESC : SortOrder.ASC; - MissingOrder missingOrder = - switch (nullDirection) { - case FIRST -> MissingOrder.FIRST; - case LAST -> MissingOrder.LAST; - default -> MissingOrder.DEFAULT; - }; - newBuckets.add(bucket.order(order).missingOrder(missingOrder)); - selected.add(collation.getFieldIndex()); + if (bucket.missingBucket()) { + MissingOrder missingOrder = + switch (nullDirection) { + case FIRST -> MissingOrder.FIRST; + case LAST -> MissingOrder.LAST; + default -> MissingOrder.DEFAULT; + }; + bucket.missingOrder(missingOrder); + } + newBuckets.add(bucket.order(order)); + newBucketNames.add(bucketName); + selected.add(bucketName); }); IntStream.range(0, buckets.size()) - .filter(i -> !selected.contains(i)) - .forEach(i -> newBuckets.add(buckets.get(i))); + .mapToObj(fieldNames::get) + .filter(name -> !selected.contains(name)) + .forEach( + name -> { + newBuckets.add(buckets.get(bucketNames.indexOf(name))); + newBucketNames.add(name); + }); Builder newAggBuilder = new Builder(); compositeAggBuilder.getSubAggregations().forEach(newAggBuilder::addAggregator); aggregationBuilder = @@ -434,8 +463,9 @@ public void pushDownSortIntoAggBucket(List collations) { Collections.singletonList( AggregationBuilders.composite("composite_buckets", newBuckets) .subAggregations(newAggBuilder) - .size(AGGREGATION_BUCKET_SIZE)), + .size(compositeAggBuilder.size())), aggregationBuilder.getRight()); + bucketNames = newBucketNames; } if (builder instanceof TermsAggregationBuilder termsAggBuilder) { termsAggBuilder.order( @@ -449,6 +479,8 @@ public void pushDownSortIntoAggBucket(List collations) { * than bucket number. */ public boolean pushDownLimitIntoBucketSize(Integer size) { + // aggregationBuilder.getLeft() could be empty when count agg optimization works + if (aggregationBuilder.getLeft().isEmpty()) return false; AggregationBuilder builder = aggregationBuilder.getLeft().getFirst(); if (builder instanceof CompositeAggregationBuilder compositeAggBuilder) { if (size < compositeAggBuilder.size()) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java index 33b80f61982..0fd326306dc 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteEnumerableIndexScan.java @@ -111,6 +111,7 @@ public Enumerator enumerator() { osIndex.getClient(), getFieldPath(), requestBuilder.getMaxResponseSize(), + requestBuilder.getMaxResultWindow(), osIndex.buildRequest(requestBuilder), osIndex.createOpenSearchResourceMonitor()); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java index 21ea41fa3af..3cdf21ddfaa 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/CalciteLogicalIndexScan.java @@ -25,6 +25,7 @@ import org.apache.calcite.rel.core.Filter; import org.apache.calcite.rel.core.Project; import org.apache.calcite.rel.hint.RelHint; +import org.apache.calcite.rel.logical.LogicalFilter; import org.apache.calcite.rel.logical.LogicalSort; import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rel.type.RelDataTypeFactory; @@ -32,10 +33,13 @@ import org.apache.calcite.rex.RexBuilder; import org.apache.calcite.rex.RexNode; import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; import org.apache.commons.lang3.tuple.Pair; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.search.aggregations.AggregationBuilder; +import org.opensearch.search.aggregations.bucket.histogram.AutoDateHistogramAggregationBuilder; +import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.data.type.ExprCoreType; @@ -49,6 +53,9 @@ import org.opensearch.sql.opensearch.request.PredicateAnalyzer.QueryExpression; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.storage.OpenSearchIndex; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.LimitDigest; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.PushDownAction; +import org.opensearch.sql.opensearch.storage.scan.AbstractCalciteIndexScan.PushDownType; /** The logical relational operator representing a scan of an OpenSearchIndex type. */ @Getter @@ -210,6 +217,12 @@ public CalciteLogicalIndexScan pushDownProject(List selectedColumns) { } RelDataType newSchema = builder.build(); + // To prevent circular pushdown for some edge cases where plan has pattern(see TPCH Q1): + // `Project($1, $0) -> Sort(sort0=[1]) -> Project($1, $0) -> Aggregate(group={0},...)...` + // We don't support pushing down the duplicated project here otherwise it will cause dead-loop. + // `ProjectMergeRule` will help merge the duplicated projects. + if (this.getPushDownContext().containsDigest(newSchema.getFieldNames())) return null; + // Projection may alter indicies in the collations. // E.g. When sorting age // `Project(age) - TableScan(schema=[name, age], collation=[$1 ASC])` should become @@ -226,18 +239,21 @@ public CalciteLogicalIndexScan pushDownProject(List selectedColumns) { newSchema, pushDownContext.clone()); - Map aliasMapping = this.osIndex.getAliasMapping(); - // For alias types, we need to push down its original path instead of the alias name. - List projectedFields = - newSchema.getFieldNames().stream() - .map(fieldName -> aliasMapping.getOrDefault(fieldName, fieldName)) - .toList(); - + AbstractAction action; + if (pushDownContext.isAggregatePushed()) { + // For aggregate, we do nothing on query builder but only change the schema of the scan. + action = requestBuilder -> {}; + } else { + Map aliasMapping = this.osIndex.getAliasMapping(); + // For alias types, we need to push down its original path instead of the alias name. + List projectedFields = + newSchema.getFieldNames().stream() + .map(fieldName -> aliasMapping.getOrDefault(fieldName, fieldName)) + .toList(); + action = requestBuilder -> requestBuilder.pushDownProjectStream(projectedFields.stream()); + } newScan.pushDownContext.add( - PushDownAction.of( - PushDownType.PROJECT, - newSchema.getFieldNames(), - requestBuilder -> requestBuilder.pushDownProjectStream(projectedFields.stream()))); + PushDownAction.of(PushDownType.PROJECT, newSchema.getFieldNames(), action)); return newScan; } @@ -259,7 +275,7 @@ private RelTraitSet reIndexCollations(List selectedColumns) { return newTraitSet; } - public CalciteLogicalIndexScan pushDownAggregate(Aggregate aggregate, Project project) { + public AbstractRelNode pushDownAggregate(Aggregate aggregate, Project project) { try { CalciteLogicalIndexScan newScan = new CalciteLogicalIndexScan( @@ -285,8 +301,30 @@ public CalciteLogicalIndexScan pushDownAggregate(Aggregate aggregate, Project pr OpenSearchDataType.of( OpenSearchTypeFactory.convertRelDataTypeToExprType( field.getType())))); - AggPushDownAction action = new AggPushDownAction(aggregationBuilder, extendedTypeMapping); + AggPushDownAction action = + new AggPushDownAction( + aggregationBuilder, + extendedTypeMapping, + outputFields.subList(0, aggregate.getGroupSet().length())); newScan.pushDownContext.add(PushDownAction.of(PushDownType.AGGREGATION, aggregate, action)); + if (aggregationBuilder.getLeft().size() == 1 + && aggregationBuilder.getLeft().getFirst() + instanceof AutoDateHistogramAggregationBuilder autoDateHistogram) { + // If it's auto_date_histogram, filter the empty bucket by using the first aggregate metrics + RexBuilder rexBuilder = getCluster().getRexBuilder(); + AggregationBuilder aggregationBuilders = + autoDateHistogram.getSubAggregations().stream().toList().getFirst(); + RexNode condition = + aggregationBuilders instanceof ValueCountAggregationBuilder + ? rexBuilder.makeCall( + SqlStdOperatorTable.GREATER_THAN, + rexBuilder.makeInputRef(newScan, 1), + rexBuilder.makeLiteral( + 0, rexBuilder.getTypeFactory().createSqlType(SqlTypeName.INTEGER))) + : rexBuilder.makeCall( + SqlStdOperatorTable.IS_NOT_NULL, rexBuilder.makeInputRef(newScan, 1)); + return LogicalFilter.create(newScan, condition); + } return newScan; } catch (Exception e) { if (LOG.isDebugEnabled()) { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java index 384ec17b470..c4118a965da 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexEnumerator.java @@ -34,11 +34,14 @@ public class OpenSearchIndexEnumerator implements Enumerator { private final List fields; /** Search request. */ - @EqualsAndHashCode.Include @ToString.Include private final OpenSearchRequest request; + @EqualsAndHashCode.Include @ToString.Include private OpenSearchRequest request; /** Largest number of rows allowed in the response. */ @EqualsAndHashCode.Include @ToString.Include private final int maxResponseSize; + /** Largest number of rows allowed in the response. */ + @EqualsAndHashCode.Include @ToString.Include private final int maxResultWindow; + /** How many moveNext() calls to perform resource check once. */ private static final long NUMBER_OF_NEXT_CALL_TO_CHECK = 1000; @@ -53,16 +56,21 @@ public class OpenSearchIndexEnumerator implements Enumerator { private ExprValue current; + /** flag to indicate whether fetch more than one batch */ + private boolean fetchOnce = false; + public OpenSearchIndexEnumerator( OpenSearchClient client, List fields, int maxResponseSize, + int maxResultWindow, OpenSearchRequest request, ResourceMonitor monitor) { this.client = client; this.fields = fields; this.request = request; this.maxResponseSize = maxResponseSize; + this.maxResultWindow = maxResultWindow; this.monitor = monitor; this.queryCount = 0; this.current = null; @@ -73,6 +81,13 @@ public OpenSearchIndexEnumerator( private void fetchNextBatch() { OpenSearchResponse response = client.search(request); + if (response.isAggregationResponse() + || response.isCountResponse() + || response.getHitsSize() < maxResultWindow) { + // no need to fetch next batch if it's for an aggregation + // or the length of response hits is less than max result window size. + fetchOnce = true; + } if (!response.isEmpty()) { iterator = response.iterator(); } else if (iterator == null) { @@ -106,7 +121,7 @@ public boolean moveNext() { throw new NonFallbackCalciteException("insufficient resources to load next row, quit."); } - if (iterator == null || !iterator.hasNext()) { + if (iterator == null || (!iterator.hasNext() && !fetchOnce)) { fetchNextBatch(); } if (iterator.hasNext()) { @@ -132,7 +147,9 @@ public void reset() { @Override public void close() { iterator = Collections.emptyIterator(); - queryCount = 0; - client.cleanup(request); + if (request != null) { + client.forceCleanup(request); + request = null; + } } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java index 6a8426749ae..a03fb6268ba 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/CalciteScriptEngine.java @@ -30,6 +30,7 @@ import static org.opensearch.sql.data.type.ExprCoreType.BYTE; import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.IP; import static org.opensearch.sql.data.type.ExprCoreType.SHORT; import com.google.common.collect.ImmutableList; @@ -79,6 +80,8 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.search.lookup.SourceLookup; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.data.model.ExprIpValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; @@ -128,8 +131,7 @@ public T compile( Map fieldTypes = (Map) objectMap.get(RelJsonSerializer.FIELD_TYPES); - JavaTypeFactoryImpl typeFactory = - new JavaTypeFactoryImpl(relJsonSerializer.getCluster().getTypeFactory().getTypeSystem()); + JavaTypeFactory typeFactory = OpenSearchTypeFactory.TYPE_FACTORY; RexToLixTranslator.InputGetter getter = new ScriptInputGetter(typeFactory, rowType, fieldTypes); String code = CalciteScriptEngine.translate( @@ -212,6 +214,10 @@ private Expression tryConvertDocValue(Expression docValueExpr, ExprType exprType return switch (exprType) { case INTEGER, SHORT, BYTE -> EnumUtils.convert(docValueExpr, Long.class); case FLOAT -> EnumUtils.convert(docValueExpr, Double.class); + // IP is scanned in as a string but used as ExprIpValue later. We call the constructor + // beforehand. + case IP -> Expressions.new_( + ExprIpValue.class, EnumUtils.convert(docValueExpr, String.class)); default -> docValueExpr; }; } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java index 562e38ae79e..408511fde3f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilder.java @@ -29,12 +29,10 @@ import org.opensearch.sql.expression.ReferenceExpression; import org.opensearch.sql.expression.aggregation.NamedAggregator; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; -import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.MetricParser; import org.opensearch.sql.opensearch.response.agg.NoBucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; -import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.BucketAggregationBuilder; import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.CompositeAggregationBuilder; import org.opensearch.sql.opensearch.storage.script.aggregation.dsl.MetricAggregationBuilder; import org.opensearch.sql.opensearch.storage.serde.ExpressionSerializer; @@ -49,9 +47,6 @@ public class AggregationQueryBuilder extends ExpressionNodeVisitor buildHistogram( String name, String field, Double value, SpanUnit unit) { switch (unit) { case NONE: - HistogramAggregationBuilder builder = new HistogramAggregationBuilder(name); - builder.field(field).interval(value); - return builder; + return new HistogramAggregationBuilder(name).field(field).interval(value); case UNKNOWN: throw new IllegalStateException("Invalid span unit"); default: @@ -102,11 +101,14 @@ public static ValuesSourceAggregationBuilder buildHistogram( } } + public static ValuesSourceAggregationBuilder buildAutoDateHistogram( + String name, String field, Integer bucketSize) { + return new AutoDateHistogramAggregationBuilder(name).field(field).setNumBuckets(bucketSize); + } + public static ValuesSourceAggregationBuilder buildDateHistogram( String name, String field, Integer value, SpanUnit unit) { String spanValue = value + unit.getName(); - DateHistogramAggregationBuilder builder = new DateHistogramAggregationBuilder(name); - builder.field(field); switch (unit) { case MILLISECOND: case MS: @@ -118,11 +120,13 @@ public static ValuesSourceAggregationBuilder buildDateHistogram( case H: case DAY: case D: - builder.fixedInterval(new DateHistogramInterval(spanValue)); - break; + return new DateHistogramAggregationBuilder(name) + .field(field) + .fixedInterval(new DateHistogramInterval(spanValue)); default: - builder.calendarInterval(new DateHistogramInterval(spanValue)); + return new DateHistogramAggregationBuilder(name) + .field(field) + .calendarInterval(new DateHistogramInterval(spanValue)); } - return builder; } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilder.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilder.java index e569bbabbf1..954bbc93f8e 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilder.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilder.java @@ -37,20 +37,23 @@ public CompositeAggregationBuilder(ExpressionSerializer serializer) { /** Build the list of CompositeValuesSourceBuilder. */ public List> build( - List> groupList) { + List> groupList, boolean bucketNullable) { ImmutableList.Builder> resultBuilder = new ImmutableList.Builder<>(); for (Triple groupPair : groupList) { resultBuilder.add( buildCompositeValuesSourceBuilder( - groupPair.getLeft(), groupPair.getMiddle(), groupPair.getRight())); + groupPair.getLeft(), groupPair.getMiddle(), groupPair.getRight(), bucketNullable)); } return resultBuilder.build(); } // todo, Expression should implement buildCompositeValuesSourceBuilder() interface. private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( - NamedExpression expr, SortOrder sortOrder, MissingOrder missingOrder) { + NamedExpression expr, + SortOrder sortOrder, + MissingOrder missingOrder, + boolean bucketNullable) { if (expr.getDelegated() instanceof SpanExpression) { SpanExpression spanExpr = (SpanExpression) expr.getDelegated(); return buildHistogram( @@ -58,13 +61,14 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( spanExpr.getField().toString(), spanExpr.getValue().valueOf().doubleValue(), spanExpr.getUnit(), - missingOrder); + missingOrder, + bucketNullable); } else { CompositeValuesSourceBuilder sourceBuilder = - new TermsValuesSourceBuilder(expr.getName()) - .missingBucket(true) - .missingOrder(missingOrder) - .order(sortOrder); + new TermsValuesSourceBuilder(expr.getName()).order(sortOrder); + if (bucketNullable) { + sourceBuilder.missingBucket(true).missingOrder(missingOrder); + } // Time types values are converted to LONG in ExpressionAggregationScript::execute if ((expr.getDelegated().type() instanceof OpenSearchDateType && List.of(TIMESTAMP, TIME, DATE) @@ -77,23 +81,29 @@ private CompositeValuesSourceBuilder buildCompositeValuesSourceBuilder( } public static CompositeValuesSourceBuilder buildHistogram( - String name, String field, Double value, SpanUnit unit, MissingOrder missingOrder) { + String name, + String field, + Double value, + SpanUnit unit, + MissingOrder missingOrder, + boolean bucketNullable) { switch (unit) { case NONE: - return new HistogramValuesSourceBuilder(name) - .field(field) - .interval(value) - .missingBucket(true) - .missingOrder(missingOrder); + HistogramValuesSourceBuilder histogramBuilder = + new HistogramValuesSourceBuilder(name).field(field).interval(value); + if (bucketNullable) { + histogramBuilder.missingBucket(true).missingOrder(missingOrder); + } + return histogramBuilder; case UNKNOWN: throw new IllegalStateException("Invalid span unit"); default: - return buildDateHistogram(name, field, value.intValue(), unit, missingOrder); + return buildDateHistogram(name, field, value.intValue(), unit); } } public static CompositeValuesSourceBuilder buildDateHistogram( - String name, String field, Integer value, SpanUnit unit, MissingOrder missingOrder) { + String name, String field, Integer value, SpanUnit unit) { String spanValue = value + unit.getName(); switch (unit) { case MILLISECOND: @@ -108,14 +118,10 @@ public static CompositeValuesSourceBuilder buildDateHistogram( case D: return new DateHistogramValuesSourceBuilder(name) .field(field) - .missingBucket(true) - .missingOrder(missingOrder) .fixedInterval(new DateHistogramInterval(spanValue)); default: return new DateHistogramValuesSourceBuilder(name) .field(field) - .missingBucket(true) - .missingOrder(missingOrder) .calendarInterval(new DateHistogramInterval(spanValue)); } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java new file mode 100644 index 00000000000..c7451adbf8b --- /dev/null +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJson.java @@ -0,0 +1,805 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.serde; + +import static java.util.Objects.requireNonNull; +import static org.apache.calcite.util.Static.RESOURCE; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.calcite.avatica.AvaticaUtils; +import org.apache.calcite.avatica.util.TimeUnitRange; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelCollation; +import org.apache.calcite.rel.RelDistribution; +import org.apache.calcite.rel.RelFieldCollation; +import org.apache.calcite.rel.RelInput; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.AggregateCall; +import org.apache.calcite.rel.core.CorrelationId; +import org.apache.calcite.rel.core.TableModify; +import org.apache.calcite.rel.externalize.RelEnumTypes; +import org.apache.calcite.rel.externalize.RelJson; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeFactory; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexCorrelVariable; +import org.apache.calcite.rex.RexDynamicParam; +import org.apache.calcite.rex.RexFieldAccess; +import org.apache.calcite.rex.RexFieldCollation; +import org.apache.calcite.rex.RexLiteral; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexOver; +import org.apache.calcite.rex.RexSlot; +import org.apache.calcite.rex.RexUnknownAs; +import org.apache.calcite.rex.RexWindow; +import org.apache.calcite.rex.RexWindowBound; +import org.apache.calcite.rex.RexWindowBounds; +import org.apache.calcite.rex.RexWindowExclusion; +import org.apache.calcite.sql.JoinConditionType; +import org.apache.calcite.sql.JoinType; +import org.apache.calcite.sql.SqlAggFunction; +import org.apache.calcite.sql.SqlExplain; +import org.apache.calcite.sql.SqlExplainFormat; +import org.apache.calcite.sql.SqlExplainLevel; +import org.apache.calcite.sql.SqlFunction; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlInsertKeyword; +import org.apache.calcite.sql.SqlJsonConstructorNullClause; +import org.apache.calcite.sql.SqlJsonQueryWrapperBehavior; +import org.apache.calcite.sql.SqlJsonValueEmptyOrErrorBehavior; +import org.apache.calcite.sql.SqlKind; +import org.apache.calcite.sql.SqlMatchRecognize; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.SqlSelectKeyword; +import org.apache.calcite.sql.SqlSyntax; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.fun.SqlTrimFunction; +import org.apache.calcite.sql.parser.SqlParserPos; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.validate.SqlNameMatchers; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.calcite.util.JsonBuilder; +import org.apache.calcite.util.Sarg; +import org.checkerframework.checker.nullness.qual.Nullable; +import org.checkerframework.checker.nullness.qual.PolyNull; +import org.opensearch.sql.calcite.type.AbstractExprRelDataType; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; + +/** + * An extension to {@link RelJson} to allow serialization & deserialization of UDTs + * + *

    It replicates a lot of methods from {@link RelJson} because we can not override some private + * methods and because we cannot create an instance of ExtendedRelJson with the custom values for + * private final fields not included in public constructors of RelJson. For example, there is no + * public constructor with inputTranslator and operatorTable. As a result, every method that make + * use of these private fields has to be copied in ExtendedRelJson. + */ +public class ExtendedRelJson extends RelJson { + private final JsonBuilder jsonBuilder; + private final InputTranslator inputTranslator; + private final SqlOperatorTable operatorTable; + + /** + * Registry of enum classes that can be serialized to JSON, replicated from {@link RelEnumTypes} + * as toEnum(String) method is package private + */ + private static final ImmutableMap> ENUM_BY_NAME; + + static { + // Build a mapping from enum constants to the enum instances, same as RelEnumTypes + final ImmutableMap.Builder> enumByName = ImmutableMap.builder(); + registerEnum(enumByName, JoinConditionType.class); + registerEnum(enumByName, JoinType.class); + registerEnum(enumByName, RexUnknownAs.class); + registerEnum(enumByName, SqlExplain.Depth.class); + registerEnum(enumByName, SqlExplainFormat.class); + registerEnum(enumByName, SqlExplainLevel.class); + registerEnum(enumByName, SqlInsertKeyword.class); + registerEnum(enumByName, SqlJsonConstructorNullClause.class); + registerEnum(enumByName, SqlJsonQueryWrapperBehavior.class); + registerEnum(enumByName, SqlJsonValueEmptyOrErrorBehavior.class); + registerEnum(enumByName, SqlMatchRecognize.AfterOption.class); + registerEnum(enumByName, SqlSelectKeyword.class); + registerEnum(enumByName, SqlTrimFunction.Flag.class); + registerEnum(enumByName, TimeUnitRange.class); + registerEnum(enumByName, TableModify.Operation.class); + ENUM_BY_NAME = enumByName.build(); + } + + private static void registerEnum( + ImmutableMap.Builder> builder, Class> enumClass) { + for (Enum enumConstant : enumClass.getEnumConstants()) { + builder.put(enumConstant.name(), enumConstant); + } + } + + private ExtendedRelJson(JsonBuilder jsonBuilder) { + super(jsonBuilder); + this.jsonBuilder = jsonBuilder; + this.inputTranslator = null; + this.operatorTable = SqlStdOperatorTable.instance(); + } + + private ExtendedRelJson( + @Nullable JsonBuilder jsonBuilder, + InputTranslator inputTranslator, + SqlOperatorTable operatorTable) { + super(jsonBuilder); + this.jsonBuilder = jsonBuilder; + this.inputTranslator = requireNonNull(inputTranslator, "inputTranslator"); + this.operatorTable = requireNonNull(operatorTable, "operatorTable"); + } + + /** Creates a ExtendedRelJson. */ + public static ExtendedRelJson create(JsonBuilder jsonBuilder) { + return new ExtendedRelJson(jsonBuilder); + } + + @Override + public RelJson withInputTranslator(InputTranslator inputTranslator) { + if (inputTranslator == this.inputTranslator) { + return this; + } + return new ExtendedRelJson(jsonBuilder, inputTranslator, operatorTable); + } + + @Override + public RelJson withOperatorTable(SqlOperatorTable operatorTable) { + if (operatorTable == this.operatorTable) { + return this; + } + return new ExtendedRelJson(jsonBuilder, inputTranslator, operatorTable); + } + + @Override + public @Nullable Object toJson(@Nullable Object value) { + if (value instanceof RelDataTypeField) { + return toJson((RelDataTypeField) value); + } else if (value instanceof RelDataType) { + return toJson((RelDataType) value); + } + return super.toJson(value); + } + + // Copied from RelJson since many overrides of toJson are private, thus not overridable. + @Override + public Object toJson(RexNode node) { + final Map map; + switch (node.getKind()) { + case DYNAMIC_PARAM: + map = jsonBuilder().map(); + final RexDynamicParam rexDynamicParam = (RexDynamicParam) node; + final RelDataType rdpType = rexDynamicParam.getType(); + map.put("dynamicParam", rexDynamicParam.getIndex()); + map.put("type", toJson(rdpType)); + return map; + case FIELD_ACCESS: + map = jsonBuilder().map(); + final RexFieldAccess fieldAccess = (RexFieldAccess) node; + map.put("field", fieldAccess.getField().getName()); + map.put("expr", toJson(fieldAccess.getReferenceExpr())); + return map; + case LITERAL: + final RexLiteral literal = (RexLiteral) node; + final Object value = literal.getValue3(); + map = jsonBuilder().map(); + //noinspection rawtypes + map.put( + "literal", value instanceof Enum ? RelEnumTypes.fromEnum((Enum) value) : toJson(value)); + map.put("type", toJson(node.getType())); + return map; + case INPUT_REF: + // reduce copy when possible + // noinspection unchecked cast + map = (Map) super.toJson(node); + return map; + case LOCAL_REF: + map = jsonBuilder().map(); + map.put("input", ((RexSlot) node).getIndex()); + map.put("name", ((RexSlot) node).getName()); + map.put("type", toJson(node.getType())); + return map; + case CORREL_VARIABLE: + map = jsonBuilder().map(); + map.put("correl", ((RexCorrelVariable) node).getName()); + map.put("type", toJson(node.getType())); + return map; + default: + if (node instanceof RexCall) { + final RexCall call = (RexCall) node; + map = jsonBuilder().map(); + map.put("op", toJson(call.getOperator())); + final List<@Nullable Object> list = jsonBuilder().list(); + for (RexNode operand : call.getOperands()) { + list.add(toJson(operand)); + } + map.put("operands", list); + switch (node.getKind()) { + case MINUS: + case CAST: + case SAFE_CAST: + map.put("type", toJson(node.getType())); + break; + default: + break; + } + if (call.getOperator() instanceof SqlFunction) { + if (((SqlFunction) call.getOperator()).getFunctionType().isUserDefined()) { + SqlOperator op = call.getOperator(); + map.put("class", op.getClass().getName()); + map.put("type", toJson(node.getType())); + map.put("deterministic", op.isDeterministic()); + map.put("dynamic", op.isDynamicFunction()); + } + } + if (call instanceof RexOver) { + RexOver over = (RexOver) call; + map.put("distinct", over.isDistinct()); + map.put("type", toJson(node.getType())); + map.put("window", toJson(over.getWindow())); + } + return map; + } + throw new UnsupportedOperationException("unknown rex " + node); + } + } + + // Copied from RelJson as its private but used in toJson(RexNode) + private Object toJson(RelDataTypeField node) { + final Map map; + if (node.getType().isStruct()) { + map = jsonBuilder().map(); + map.put("fields", toJson(node.getType())); + map.put("nullable", node.getType().isNullable()); + } else { + //noinspection unchecked + map = (Map) toJson(node.getType()); + } + map.put("name", node.getName()); + return map; + } + + /** Modifies behavior for AbstractExprRelDataType instances, delegates to RelJson otherwise. */ + private Object toJson(RelDataType node) { + final Map map = jsonBuilder().map(); + if (node.isStruct()) { + final List<@Nullable Object> list = jsonBuilder().list(); + for (RelDataTypeField field : node.getFieldList()) { + list.add(toJson(field)); + } + map.put("fields", list); + map.put("nullable", node.isNullable()); + } else { + // For UDT like EXPR_TIMESTAMP, we additionally save its UDT info as a tag. + if (node instanceof AbstractExprRelDataType) { + map.put("udt", ((AbstractExprRelDataType) node).getUdt().name()); + } + map.put("type", node.getSqlTypeName().name()); + map.put("nullable", node.isNullable()); + if (node.getComponentType() != null) { + map.put("component", toJson(node.getComponentType())); + } + RelDataType keyType = node.getKeyType(); + if (keyType != null) { + map.put("key", toJson(keyType)); + } + RelDataType valueType = node.getValueType(); + if (valueType != null) { + map.put("value", toJson(valueType)); + } + if (node.getSqlTypeName().allowsPrec()) { + map.put("precision", node.getPrecision()); + } + if (node.getSqlTypeName().allowsScale()) { + map.put("scale", node.getScale()); + } + } + return map; + } + + // Copied from RelJson since its private but used in toJson(RexNode) + private Map toJson(SqlOperator operator) { + // User-defined operators are not yet handled. + Map map = jsonBuilder().map(); + map.put("name", operator.getName()); + map.put("kind", operator.kind.toString()); + map.put("syntax", operator.getSyntax().toString()); + return map; + } + + // Copied from RelJson since its private but used in toJson(RexNode) + private Object toJson(RexWindow window) { + final Map map = jsonBuilder().map(); + if (!window.partitionKeys.isEmpty()) { + map.put("partition", toJson(window.partitionKeys)); + } + if (!window.orderKeys.isEmpty()) { + map.put("order", toJson(window.orderKeys)); + } + if (window.getLowerBound() == null) { + // No ROWS or RANGE clause + } else if (window.getUpperBound() == null) { + if (window.isRows()) { + map.put("rows-lower", toJson(window.getLowerBound())); + } else { + map.put("range-lower", toJson(window.getLowerBound())); + } + } else { + if (window.isRows()) { + map.put("rows-lower", toJson(window.getLowerBound())); + map.put("rows-upper", toJson(window.getUpperBound())); + } else { + map.put("range-lower", toJson(window.getLowerBound())); + map.put("range-upper", toJson(window.getUpperBound())); + } + } + return map; + } + + /** + * Reconstruct a RelDataType from a json map. It overrides {@link + * RelJson#toType(RelDataTypeFactory, Object)} to handle the reconstruction of user-defined types + * (UDTs). + */ + @Override + public RelDataType toType(RelDataTypeFactory typeFactory, Object o) { + if (o instanceof Map + && ((Map) o).containsKey("udt") + && typeFactory instanceof OpenSearchTypeFactory) { + // Reconstruct UDT from its udt tag + Object udtName = ((Map) o).get("udt"); + OpenSearchTypeFactory.ExprUDT udt = OpenSearchTypeFactory.ExprUDT.valueOf((String) udtName); + return ((OpenSearchTypeFactory) typeFactory).createUDT(udt); + } + return super.toType(typeFactory, o); + } + + /** + * Reconstruct a RexNode from a Json object using the provided cluster context. + * + *

    This method overrides {@link RelJson#toRex(RelOptCluster, Object)} to use our custom {@code + * toRex(RelInput, Object)} implementation which supports UDT deserialization and uses our own + * {@code inputTranslator} and {@code operatorTable} instances. + */ + @Override + public RexNode toRex(RelOptCluster cluster, Object o) { + RelInput input = new RelInputForCluster(cluster); + return toRex(input, o); + } + + // Copied from RelJson for the access of custom inputTranslator and operatorTable + @SuppressWarnings({"rawtypes", "unchecked"}) + @PolyNull + RexNode toRex(RelInput relInput, @PolyNull Object o) { + final RelOptCluster cluster = relInput.getCluster(); + final RexBuilder rexBuilder = cluster.getRexBuilder(); + if (o == null) { + return null; + // Support JSON deserializing of non-default Map classes such as gson LinkedHashMap + } else if (Map.class.isAssignableFrom(o.getClass())) { + final Map map = (Map) o; + final RelDataTypeFactory typeFactory = cluster.getTypeFactory(); + if (map.containsKey("op")) { + final Map opMap = get(map, "op"); + if (map.containsKey("class")) { + opMap.put("class", get(map, "class")); + } + final List operands = get(map, "operands"); + final List rexOperands = toRexList(relInput, operands); + final Object jsonType = map.get("type"); + final Map window = (Map) map.get("window"); + if (window != null) { + final SqlAggFunction operator = requireNonNull(toAggregation(opMap), "operator"); + final RelDataType type = toType(typeFactory, requireNonNull(jsonType, "jsonType")); + List partitionKeys = new ArrayList<>(); + Object partition = window.get("partition"); + if (partition != null) { + partitionKeys = toRexList(relInput, (List) partition); + } + List orderKeys = new ArrayList<>(); + if (window.containsKey("order")) { + addRexFieldCollationList(orderKeys, relInput, (List) window.get("order")); + } + final RexWindowBound lowerBound; + final RexWindowBound upperBound; + final boolean physical; + if (window.get("rows-lower") != null) { + lowerBound = toRexWindowBound(relInput, (Map) window.get("rows-lower")); + upperBound = toRexWindowBound(relInput, (Map) window.get("rows-upper")); + physical = true; + } else if (window.get("range-lower") != null) { + lowerBound = toRexWindowBound(relInput, (Map) window.get("range-lower")); + upperBound = toRexWindowBound(relInput, (Map) window.get("range-upper")); + physical = false; + } else { + // No ROWS or RANGE clause + // Note: lower and upper bounds are non-nullable, so this branch is not reachable + lowerBound = null; + upperBound = null; + physical = false; + } + final RexWindowExclusion exclude; + if (window.get("exclude") != null) { + exclude = toRexWindowExclusion((Map) window.get("exclude")); + } else { + exclude = RexWindowExclusion.EXCLUDE_NO_OTHER; + } + final boolean distinct = get(map, "distinct"); + return rexBuilder.makeOver( + type, + operator, + rexOperands, + partitionKeys, + ImmutableList.copyOf(orderKeys), + requireNonNull(lowerBound, "lowerBound"), + requireNonNull(upperBound, "upperBound"), + requireNonNull(exclude, "exclude"), + physical, + true, + false, + distinct, + false); + } else { + final SqlOperator operator = requireNonNull(toOp(opMap), "operator"); + final RelDataType type; + if (jsonType != null) { + type = toType(typeFactory, jsonType); + } else { + type = rexBuilder.deriveReturnType(operator, rexOperands); + } + return rexBuilder.makeCall(type, operator, rexOperands); + } + } + final Integer input = (Integer) map.get("input"); + if (input != null) { + return inputTranslator.translateInput(this, input, map, relInput); + } + final String field = (String) map.get("field"); + if (field != null) { + final Object jsonExpr = get(map, "expr"); + final RexNode expr = toRex(relInput, jsonExpr); + return rexBuilder.makeFieldAccess(expr, field, true); + } + final String correl = (String) map.get("correl"); + if (correl != null) { + final Object jsonType = get(map, "type"); + RelDataType type = toType(typeFactory, jsonType); + return rexBuilder.makeCorrel(type, new CorrelationId(correl)); + } + if (map.containsKey("literal")) { + Object literal = map.get("literal"); + if (literal == null) { + final RelDataType type = toType(typeFactory, get(map, "type")); + return rexBuilder.makeNullLiteral(type); + } + if (!map.containsKey("type")) { + // In previous versions, type was not specified for all literals. + // To keep backwards compatibility, if type is not specified + // we just interpret the literal + return toRex(relInput, literal); + } + final RelDataType type = toType(typeFactory, get(map, "type")); + if (literal instanceof Map && ((Map) literal).containsKey("rangeSet")) { + Sarg sarg = sargFromJson((Map) literal); + return rexBuilder.makeSearchArgumentLiteral(sarg, type); + } + if (type.getSqlTypeName() == SqlTypeName.SYMBOL) { + literal = toEnum((String) literal); + } + return rexBuilder.makeLiteral(literal, type); + } + if (map.containsKey("sargLiteral")) { + Object sargObject = map.get("sargLiteral"); + if (sargObject == null) { + final RelDataType type = toType(typeFactory, get(map, "type")); + return rexBuilder.makeNullLiteral(type); + } + final RelDataType type = toType(typeFactory, get(map, "type")); + Sarg sarg = sargFromJson((Map) sargObject); + return rexBuilder.makeSearchArgumentLiteral(sarg, type); + } + if (map.containsKey("dynamicParam")) { + final Object dynamicParamObject = requireNonNull(map.get("dynamicParam")); + final Integer index = (Integer) dynamicParamObject; + final RelDataType type = toType(typeFactory, get(map, "type")); + return rexBuilder.makeDynamicParam(type, index); + } + throw new UnsupportedOperationException("cannot convert to rex " + o); + } else if (o instanceof Boolean) { + return rexBuilder.makeLiteral((Boolean) o); + } else if (o instanceof String) { + return rexBuilder.makeLiteral((String) o); + } else if (o instanceof Number) { + final Number number = (Number) o; + if (number instanceof Double || number instanceof Float) { + return rexBuilder.makeApproxLiteral(BigDecimal.valueOf(number.doubleValue())); + } else { + return rexBuilder.makeExactLiteral(BigDecimal.valueOf(number.longValue())); + } + } else { + throw new UnsupportedOperationException("cannot convert to rex " + o); + } + } + + private JsonBuilder jsonBuilder() { + return requireNonNull(jsonBuilder, "jsonBuilder"); + } + + // Copied from RelJson because it's private but used in toRex(RelInput, Object) + @SuppressWarnings("unchecked") + private static T get(Map map, String key) { + return (T) requireNonNull(map.get(key), () -> "entry for key " + key); + } + + // Copied from RelJson for the usage of custom operatorTable + @Nullable SqlOperator toOp(Map map) { + // in case different operator has the same kind, check with both name and kind. + String name = get(map, "name"); + String kind = get(map, "kind"); + String syntax = get(map, "syntax"); + SqlKind sqlKind = SqlKind.valueOf(kind); + SqlSyntax sqlSyntax = SqlSyntax.valueOf(syntax); + List operators = new ArrayList<>(); + operatorTable.lookupOperatorOverloads( + new SqlIdentifier(name, SqlParserPos.ZERO), + null, + sqlSyntax, + operators, + SqlNameMatchers.liberal()); + for (SqlOperator operator : operators) { + if (operator.kind == sqlKind) { + return operator; + } + } + String class_ = (String) map.get("class"); + if (class_ != null) { + return AvaticaUtils.instantiatePlugin(SqlOperator.class, class_); + } + throw RESOURCE.noOperator(name, kind, syntax).ex(); + } + + // Copied from RelJson for the usage of custom operatorTable + @Nullable SqlAggFunction toAggregation(Map map) { + return (SqlAggFunction) toOp(map); + } + + // Copied from RelJson because it's private but used in toRex(RelInput, Object) + private List toRexList(RelInput relInput, List operands) { + final List list = new ArrayList<>(); + for (Object operand : operands) { + list.add(toRex(relInput, operand)); + } + return list; + } + + // Copied from RelJson because it's private but used in toRex(RelInput, Object) + private void addRexFieldCollationList( + List list, RelInput relInput, @Nullable List> order) { + if (order == null) { + return; + } + + for (Map o : order) { + RexNode expr = requireNonNull(toRex(relInput, o.get("expr")), "expr"); + Set directions = new HashSet<>(); + if (RelFieldCollation.Direction.valueOf(get(o, "direction")) + == RelFieldCollation.Direction.DESCENDING) { + directions.add(SqlKind.DESCENDING); + } + if (RelFieldCollation.NullDirection.valueOf(get(o, "null-direction")) + == RelFieldCollation.NullDirection.FIRST) { + directions.add(SqlKind.NULLS_FIRST); + } else { + directions.add(SqlKind.NULLS_LAST); + } + list.add(new RexFieldCollation(expr, directions)); + } + } + + /** + * Converts a string to an enum value. Replicated from RelEnumTypes.toEnum() since it's + * package-private. + */ + @SuppressWarnings("unchecked") + private static > E toEnum(String name) { + return (E) requireNonNull(ENUM_BY_NAME.get(name), () -> "No enum registered for name: " + name); + } + + // Copied from RelJson because it's private but used in toRex(RelInput, Object) + private @Nullable RexWindowBound toRexWindowBound( + RelInput relInput, @Nullable Map map) { + if (map == null) { + return null; + } + + final String type = get(map, "type"); + switch (type) { + case "CURRENT_ROW": + return RexWindowBounds.CURRENT_ROW; + case "UNBOUNDED_PRECEDING": + return RexWindowBounds.UNBOUNDED_PRECEDING; + case "UNBOUNDED_FOLLOWING": + return RexWindowBounds.UNBOUNDED_FOLLOWING; + case "PRECEDING": + return RexWindowBounds.preceding(toRex(relInput, get(map, "offset"))); + case "FOLLOWING": + return RexWindowBounds.following(toRex(relInput, get(map, "offset"))); + default: + throw new UnsupportedOperationException("cannot convert " + type + " to rex window bound"); + } + } + + // Copied from RelJson because it's private but used in toRex(RelInput, Object) + private static @Nullable RexWindowExclusion toRexWindowExclusion( + @Nullable Map map) { + if (map == null) { + return null; + } + final String type = get(map, "type"); + switch (type) { + case "CURRENT_ROW": + return RexWindowExclusion.EXCLUDE_CURRENT_ROW; + case "GROUP": + return RexWindowExclusion.EXCLUDE_GROUP; + case "TIES": + return RexWindowExclusion.EXCLUDE_TIES; + case "NO OTHERS": + return RexWindowExclusion.EXCLUDE_NO_OTHER; + default: + throw new UnsupportedOperationException( + "cannot convert " + type + " to rex window exclusion"); + } + } + + /** + * Special context from which a relational expression can be initialized, reading from a + * serialized form of the relational expression. + * + *

    Contains only a cluster and an empty list of inputs; most methods throw {@link + * UnsupportedOperationException}. + * + *

    Replicated from {@link RelJson} + */ + private static class RelInputForCluster implements RelInput { + private final RelOptCluster cluster; + + RelInputForCluster(RelOptCluster cluster) { + this.cluster = cluster; + } + + @Override + public RelOptCluster getCluster() { + return cluster; + } + + @Override + public RelTraitSet getTraitSet() { + throw new UnsupportedOperationException(); + } + + @Override + public RelOptTable getTable(String table) { + throw new UnsupportedOperationException(); + } + + @Override + public RelNode getInput() { + throw new UnsupportedOperationException(); + } + + @Override + public List getInputs() { + return ImmutableList.of(); + } + + @Override + public @Nullable RexNode getExpression(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public ImmutableBitSet getBitSet(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable List getBitSetList(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public List getAggregateCalls(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable Object get(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable String getString(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public float getFloat(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public BigDecimal getBigDecimal(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public > @Nullable E getEnum(String tag, Class enumClass) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable List getExpressionList(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable List getStringList(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable List getIntegerList(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public @Nullable List> getIntegerListList(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public RelDataType getRowType(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public RelDataType getRowType(String expressionsTag, String fieldsTag) { + throw new UnsupportedOperationException(); + } + + @Override + public RelCollation getCollation() { + throw new UnsupportedOperationException(); + } + + @Override + public RelDistribution getDistribution() { + throw new UnsupportedOperationException(); + } + + @Override + public ImmutableList> getTuples(String tag) { + throw new UnsupportedOperationException(); + } + + @Override + public boolean getBoolean(String tag, boolean default_) { + throw new UnsupportedOperationException(); + } + } +} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java index 3c6154f23bb..12ae05ea735 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializer.java @@ -74,18 +74,18 @@ public RelJsonSerializer(RelOptCluster cluster) { *

  • Encodes the resulting map into a final object string * * @param rexNode pushed down RexNode - * @param relDataType row type of RexNode input + * @param rowType row type of RexNode input * @param fieldTypes input field and ExprType mapping * @return serialized string of map structure for inputs */ - public String serialize( - RexNode rexNode, RelDataType relDataType, Map fieldTypes) { + public String serialize(RexNode rexNode, RelDataType rowType, Map fieldTypes) { try { // Serialize RexNode and RelDataType by JSON JsonBuilder jsonBuilder = new JsonBuilder(); - RelJson relJson = RelJson.create().withJsonBuilder(jsonBuilder); + RelJson relJson = ExtendedRelJson.create(jsonBuilder); String rexNodeJson = jsonBuilder.toJsonString(relJson.toJson(rexNode)); - String rowTypeJson = jsonBuilder.toJsonString(relJson.toJson(relDataType)); + Object rowTypeJsonObj = relJson.toJson(rowType); + String rowTypeJson = jsonBuilder.toJsonString(rowTypeJsonObj); // Construct envelope of serializable objects Map envelope = Map.of(EXPR, rexNodeJson, FIELD_TYPES, fieldTypes, ROW_TYPE, rowTypeJson); @@ -121,7 +121,7 @@ public Map deserialize(String struct) { // PPL Expr types are all serializable Map fieldTypes = (Map) objectMap.get(FIELD_TYPES); // Deserialize RelDataType and RexNode by JSON - RelJson relJson = RelJson.create(); + RelJson relJson = ExtendedRelJson.create((JsonBuilder) null); Map rowTypeMap = mapper.readValue((String) objectMap.get(ROW_TYPE), TYPE_REF); RelDataType rowType = relJson.toType(cluster.getTypeFactory(), rowTypeMap); OpenSearchRelInputTranslator inputTranslator = new OpenSearchRelInputTranslator(rowType); diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java index 554e5f4b108..9f2076ff59f 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/util/JdbcOpenSearchDataTypeConvertor.java @@ -10,6 +10,8 @@ import java.sql.SQLException; import java.sql.Types; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; import lombok.experimental.UtilityClass; import org.apache.calcite.avatica.util.ArrayImpl; import org.apache.calcite.rel.type.RelDataType; @@ -128,11 +130,47 @@ public static ExprValue getExprValueFromSqlType( "Unchecked sql type: {}, return Object type {}", sqlType, value.getClass().getTypeName()); - return ExprValueUtils.fromObjectValue(value); + return convertComplexValue(value); } } catch (SQLException e) { LOG.error("Error converting SQL type {}: {}", sqlType, e.getMessage()); throw e; } } + + /** + * Convert complex values like Maps that may contain geo points. This method recursively processes + * Maps to handle nested geo points and converts them to appropriate ExprValue representations. + */ + private static ExprValue convertComplexValue(Object value) { + Object converted = processValue(value); + return ExprValueUtils.fromObjectValue(converted); + } + + /** + * Process values recursively, handling geo points and nested maps. Geo points are converted to + * OpenSearchExprGeoPointValue. Maps are recursively processed to handle nested structures. + */ + private static Object processValue(Object value) { + if (value == null) { + return null; + } + + if (value instanceof Point) { + Point point = (Point) value; + return new OpenSearchExprGeoPointValue(point.getY(), point.getX()); + } + + if (value instanceof Map) { + Map map = (Map) value; + Map convertedMap = new HashMap<>(); + for (Map.Entry entry : map.entrySet()) { + convertedMap.put(entry.getKey(), processValue(entry.getValue())); + } + return convertedMap; + } + + // For other types, return as-is + return value; + } } diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 6e8cb789b42..948ac4854c0 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -241,6 +241,12 @@ void get_index_mappings_with_IOException() { assertThrows(IllegalStateException.class, () -> client.getIndexMappings(indexName)); } + @Test + void get_index_mappings_with_index_patterns() { + mockNodeClientIndicesMappings("", null); + assertThrows(IndexNotFoundException.class, () -> client.getIndexMappings("test*")); + } + @Test void get_index_mappings_with_non_exist_index() { when(nodeClient.admin().indices().prepareGetMappings(any()).setLocal(anyBoolean()).get()) @@ -494,7 +500,9 @@ public void mockNodeClientIndicesMappings(String indexName, String mappings) { .thenReturn(mockResponse); try { Map metadata; - if (mappings.isEmpty()) { + if (mappings == null) { + metadata = Map.of(); + } else if (mappings.isEmpty()) { when(emptyMapping.getSourceAsMap()).thenReturn(Map.of()); metadata = Map.of(indexName, emptyMapping); } else { diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 4991fbbbdfd..6be02c9d6f1 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -62,6 +62,7 @@ import org.opensearch.core.xcontent.DeprecationHandler; import org.opensearch.core.xcontent.NamedXContentRegistry; import org.opensearch.core.xcontent.XContentParser; +import org.opensearch.index.IndexNotFoundException; import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.builder.SearchSourceBuilder; @@ -225,6 +226,15 @@ void get_index_mappings() throws IOException { OpenSearchTextType.of(MappingType.Long), parsedTypes.get("manager.salary"))); } + @Test + void get_index_mappings_with_index_patterns() throws IOException { + GetMappingsResponse response = mock(GetMappingsResponse.class); + when(response.mappings()).thenReturn(Map.of()); + when(restClient.indices().getMapping(any(GetMappingsRequest.class), any())) + .thenReturn(response); + assertThrows(IndexNotFoundException.class, () -> client.getIndexMappings("test*")); + } + @Test void get_index_mappings_with_IOException() throws IOException { when(restClient.indices().getMapping(any(GetMappingsRequest.class), any())) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java deleted file mode 100644 index 8b137891791..00000000000 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprIpValueTest.java +++ /dev/null @@ -1 +0,0 @@ - diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java index 8834edbac36..651c176c866 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/executor/OpenSearchQueryManagerTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.opensearch.executor; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java index e304f504242..b3a1d766d8b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/request/AggregateAnalyzerTest.java @@ -278,8 +278,7 @@ void analyze_groupBy() throws ExpressionNotAnalyzableException { assertEquals( "[{\"composite_buckets\":{\"composite\":{\"size\":1000,\"sources\":[" + "{\"a\":{\"terms\":{\"field\":\"a\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}," - + "{\"b\":{\"terms\":{\"field\":\"b.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}," - + "\"aggregations\":{\"cnt\":{\"value_count\":{\"field\":\"_index\"}}}}}]", + + "{\"b\":{\"terms\":{\"field\":\"b.keyword\",\"missing_bucket\":true,\"missing_order\":\"first\",\"order\":\"asc\"}}}]}}}]", result.getLeft().toString()); assertInstanceOf(CompositeAggregationParser.class, result.getRight()); MetricParserHelper metricsParser = diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 95fc07a28e3..6c509c20c36 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -288,7 +288,7 @@ void response_isnot_aggregation_when_aggregation_is_empty() { void aggregation_iterator() { final List includes = List.of("id1", "id2"); - when(parser.parse(any())) + when(parser.parse((Aggregations) any())) .thenReturn(Arrays.asList(ImmutableMap.of("id1", 1), ImmutableMap.of("id2", 2))); when(searchResponse.getAggregations()).thenReturn(aggregations); when(factory.getParser()).thenReturn(parser); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java index 9bf70602e8b..5024d416086 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/setting/OpenSearchSettingsTest.java @@ -96,6 +96,25 @@ void update() { assertNotEquals(newValue.getBytes(), oldValue.getBytes()); } + @Test + void testPplValuesMaxLimitSetting() { + when(clusterSettings.get(ClusterName.CLUSTER_NAME_SETTING)).thenReturn(ClusterName.DEFAULT); + when(clusterSettings.get(not((eq(ClusterName.CLUSTER_NAME_SETTING))))).thenReturn(null); + OpenSearchSettings settings = new OpenSearchSettings(clusterSettings); + + // Test default value is 0 (unlimited) + Integer defaultLimit = settings.getSettingValue(Settings.Key.PPL_VALUES_MAX_LIMIT); + assertEquals(0, defaultLimit); + + // Test setting update + OpenSearchSettings.Updater updater = settings.new Updater(Settings.Key.PPL_VALUES_MAX_LIMIT); + updater.accept(5000); + + // Test retrieval after update + Integer newLimit = settings.getSettingValue(Settings.Key.PPL_VALUES_MAX_LIMIT); + assertEquals(5000, newLimit); + } + @Test void getSparkExecutionEngineConfigSetting() { // Default is empty string diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java index 2b0f77e5744..06cc0b82fd7 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/OpenSearchIndexScanOptimizationTest.java @@ -19,7 +19,6 @@ import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.expression.DSL.literal; -import static org.opensearch.sql.opensearch.storage.script.aggregation.AggregationQueryBuilder.AGGREGATION_BUCKET_SIZE; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.aggregation; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.filter; import static org.opensearch.sql.planner.logical.LogicalPlanDSL.highlight; @@ -57,9 +56,7 @@ import org.opensearch.index.query.QueryBuilders; import org.opensearch.search.aggregations.AggregationBuilder; import org.opensearch.search.aggregations.AggregationBuilders; -import org.opensearch.search.aggregations.BucketOrder; import org.opensearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder; -import org.opensearch.search.aggregations.bucket.terms.TermsAggregationBuilder; import org.opensearch.search.sort.NestedSortBuilder; import org.opensearch.search.sort.SortBuilder; import org.opensearch.search.sort.SortBuilders; @@ -77,7 +74,6 @@ import org.opensearch.sql.expression.function.OpenSearchFunctions; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.request.OpenSearchRequestBuilder; -import org.opensearch.sql.opensearch.response.agg.BucketAggregationParser; import org.opensearch.sql.opensearch.response.agg.CompositeAggregationParser; import org.opensearch.sql.opensearch.response.agg.OpenSearchAggregationResponseParser; import org.opensearch.sql.opensearch.response.agg.SingleValueParser; @@ -788,35 +784,26 @@ private Runnable withAggregationPushedDown( // Assume single term bucket and AVG metric in all tests in this suite AggregationBuilder aggBuilder; OpenSearchAggregationResponseParser responseParser; - if (bucketNullable) { - aggBuilder = - AggregationBuilders.composite( - "composite_buckets", - Collections.singletonList( - new TermsValuesSourceBuilder(aggregation.groupBy) - .field(aggregation.groupBy) - .order(aggregation.sortBy.getSortOrder() == ASC ? "asc" : "desc") - .missingOrder( - aggregation.sortBy.getNullOrder() == NULL_FIRST ? "first" : "last") - .missingBucket(true))) - .subAggregation( - AggregationBuilders.avg(aggregation.aggregateName).field(aggregation.aggregateBy)) - .size(AggregationQueryBuilder.AGGREGATION_BUCKET_SIZE); - } else { - aggBuilder = - new TermsAggregationBuilder(aggregation.groupBy) - .field(aggregation.groupBy) - .size(AGGREGATION_BUCKET_SIZE) - .order(BucketOrder.key(true)) - .subAggregation( - AggregationBuilders.avg(aggregation.aggregateName) - .field(aggregation.aggregateBy)); - } + aggBuilder = + AggregationBuilders.composite( + "composite_buckets", + Collections.singletonList( + bucketNullable + ? new TermsValuesSourceBuilder(aggregation.groupBy) + .field(aggregation.groupBy) + .order(aggregation.sortBy.getSortOrder() == ASC ? "asc" : "desc") + .missingOrder( + aggregation.sortBy.getNullOrder() == NULL_FIRST ? "first" : "last") + .missingBucket(true) + : new TermsValuesSourceBuilder(aggregation.groupBy) + .field(aggregation.groupBy) + .order(aggregation.sortBy.getSortOrder() == ASC ? "asc" : "desc"))) + .subAggregation( + AggregationBuilders.avg(aggregation.aggregateName).field(aggregation.aggregateBy)) + .size(AggregationQueryBuilder.AGGREGATION_BUCKET_SIZE); List aggBuilders = Collections.singletonList(aggBuilder); responseParser = - bucketNullable - ? new CompositeAggregationParser(new SingleValueParser(aggregation.aggregateName)) - : new BucketAggregationParser(new SingleValueParser(aggregation.aggregateName)); + new CompositeAggregationParser(new SingleValueParser(aggregation.aggregateName)); return () -> { verify(requestBuilder, times(1)).pushDownAggregation(Pair.of(aggBuilders, responseParser)); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/PushDownQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/PushDownQueryBuilderTest.java index 5f233d7f455..f94fcc82ff5 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/PushDownQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/scan/PushDownQueryBuilderTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.opensearch.storage.scan; import static org.junit.jupiter.api.Assertions.assertAll; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java index 124028c7fc8..721b0b66a4c 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/AggregationQueryBuilderTest.java @@ -460,16 +460,18 @@ void should_build_filter_aggregation_group_by() { assertEquals( format( "{%n" - + " \"gender\" : {%n" - + " \"terms\" : {%n" - + " \"field\" : \"gender\",%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + " \"size\" : 1000,%n" - + " \"min_doc_count\" : 1,%n" - + " \"shard_min_doc_count\" : 0,%n" - + " \"show_term_doc_count_error\" : false,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " }%n" + + " \"sources\" : [ {%n" + + " \"gender\" : {%n" + + " \"terms\" : {%n" + + " \"field\" : \"gender\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\"%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"avg(age) filter(where age > 34)\" : {%n" @@ -522,16 +524,19 @@ void should_build_histogram() { assertEquals( format( "{%n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" - + " \"histogram\" : {%n" - + " \"field\" : \"age\",%n" - + " \"interval\" : 10.0,%n" - + " \"offset\" : 0.0,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " },%n" - + " \"keyed\" : false,%n" - + " \"min_doc_count\" : 0%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"count(a)\" : {%n" @@ -554,16 +559,19 @@ void should_build_histogram_two_metrics() { assertEquals( format( "{%n" - + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" - + " \"histogram\" : {%n" - + " \"field\" : \"age\",%n" - + " \"interval\" : 10.0,%n" - + " \"offset\" : 0.0,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " },%n" - + " \"keyed\" : false,%n" - + " \"min_doc_count\" : 0%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=10, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 10.0%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"count(a)\" : {%n" @@ -592,16 +600,19 @@ void fixed_interval_time_span() { assertEquals( format( "{%n" - + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {%n" - + " \"date_histogram\" : {%n" - + " \"field\" : \"timestamp\",%n" - + " \"fixed_interval\" : \"1h\",%n" - + " \"offset\" : 0,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " },%n" - + " \"keyed\" : false,%n" - + " \"min_doc_count\" : 0%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=timestamp, value=1, unit=H)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"timestamp\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\",%n" + + " \"fixed_interval\" : \"1h\"%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"count(a)\" : {%n" @@ -624,16 +635,19 @@ void calendar_interval_time_span() { assertEquals( format( "{%n" - + " \"SpanExpression(field=date, value=1, unit=W)\" : {%n" - + " \"date_histogram\" : {%n" - + " \"field\" : \"date\",%n" - + " \"calendar_interval\" : \"1w\",%n" - + " \"offset\" : 0,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " },%n" - + " \"keyed\" : false,%n" - + " \"min_doc_count\" : 0%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=date, value=1, unit=W)\" : {%n" + + " \"date_histogram\" : {%n" + + " \"field\" : \"date\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\",%n" + + " \"calendar_interval\" : \"1w\"%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"count(a)\" : {%n" @@ -656,16 +670,19 @@ void general_span() { assertEquals( format( "{%n" - + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {%n" - + " \"histogram\" : {%n" - + " \"field\" : \"age\",%n" - + " \"interval\" : 1.0,%n" - + " \"offset\" : 0.0,%n" - + " \"order\" : {%n" - + " \"_key\" : \"asc\"%n" - + " },%n" - + " \"keyed\" : false,%n" - + " \"min_doc_count\" : 0%n" + + " \"composite_buckets\" : {%n" + + " \"composite\" : {%n" + + " \"size\" : 1000,%n" + + " \"sources\" : [ {%n" + + " \"SpanExpression(field=age, value=1, unit=NONE)\" : {%n" + + " \"histogram\" : {%n" + + " \"field\" : \"age\",%n" + + " \"missing_bucket\" : false,%n" + + " \"order\" : \"asc\",%n" + + " \"interval\" : 1.0%n" + + " }%n" + + " }%n" + + " } ]%n" + " },%n" + " \"aggregations\" : {%n" + " \"count(a)\" : {%n" diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilderTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilderTest.java index 8f364d90f77..4f90b5a5d76 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilderTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/aggregation/dsl/CompositeAggregationBuilderTest.java @@ -194,7 +194,7 @@ private String buildQuery( XContentBuilder builder = XContentFactory.jsonBuilder().prettyPrint(); builder.startObject(); CompositeValuesSourceBuilder sourceBuilder = - compositeBuilder.build(groupByExpressions).get(0); + compositeBuilder.build(groupByExpressions, true).get(0); sourceBuilder.toXContent(builder, EMPTY_PARAMS); builder.endObject(); return BytesReference.bytes(builder).utf8ToString(); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java index def9fafba32..33fd24f9e02 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/script/filter/lucene/TermQueryTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.opensearch.storage.script.filter.lucene; import static org.junit.Assert.*; diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java new file mode 100644 index 00000000000..e4703db86c1 --- /dev/null +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/ExtendedRelJsonTest.java @@ -0,0 +1,427 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.opensearch.storage.serde; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.mockito.Mockito.mock; + +import java.util.List; +import java.util.Map; +import java.util.Objects; +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.volcano.VolcanoPlanner; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.rex.RexBuilder; +import org.apache.calcite.rex.RexCall; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.sql.SqlOperatorTable; +import org.apache.calcite.sql.fun.SqlLibrary; +import org.apache.calcite.sql.fun.SqlLibraryOperatorTableFactory; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.type.SqlTypeName; +import org.apache.calcite.sql.util.SqlOperatorTables; +import org.apache.calcite.util.JsonBuilder; +import org.junit.jupiter.api.Test; +import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.expression.function.PPLBuiltinOperators; + +public class ExtendedRelJsonTest { + private static final SqlOperatorTable pplSqlOperatorTable = + SqlOperatorTables.chain( + PPLBuiltinOperators.instance(), + SqlStdOperatorTable.instance(), + // Add a list of necessary SqlLibrary if needed + SqlLibraryOperatorTableFactory.INSTANCE.getOperatorTable( + SqlLibrary.MYSQL, SqlLibrary.BIG_QUERY, SqlLibrary.SPARK, SqlLibrary.POSTGRESQL)); + private final ExtendedRelJson relJson = + (ExtendedRelJson) + ExtendedRelJson.create(new JsonBuilder()) + .withInputTranslator(new OpenSearchRelInputTranslator(mock(RelDataType.class))) + .withOperatorTable(pplSqlOperatorTable); + private final OpenSearchTypeFactory typeFactory = OpenSearchTypeFactory.TYPE_FACTORY; + + @Test + void testSerializeSqlType() { + RelDataType varcharType = typeFactory.createSqlType(SqlTypeName.VARCHAR); + RelDataType integerType = typeFactory.createSqlType(SqlTypeName.INTEGER, true); + RelDataType decimalType = typeFactory.createSqlType(SqlTypeName.DECIMAL, 4, 4); + + assertEquals( + Map.of("type", "VARCHAR", "nullable", false, "precision", -1), relJson.toJson(varcharType)); + assertEquals(Map.of("type", "INTEGER", "nullable", true), relJson.toJson(integerType)); + assertEquals( + Map.of("type", "DECIMAL", "nullable", false, "precision", 4, "scale", 4), + relJson.toJson(decimalType)); + } + + @Test + void testSerializeUDT() { + RelDataType dateType = typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_DATE); + RelDataType timeType = typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIME, true); + RelDataType timestampType = typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP); + + assertEquals( + Map.of("udt", "EXPR_DATE", "type", "VARCHAR", "nullable", false, "precision", -1), + relJson.toJson(dateType)); + assertEquals( + Map.of("udt", "EXPR_TIME", "type", "VARCHAR", "nullable", true, "precision", -1), + relJson.toJson(timeType)); + assertEquals( + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1), + relJson.toJson(timestampType)); + } + + @Test + void testDeserializeSqlType() { + Map serializedDecimal = + Map.of("type", "DECIMAL", "nullable", false, "precision", 4, "scale", 4); + assertEquals( + typeFactory.createSqlType(SqlTypeName.DECIMAL, 4, 4), + relJson.toType(typeFactory, serializedDecimal)); + } + + @Test + void testDeserializeUDT() { + Map serializedTimestamp = + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", true, "precision", -1); + assertEquals( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP, true).toString(), + relJson.toType(typeFactory, serializedTimestamp).toString()); + } + + @Test + void testSerializeRelDataTypeField() { + RelDataType structType = + typeFactory + .builder() + .add("name", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("timestamp", typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP)) + .build(); + + RelDataTypeField nameField = structType.getFieldList().get(0); + RelDataTypeField timestampField = structType.getFieldList().get(1); + + // Test serialization of regular field + Object nameFieldJson = relJson.toJson((Object) nameField); + assertEquals( + Map.of("type", "VARCHAR", "nullable", false, "precision", -1, "name", "name"), + nameFieldJson); + + // Test serialization of UDT field + Object timestampFieldJson = relJson.toJson(timestampField); + assertEquals( + Map.of( + "udt", + "EXPR_TIMESTAMP", + "type", + "VARCHAR", + "nullable", + false, + "precision", + -1, + "name", + "timestamp"), + timestampFieldJson); + } + + @Test + void testDeserializeRelDataTypeField() { + RelDataType expectedType = + typeFactory + .builder() + .add("name", typeFactory.createSqlType(SqlTypeName.VARCHAR)) + .add("timestamp", typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP)) + .build(); + + Map nameFieldMap = + Map.of("type", "VARCHAR", "nullable", false, "precision", -1, "name", "name"); + Map udtFieldMap = + Map.of( + "udt", + "EXPR_TIMESTAMP", + "type", + "VARCHAR", + "nullable", + false, + "precision", + -1, + "name", + "timestamp"); + Map structMap = + Map.of( + "fields", + java.util.Arrays.asList(nameFieldMap, udtFieldMap), + "type", + "struct", + "nullable", + false); + RelDataType resultType = relJson.toType(typeFactory, structMap); + + assertEquals(resultType, expectedType); + } + + @SuppressWarnings("unchecked") + @Test + void testSerializeArrayTypes() { + RelDataType stringArrayType = + typeFactory.createArrayType(typeFactory.createSqlType(SqlTypeName.VARCHAR), -1); + + RelDataType timestampArrayType = + typeFactory.createArrayType( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), -1); + + RelDataType ipArrayType = + typeFactory.createArrayType( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), -1); + + assertEquals( + Map.of( + "type", + "ARRAY", + "nullable", + false, + "component", + Map.of("type", "VARCHAR", "nullable", false, "precision", -1)), + relJson.toJson(stringArrayType)); + + assertEquals( + Map.of( + "type", + "ARRAY", + "nullable", + false, + "component", + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1)), + relJson.toJson(timestampArrayType)); + + Object serializedIpArray = relJson.toJson(ipArrayType); + Map serializedMap = (Map) serializedIpArray; + assertEquals("ARRAY", serializedMap.get("type")); + assertEquals(false, serializedMap.get("nullable")); + assertInstanceOf(Map.class, serializedMap.get("component")); + Map componentMap = (Map) serializedMap.get("component"); + assertEquals("EXPR_IP", componentMap.get("udt")); + } + + @Test + void testDeserializeArrayTypes() { + Map serializedTimestampArray = + Map.of( + "type", + "ARRAY", + "nullable", + false, + "component", + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1)); + + RelDataType expectedTimestampArray = + typeFactory.createArrayType( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), -1); + + RelDataType deserializedType = relJson.toType(typeFactory, serializedTimestampArray); + assertEquals(expectedTimestampArray, deserializedType); + + assertEquals( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + deserializedType.getComponentType()); + } + + @SuppressWarnings("unchecked") + @Test + void testSerializeMapTypes() { + RelDataType regularMapType = + typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createSqlType(SqlTypeName.INTEGER), + false); + + RelDataType mapWithUdtValueType = + typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + true); + + RelDataType complexMapType = + typeFactory.createMapType( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + false); + + Map expectedRegularMap = + Map.of( + "type", + "MAP", + "nullable", + false, + "key", + Map.of("type", "VARCHAR", "nullable", false, "precision", -1), + "value", + Map.of("type", "INTEGER", "nullable", false)); + assertEquals(expectedRegularMap, relJson.toJson(regularMapType)); + + Map expectedUdtValueMap = + Map.of( + "type", + "MAP", + "nullable", + true, + "key", + Map.of("type", "VARCHAR", "nullable", false, "precision", -1), + "value", + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1)); + assertEquals(expectedUdtValueMap, relJson.toJson(mapWithUdtValueType)); + + Object serializedComplexMap = relJson.toJson(complexMapType); + assertInstanceOf(Map.class, serializedComplexMap); + Map serializedMap = (Map) serializedComplexMap; + + assertEquals("MAP", serializedMap.get("type")); + assertEquals(false, serializedMap.get("nullable")); + + assertInstanceOf(Map.class, serializedMap.get("key")); + Map keyMap = (Map) serializedMap.get("key"); + assertEquals("EXPR_IP", keyMap.get("udt")); + + assertInstanceOf(Map.class, serializedMap.get("value")); + Map valueMap = (Map) serializedMap.get("value"); + assertEquals("EXPR_TIMESTAMP", valueMap.get("udt")); + assertEquals("VARCHAR", valueMap.get("type")); + } + + @Test + void testDeserializeMapTypes() { + Map serializedComplexMap = + Map.of( + "type", + "MAP", + "nullable", + false, + "key", + Map.of("udt", "EXPR_IP", "type", "VARCHAR", "nullable", false, "precision", -1), + "value", + Map.of("udt", "EXPR_TIMESTAMP", "type", "VARCHAR", "nullable", false, "precision", -1)); + + RelDataType expectedComplexMap = + typeFactory.createMapType( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + false); + + RelDataType deserializedType = relJson.toType(typeFactory, serializedComplexMap); + assertEquals(expectedComplexMap, deserializedType); + + assertEquals( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_IP), + deserializedType.getKeyType()); + assertEquals( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + deserializedType.getValueType()); + } + + @Test + void testSerializeAndDeserializeNestedStructure() { + RelDataType innerMapType = + typeFactory.createMapType( + typeFactory.createSqlType(SqlTypeName.VARCHAR), + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + false); + RelDataType complexType = typeFactory.createArrayType(innerMapType, -1); + + Object serialized = relJson.toJson(complexType); + RelDataType deserialized = relJson.toType(typeFactory, serialized); + + assertEquals(complexType, deserialized); + + assertEquals( + SqlTypeName.MAP, Objects.requireNonNull(deserialized.getComponentType()).getSqlTypeName()); + assertEquals( + SqlTypeName.VARCHAR, + Objects.requireNonNull(deserialized.getComponentType().getKeyType()).getSqlTypeName()); + assertEquals( + typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP), + deserialized.getComponentType().getValueType()); + } + + @Test + void testSerializeAndDeserializeRexCallWithUDT() { + // Create a cluster for building RexNodes + VolcanoPlanner planner = new VolcanoPlanner(); + RelOptCluster cluster = RelOptCluster.create(planner, new RexBuilder(typeFactory)); + RexBuilder rexBuilder = cluster.getRexBuilder(); + ExtendedRelJson relJson = + (ExtendedRelJson) + this.relJson.withInputTranslator( + new OpenSearchRelInputTranslator(mock(RelDataType.class))); + + // Create UDT types for operands + RelDataType timestampType = typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_TIMESTAMP); + RelDataType dateType = typeFactory.createUDT(OpenSearchTypeFactory.ExprUDT.EXPR_DATE); + + // Create RexNodes with UDT types - using literal values + RexNode timestamp = + rexBuilder.makeCall( + timestampType, + PPLBuiltinOperators.TIMESTAMP, + List.of(rexBuilder.makeLiteral("2023-01-01 12:00:00"))); + RexNode date = + rexBuilder.makeCall( + dateType, PPLBuiltinOperators.DATE, List.of(rexBuilder.makeLiteral("2023-01-01"))); + + // Create a RexCall using PLUS operator (as an example operation between UDTs) + RexCall rexCall = + (RexCall) + cluster + .getRexBuilder() + .makeCall( + timestampType, // result type + PPLBuiltinOperators.DATE_ADD, + java.util.Arrays.asList(timestamp, date)); + + // Serialize the RexCall + Object serializedRexCall = relJson.toJson(rexCall); + + // Verify the serialized structure contains basic call information + assertInstanceOf(Map.class, serializedRexCall); + @SuppressWarnings("unchecked") + Map serializedMap = (Map) serializedRexCall; + + // Check that it's a call operation + assertInstanceOf(Map.class, serializedMap.get("op")); + @SuppressWarnings("unchecked") + Map opMap = (Map) serializedMap.get("op"); + assertEquals("DATE_ADD", opMap.get("name")); + + // Check that operands exist + assertInstanceOf(java.util.List.class, serializedMap.get("operands")); + @SuppressWarnings("unchecked") + java.util.List operands = (java.util.List) serializedMap.get("operands"); + assertEquals(2, operands.size()); + + // Verify operands are literal structures + assertInstanceOf(Map.class, operands.get(0)); + assertInstanceOf(Map.class, operands.get(1)); + + // Most importantly, test that deserialization works and preserves UDT types + RexNode deserializedRexCall = relJson.toRex(cluster, serializedRexCall); + + // Verify the deserialized RexCall + assertInstanceOf(RexCall.class, deserializedRexCall); + RexCall deserializedCall = (RexCall) deserializedRexCall; + + // Check operator is preserved + assertEquals(PPLBuiltinOperators.DATE_ADD, deserializedCall.getOperator()); + + // Check operand count is preserved + assertEquals(2, deserializedCall.getOperands().size()); + + // Most importantly: Check that UDT type information is preserved through round-trip + assertEquals(timestampType, deserializedCall.getType()); + assertEquals(timestampType, deserializedCall.getOperands().get(0).getType()); + assertEquals(dateType, deserializedCall.getOperands().get(1).getType()); + } +} diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java index df6f77fa1d3..eee479f9504 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/storage/serde/RelJsonSerializerTest.java @@ -21,10 +21,14 @@ import org.junit.jupiter.api.DisplayNameGenerator; import org.junit.jupiter.api.Test; import org.opensearch.sql.calcite.utils.OpenSearchTypeFactory; +import org.opensearch.sql.calcite.utils.UserDefinedFunctionUtils; import org.opensearch.sql.data.type.ExprCoreType; import org.opensearch.sql.data.type.ExprType; import org.opensearch.sql.expression.function.BuiltinFunctionName; import org.opensearch.sql.expression.function.PPLFuncImpTable; +import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; +import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; @DisplayNameGeneration(DisplayNameGenerator.ReplaceUnderscores.class) public class RelJsonSerializerTest { @@ -57,6 +61,45 @@ void testSerializeAndDeserialize() { assertEquals(fieldTypes, objects.get(RelJsonSerializer.FIELD_TYPES)); } + @Test + void testSerializeAndDeserializeUDT() { + RelDataType rowTypeWithUDT = + rexBuilder + .getTypeFactory() + .builder() + .kind(StructKind.FULLY_QUALIFIED) + .add("date", UserDefinedFunctionUtils.NULLABLE_DATE_UDT) + .add("time", UserDefinedFunctionUtils.NULLABLE_TIME_UDT) + .add("timestamp", UserDefinedFunctionUtils.NULLABLE_TIMESTAMP_UDT) + .add("ip", UserDefinedFunctionUtils.NULLABLE_IP_UDT) + .add( + "binary", + OpenSearchTypeFactory.TYPE_FACTORY.createUDT( + OpenSearchTypeFactory.ExprUDT.EXPR_BINARY)) + .build(); + Map fieldTypesWithUDT = + Map.ofEntries( + Map.entry("date", OpenSearchDateType.of(ExprCoreType.DATE)), + Map.entry("time", OpenSearchDateType.of(ExprCoreType.TIME)), + Map.entry("timestamp", OpenSearchDateType.of(ExprCoreType.TIMESTAMP)), + Map.entry("ip", OpenSearchDataType.of(ExprCoreType.IP)), + Map.entry("binary", OpenSearchBinaryType.of())); + RexNode rexNode = + PPLFuncImpTable.INSTANCE.resolve( + rexBuilder, + BuiltinFunctionName.JSON_ARRAY, + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(0).getType(), 0), + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(1).getType(), 1), + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(2).getType(), 2), + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(3).getType(), 3), + rexBuilder.makeInputRef(rowTypeWithUDT.getFieldList().get(4).getType(), 4)); + String serialized = serializer.serialize(rexNode, rowTypeWithUDT, fieldTypesWithUDT); + Map objects = serializer.deserialize(serialized); + assertEquals(rexNode, objects.get(RelJsonSerializer.EXPR)); + assertEquals(rowTypeWithUDT.toString(), objects.get(RelJsonSerializer.ROW_TYPE).toString()); + assertEquals(fieldTypesWithUDT, objects.get(RelJsonSerializer.FIELD_TYPES)); + } + @Test void testSerializeUnsupportedRexNode() { RexNode illegalRex = rexBuilder.makeRangeReference(rowType, 0, true); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/util/RestRequestUtilTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/util/RestRequestUtilTest.java index 168fabee74b..2d9953b7821 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/util/RestRequestUtilTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/util/RestRequestUtilTest.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.opensearch.util; import org.junit.jupiter.api.Assertions; diff --git a/plugin/build.gradle b/plugin/build.gradle index 55ce4c48076..65a24f6658e 100644 --- a/plugin/build.gradle +++ b/plugin/build.gradle @@ -30,7 +30,7 @@ plugins { id "io.freefair.lombok" id 'jacoco' id 'opensearch.opensearchplugin' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' id 'com.gradleup.shadow' } @@ -137,10 +137,10 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/ppl/build.gradle b/ppl/build.gradle index 047790b523c..8d00d7984ca 100644 --- a/ppl/build.gradle +++ b/ppl/build.gradle @@ -27,7 +27,7 @@ plugins { id "io.freefair.lombok" id 'jacoco' id 'antlr' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } generateGrammarSource { @@ -74,10 +74,10 @@ spotless { exclude '**/build/**', '**/build-*/**', 'src/main/gen/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 1b8399c3e7f..0bc7b784338 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -648,6 +648,7 @@ statsFunction | PERCENTILE_SHORTCUT LT_PRTHS valueExpression RT_PRTHS # percentileShortcutFunctionCall | (DISTINCT_COUNT | DC | DISTINCT_COUNT_APPROX) LT_PRTHS valueExpression RT_PRTHS # distinctCountFunctionCall | takeAggFunction # takeAggFunctionCall + | valuesAggFunction # valuesAggFunctionCall | percentileApproxFunction # percentileApproxFunctionCall | statsFunctionName LT_PRTHS functionArgs RT_PRTHS # statsFunctionCall ; @@ -676,6 +677,10 @@ takeAggFunction : TAKE LT_PRTHS fieldExpression (COMMA size = integerLiteral)? RT_PRTHS ; +valuesAggFunction + : VALUES LT_PRTHS valueExpression RT_PRTHS + ; + percentileApproxFunction : (PERCENTILE | PERCENTILE_APPROX) LT_PRTHS aggField = valueExpression COMMA percent = numericLiteral (COMMA compression = numericLiteral)? RT_PRTHS diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index b848db7c551..774eb73dff6 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -144,6 +144,10 @@ public AstBuilder(String query, Settings settings) { this.settings = settings; } + public Settings getSettings() { + return settings; + } + @Override public UnresolvedPlan visitQueryStatement(OpenSearchPPLParser.QueryStatementContext ctx) { UnresolvedPlan pplCommand = visit(ctx.pplCommands()); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 79be1ca0e9f..86d76f09890 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -242,10 +242,37 @@ public UnresolvedExpression visitStatsFunctionCall(StatsFunctionCallContext ctx) ctx.statsFunctionName().getText(), ctx.functionArgs().functionArg()); } + @Override + public UnresolvedExpression visitValuesAggFunctionCall( + OpenSearchPPLParser.ValuesAggFunctionCallContext ctx) { + ImmutableList.Builder builder = ImmutableList.builder(); + + // Get limit from settings + int limit = 0; // Default to unlimited + if (astBuilder.getSettings() != null) { + Integer settingValue = + astBuilder + .getSettings() + .getSettingValue(org.opensearch.sql.common.setting.Settings.Key.PPL_VALUES_MAX_LIMIT); + if (settingValue != null) { + limit = settingValue; + } + } + + // Only add limit parameter if it's non-zero (i.e., explicitly configured) + if (limit > 0) { + builder.add(new UnresolvedArgument("limit", AstDSL.intLiteral(limit))); + } + + return new AggregateFunction( + "values", visit(ctx.valuesAggFunction().valueExpression()), builder.build()); + } + private AggregateFunction buildAggregateFunction( String functionName, List args) { List unresolvedArgs = args.stream().map(this::visitFunctionArg).collect(Collectors.toList()); + return new AggregateFunction( functionName, unresolvedArgs.get(0), unresolvedArgs.subList(1, unresolvedArgs.size())); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java index 1a5b198b42c..a18da36c487 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstStatementBuilder.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ppl.parser; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index d5c55d10258..eeca282cb9d 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -105,6 +105,10 @@ public class PPLQueryDataAnonymizer extends AbstractNodeVisitor private static final String MASK_LITERAL = "***"; + private static final String MASK_COLUMN = "identifier"; + + private static final String MASK_TABLE = "table"; + private final AnonymizerExpressionAnalyzer expressionAnalyzer; private final Settings settings; @@ -143,12 +147,9 @@ public String visitExplain(Explain node, String context) { @Override public String visitRelation(Relation node, String context) { if (node instanceof DescribeRelation) { - // remove the system table suffix - String systemTable = node.getTableQualifiedName().toString(); - return StringUtils.format( - "describe %s", systemTable.substring(0, systemTable.lastIndexOf('.'))); + return StringUtils.format("describe %s", MASK_TABLE); } - return StringUtils.format("source=%s", node.getTableQualifiedName().toString()); + return StringUtils.format("source=%s", MASK_TABLE); } @Override @@ -183,23 +184,22 @@ public String visitJoin(Join node, String context) { .toList()); return StringUtils.format( "%s | join type=%s overwrite=%s max=%s %s %s", - left, joinType, overwrite, max, fieldList, right); + left, joinType, MASK_LITERAL, MASK_LITERAL, fieldList, right); } else { String joinType = node.getJoinType().name().toLowerCase(Locale.ROOT); - String leftAlias = node.getLeftAlias().map(l -> " left = " + l).orElse(""); - String rightAlias = node.getRightAlias().map(r -> " right = " + r).orElse(""); + String leftAlias = node.getLeftAlias().map(l -> " left = " + MASK_COLUMN).orElse(""); + String rightAlias = node.getRightAlias().map(r -> " right = " + MASK_COLUMN).orElse(""); String condition = node.getJoinCondition().map(c -> expressionAnalyzer.analyze(c, context)).orElse("true"); return StringUtils.format( "%s | %s join max=%s%s%s on %s %s", - left, joinType, max, leftAlias, rightAlias, condition, right); + left, joinType, MASK_LITERAL, leftAlias, rightAlias, condition, right); } } @Override public String visitLookup(Lookup node, String context) { String child = node.getChild().get(0).accept(this, context); - String lookupTable = ((Relation) node.getLookupRelation()).getTableQualifiedName().toString(); String mappingFields = formatFieldAlias(node.getMappingAliasMap()); String strategy = node.getOutputAliasMap().isEmpty() @@ -207,7 +207,7 @@ public String visitLookup(Lookup node, String context) { : String.format(" %s ", node.getOutputStrategy().toString().toLowerCase()); String outputFields = formatFieldAlias(node.getOutputAliasMap()); return StringUtils.format( - "%s | lookup %s %s%s%s", child, lookupTable, mappingFields, strategy, outputFields); + "%s | lookup %s %s%s%s", child, MASK_TABLE, mappingFields, strategy, outputFields); } private String formatFieldAlias(java.util.Map fieldMap) { @@ -230,7 +230,7 @@ public String visitSubqueryAlias(SubqueryAlias node, String context) { } // add "[]" only if its child is not a root String format = childNode.getChild().isEmpty() ? "%s as %s" : "[ %s ] as %s"; - return StringUtils.format(format, child, node.getAlias()); + return StringUtils.format(format, child, MASK_COLUMN); } @Override @@ -270,8 +270,8 @@ public String visitRename(Rename node, String context) { ((Field) renameMap.getTarget()).getField().toString()); } String renames = - renameMapBuilder.build().entrySet().stream() - .map(entry -> StringUtils.format("%s as %s", entry.getKey(), entry.getValue())) + node.getRenameList().stream() + .map(entry -> StringUtils.format("%s as %s", MASK_COLUMN, MASK_COLUMN)) .collect(Collectors.joining(",")); return StringUtils.format("%s | rename %s", child, renames); } @@ -336,7 +336,7 @@ public String visitBin(Bin node, String context) { } if (node.getAlias() != null) { - binCommand.append(" as ").append(node.getAlias()); + binCommand.append(" as ").append(MASK_COLUMN); } return StringUtils.format("%s%s", child, binCommand.toString()); @@ -406,7 +406,7 @@ public String visitEval(Eval node, String context) { } String expressions = expressionsBuilder.build().stream() - .map(pair -> StringUtils.format("%s" + "=%s", pair.getLeft(), pair.getRight())) + .map(pair -> StringUtils.format("%s" + "=%s", MASK_COLUMN, pair.getRight())) .collect(Collectors.joining(" ")); return StringUtils.format("%s | eval %s", child, expressions); } @@ -496,7 +496,7 @@ public String visitTimechart(Timechart node, String context) { public String visitRex(Rex node, String context) { String child = node.getChild().get(0).accept(this, context); String field = visitExpression(node.getField()); - String pattern = "\"" + node.getPattern().toString() + "\""; + String pattern = "\"" + MASK_LITERAL + "\""; StringBuilder command = new StringBuilder(); command.append( @@ -505,11 +505,11 @@ public String visitRex(Rex node, String context) { child, field, node.getMode().toString().toLowerCase(), pattern)); if (node.getMaxMatch().isPresent()) { - command.append(" max_match=").append(node.getMaxMatch().get()); + command.append(" max_match=").append(MASK_LITERAL); } if (node.getOffsetField().isPresent()) { - command.append(" offset_field=").append(node.getOffsetField().get()); + command.append(" offset_field=").append(MASK_COLUMN); } return command.toString(); @@ -535,7 +535,7 @@ public String visitParse(Parse node, String context) { } return ParseMethod.PATTERNS.equals(node.getParseMethod()) && regex.isEmpty() ? StringUtils.format("%s | %s %s", child, commandName, source) - : StringUtils.format("%s | %s %s '%s'", child, commandName, source, regex); + : StringUtils.format("%s | %s %s '%s'", child, commandName, source, MASK_LITERAL); } @Override @@ -646,7 +646,7 @@ public String visitPatterns(Patterns node, String context) { builder.append(" mode=").append(node.getPatternMode().toString()); builder.append(" max_sample_count=").append(visitExpression(node.getPatternMaxSampleCount())); builder.append(" buffer_limit=").append(visitExpression(node.getPatternBufferLimit())); - builder.append(" new_field=").append(node.getAlias()); + builder.append(" new_field=").append(MASK_COLUMN); if (!node.getArguments().isEmpty()) { for (java.util.Map.Entry entry : node.getArguments().entrySet()) { builder.append( @@ -780,7 +780,7 @@ public String visitIn(In node, String context) { @Override public String visitField(Field node, String context) { - return node.getField().toString(); + return MASK_COLUMN; } @Override @@ -802,7 +802,7 @@ public String visitAlias(Alias node, String context) { @Override public String visitTrendlineComputation(Trendline.TrendlineComputation node, String context) { final String dataField = node.getDataField().accept(this, context); - final String aliasClause = " as " + node.getAlias(); + final String aliasClause = " as " + MASK_COLUMN; final String computationType = node.getComputationType().name().toLowerCase(Locale.ROOT); return StringUtils.format( "%s(%d, %s)%s", computationType, node.getNumberOfDataPoints(), dataField, aliasClause); @@ -831,7 +831,7 @@ public String visitExistsSubquery(ExistsSubquery node, String context) { @Override public String visitCase(Case node, String context) { StringBuilder builder = new StringBuilder(); - builder.append("cast("); + builder.append("case("); for (When when : node.getWhenClauses()) { builder.append(analyze(when.getCondition(), context)); builder.append(","); @@ -858,7 +858,7 @@ public String visitCast(Cast node, String context) { @Override public String visitQualifiedName( org.opensearch.sql.ast.expression.QualifiedName node, String context) { - return String.join(".", node.getParts()); + return MASK_COLUMN; } } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java index a82470c1165..b5d154273cc 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAggregationTest.java @@ -58,6 +58,119 @@ public void testSimpleCount() { verifyPPLToSparkSQL(root, expectedSparkSql); } + @Test + public void testCountField() { + String ppl = "source=EMP | stats count(COMM) as c"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = "c=4\n"; + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT COUNT(`COMM`) `c`\nFROM `scott`.`EMP`\nWHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(COMM) as c1, count(COMM) as c2"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalProject(c1=[$0], c2=[$0])\n" + + " LogicalAggregate(group=[{}], c1=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c1=4; c2=4\n"; + verifyResult(root, expectedResult); + expectedSparkSql = + "SELECT COUNT(`COMM`) `c1`, COUNT(`COMM`) `c2`\n" + + "FROM `scott`.`EMP`\n" + + "WHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(), count(COMM) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], count()=[COUNT()], c=[COUNT($0)])\n" + + " LogicalProject(COMM=[$6])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "count()=14; c=4\n"; + verifyResult(root, expectedResult); + expectedSparkSql = "SELECT COUNT(*) `count()`, COUNT(`COMM`) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(COMM + 1.0) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject($f1=[+($6, 1.0:DECIMAL(2, 1))])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = "SELECT COUNT(`COMM` + 1.0) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | stats count(eval(COMM >= 500)) as c"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)])\n" + + " LogicalProject($f1=[CASE(>=($6, 500), 1, null:NULL)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=2\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = + "SELECT COUNT(CASE WHEN `COMM` >= 500 THEN 1 ELSE NULL END) `c`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | eval COMM1 = COMM | stats count(COMM) as c, count(COMM1) as c1"; + root = getRelNode(ppl); + expectedLogical = + "LogicalAggregate(group=[{}], c=[COUNT($0)], c1=[COUNT($1)])\n" + + " LogicalProject(COMM=[$6], COMM1=[$8])\n" + + " LogicalFilter(condition=[IS NOT NULL($6)])\n" + + " LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4]," + + " SAL=[$5], COMM=[$6], DEPTNO=[$7], COMM1=[$6])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4; c1=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = + "SELECT COUNT(`COMM`) `c`, COUNT(`COMM1`) `c1`\n" + + "FROM (SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`," + + " `COMM` `COMM1`\n" + + "FROM `scott`.`EMP`) `t12`\n" + + "WHERE `COMM` IS NOT NULL"; + verifyPPLToSparkSQL(root, expectedSparkSql); + + ppl = "source=EMP | eval COMM1 = COMM + 1 | stats count(COMM) as c, count(COMM1) as c1"; + root = getRelNode(ppl); + expectedLogical = + "" + + "LogicalAggregate(group=[{}], c=[COUNT($0)], c1=[COUNT($1)])\n" + + " LogicalProject(COMM=[$6], COMM1=[+($6, 1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + expectedResult = "c=4; c1=4\n"; + verifyResult(root, expectedResult); + + expectedSparkSql = "SELECT COUNT(`COMM`) `c`, COUNT(`COMM` + 1) `c1`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + @Test public void testTakeAgg() { String ppl = "source=EMP | stats take(JOB, 2) as c"; @@ -230,9 +343,9 @@ public void testAvgByFieldNonBucketNullable() { String expectedLogical = "" + "LogicalProject(avg(SAL)=[$1], DEPTNO=[$0])\n" - + " LogicalFilter(condition=[IS NOT NULL($0)])\n" - + " LogicalAggregate(group=[{0}], avg(SAL)=[AVG($1)])\n" - + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalAggregate(group=[{0}], avg(SAL)=[AVG($1)])\n" + + " LogicalProject(DEPTNO=[$7], SAL=[$5])\n" + + " LogicalFilter(condition=[IS NOT NULL($7)])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedResult = @@ -243,11 +356,10 @@ public void testAvgByFieldNonBucketNullable() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT AVG(`SAL`) `avg(SAL)`, `DEPTNO`\n" + "SELECT AVG(`SAL`) `avg(SAL)`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" - + "GROUP BY `DEPTNO`\n" - + "HAVING `DEPTNO` IS NOT NULL"; + + "WHERE `DEPTNO` IS NOT NULL\n" + + "GROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -353,12 +465,14 @@ public void testAvgByTimeSpanAndFields() { + " LogicalProject(avg(SAL)=[$2], hiredate_span=[$1], DEPTNO=[$0])\n" + " LogicalAggregate(group=[{0, 2}], avg(SAL)=[AVG($1)])\n" + " LogicalProject(DEPTNO=[$7], SAL=[$5], hiredate_span=[SPAN($4, 1, 'y')])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; + + " LogicalFilter(condition=[IS NOT NULL($4)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; verifyLogical(root, expectedLogical); String expectedSparkSql = "SELECT AVG(`SAL`) `avg(SAL)`, `SPAN`(`HIREDATE`, 1, 'y') `hiredate_span`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + + "WHERE `HIREDATE` IS NOT NULL\n" + "GROUP BY `DEPTNO`, `SPAN`(`HIREDATE`, 1, 'y')\n" + "ORDER BY `DEPTNO`, 2"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -383,8 +497,7 @@ public void testCountDistinct() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT COUNT(DISTINCT `JOB`) `distinct_count(JOB)`, `DEPTNO`\n" + "SELECT COUNT(DISTINCT `JOB`) `distinct_count(JOB)`, `DEPTNO`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -405,10 +518,7 @@ public void testCountDistinctWithAlias() { verifyResult(root, expectedResult); String expectedSparkSql = - "" - + "SELECT COUNT(DISTINCT `JOB`) `dc`, `DEPTNO`\n" - + "FROM `scott`.`EMP`\n" - + "GROUP BY `DEPTNO`"; + "SELECT COUNT(DISTINCT `JOB`) `dc`, `DEPTNO`\nFROM `scott`.`EMP`\nGROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } @@ -756,4 +866,76 @@ public void testMedian() { "SELECT `percentile_approx`(`SAL`, 50.0, DECIMAL) `median(SAL)`\n" + "FROM `scott`.`EMP`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testMaxOnStringField() { + String ppl = "source=EMP | stats max(ENAME) as max_name"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalAggregate(group=[{}], max_name=[MAX($0)])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = "max_name=WARD\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT MAX(`ENAME`) `max_name`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testMinOnStringField() { + String ppl = "source=EMP | stats min(ENAME) as min_name"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalAggregate(group=[{}], min_name=[MIN($0)])\n" + + " LogicalProject(ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = "min_name=ADAMS\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT MIN(`ENAME`) `min_name`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testMaxOnTimeField() { + String ppl = "source=EMP | stats max(HIREDATE) as max_hire_date"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalAggregate(group=[{}], max_hire_date=[MAX($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = "max_hire_date=1987-05-23\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT MAX(`HIREDATE`) `max_hire_date`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testMinOnTimeField() { + String ppl = "source=EMP | stats min(HIREDATE) as min_hire_date"; + RelNode root = getRelNode(ppl); + + String expectedLogical = + "LogicalAggregate(group=[{}], min_hire_date=[MIN($0)])\n" + + " LogicalProject(HIREDATE=[$4])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedResult = "min_hire_date=1980-12-17\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = "SELECT MIN(`HIREDATE`) `min_hire_date`\nFROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java index d1acdd168b4..285c084e936 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLEvalTest.java @@ -469,4 +469,52 @@ public void testListAggregationWithGroupBy() { + "GROUP BY `DEPTNO`"; verifyPPLToSparkSQL(root, expectedSparkSql); } + + @Test + public void testValuesAggregationAlone() { + String ppl = "source=EMP | stats values(DEPTNO)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalAggregate(group=[{}], values(DEPTNO)=[VALUES($0)])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = "SELECT `VALUES`(`DEPTNO`) `values(DEPTNO)`\n" + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testValuesAggregationWithOtherAgg() { + String ppl = "source=EMP | stats values(DEPTNO), count(DEPTNO)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalAggregate(group=[{}], values(DEPTNO)=[VALUES($0)], count(DEPTNO)=[COUNT($0)])\n" + + " LogicalProject(DEPTNO=[$7])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `VALUES`(`DEPTNO`) `values(DEPTNO)`, COUNT(`DEPTNO`) `count(DEPTNO)`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testValuesAggregationWithGroupBy() { + String ppl = "source=EMP | stats values(ENAME) by DEPTNO"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(values(ENAME)=[$1], DEPTNO=[$0])\n" + + " LogicalAggregate(group=[{0}], values(ENAME)=[VALUES($1)])\n" + + " LogicalProject(DEPTNO=[$7], ENAME=[$1])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + + String expectedSparkSql = + "SELECT `VALUES`(`ENAME`) `values(ENAME)`, `DEPTNO`\n" + + "FROM `scott`.`EMP`\n" + + "GROUP BY `DEPTNO`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } } diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java index 1ddab5ffd3a..24f5d6ccc19 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLExpandTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.ppl.calcite; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFlattenTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFlattenTest.java index aa8d5e2595f..0cb3a90e557 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFlattenTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFlattenTest.java @@ -1,8 +1,6 @@ /* - * - * * Copyright OpenSearch Contributors - * * SPDX-License-Identifier: Apache-2.0 - * + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 */ package org.opensearch.sql.ppl.calcite; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java index 855cd17db68..fc840169ee2 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFunctionTypeTest.java @@ -286,6 +286,17 @@ public void testStrftimeWithWrongNumberOfArgs() { + " [INTEGER,STRING,STRING]"); } + // Test VALUES function with array expression (which is not a supported scalar type) + @Test + public void testValuesFunctionWithArrayArgType() { + verifyQueryThrowsException( + "source=EMP | stats values(array(ENAME, JOB)) as unique_values", + "Aggregation function VALUES expects field type" + + " {[BYTE]|[SHORT]|[INTEGER]|[LONG]|[FLOAT]|[DOUBLE]|[STRING]|[BOOLEAN]|[DATE]|[TIME]|[TIMESTAMP]|[IP]|[BINARY]|[BYTE,INTEGER]" + + "|[SHORT,INTEGER]|[INTEGER,INTEGER]|[LONG,INTEGER]|[FLOAT,INTEGER]|[DOUBLE,INTEGER]|[STRING,INTEGER]|[BOOLEAN,INTEGER]|[DATE,INTEGER]|[TIME,INTEGER]|[TIMESTAMP,INTEGER]|[IP,INTEGER]|[BINARY,INTEGER]}," + + " but got [ARRAY]"); + } + // mvjoin should reject non-string single values @Test public void testMvjoinRejectsNonStringValues() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java index e8f0390a666..70dedb8dab2 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLJoinTest.java @@ -477,12 +477,12 @@ public void testJoinWithRelationSubquery() { | sort - DEPTNO | head 10 ] - | stats count(MGR) as cnt by JOB + | stats sum(MGR) as sum by JOB """; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(cnt=[$1], JOB=[$0])\n" - + " LogicalAggregate(group=[{0}], cnt=[COUNT($1)])\n" + "LogicalProject(sum=[$1], JOB=[$0])\n" + + " LogicalAggregate(group=[{0}], sum=[SUM($1)])\n" + " LogicalProject(JOB=[$2], MGR=[$3])\n" + " LogicalJoin(condition=[=($7, $8)], joinType=[inner])\n" + " LogicalTableScan(table=[[scott, EMP]])\n" @@ -491,12 +491,12 @@ public void testJoinWithRelationSubquery() { + " LogicalFilter(condition=[AND(>($0, 10), =($2, 'CHICAGO':VARCHAR))])\n" + " LogicalTableScan(table=[[scott, DEPT]])\n"; verifyLogical(root, expectedLogical); - String expectedResult = "cnt=4; JOB=SALESMAN\ncnt=1; JOB=CLERK\ncnt=1; JOB=MANAGER\n"; + String expectedResult = "sum=30792; JOB=SALESMAN\nsum=7698; JOB=CLERK\nsum=7839; JOB=MANAGER\n"; verifyResult(root, expectedResult); String expectedSparkSql = "" - + "SELECT COUNT(`EMP`.`MGR`) `cnt`, `EMP`.`JOB`\n" + + "SELECT SUM(`EMP`.`MGR`) `sum`, `EMP`.`JOB`\n" + "FROM `scott`.`EMP`\n" + "INNER JOIN (SELECT `DEPTNO`, `DNAME`\n" + "FROM `scott`.`DEPT`\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java index c7bd64d5cab..467dd36f548 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLPatternsTest.java @@ -150,8 +150,9 @@ public void testPatternsAggregationModeForSimplePatternMethod() { RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(pattern_count=[$1], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2)," - + " 'pattern'))], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))])\n" + "LogicalProject(patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'pattern'))]," + + " pattern_count=[$1], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($0, $2), 'tokens'))]," + + " sample_logs=[$2])\n" + " LogicalAggregate(group=[{1}], pattern_count=[COUNT($1)], sample_logs=[TAKE($0," + " $2)])\n" + " LogicalProject(ENAME=[$1], patterns_field=[REGEXP_REPLACE($1," @@ -160,11 +161,12 @@ public void testPatternsAggregationModeForSimplePatternMethod() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT COUNT(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')) `pattern_count`," - + " SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')," + "SELECT SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')," + " `TAKE`(`ENAME`, 10))['pattern'] AS STRING) `patterns_field`," + + " COUNT(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')) `pattern_count`," + " SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')," - + " `TAKE`(`ENAME`, 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " `TAKE`(`ENAME`, 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`," + + " `TAKE`(`ENAME`, 10) `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -176,9 +178,9 @@ public void testPatternsAggregationModeWithGroupByForSimplePatternMethod() { RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(DEPTNO=[$0], pattern_count=[$2]," - + " patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($1, $3), 'pattern'))]," - + " tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($1, $3), 'tokens'))])\n" + "LogicalProject(DEPTNO=[$0], patterns_field=[SAFE_CAST(ITEM(PATTERN_PARSER($1, $3)," + + " 'pattern'))], pattern_count=[$2], tokens=[SAFE_CAST(ITEM(PATTERN_PARSER($1, $3)," + + " 'tokens'))], sample_logs=[$3])\n" + " LogicalAggregate(group=[{1, 2}], pattern_count=[COUNT($2)], sample_logs=[TAKE($0," + " $3)])\n" + " LogicalProject(ENAME=[$1], DEPTNO=[$7], patterns_field=[REGEXP_REPLACE($1," @@ -187,11 +189,12 @@ public void testPatternsAggregationModeWithGroupByForSimplePatternMethod() { verifyLogical(root, expectedLogical); String expectedSparkSql = - "SELECT `DEPTNO`, COUNT(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')) `pattern_count`," - + " SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')," - + " `TAKE`(`ENAME`, 10))['pattern'] AS STRING) `patterns_field`," + "SELECT `DEPTNO`, SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+'," + + " '<*>'), `TAKE`(`ENAME`, 10))['pattern'] AS STRING) `patterns_field`," + + " COUNT(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')) `pattern_count`," + " SAFE_CAST(`PATTERN_PARSER`(REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')," - + " `TAKE`(`ENAME`, 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`\n" + + " `TAKE`(`ENAME`, 10))['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >) `tokens`," + + " `TAKE`(`ENAME`, 10) `sample_logs`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`, REGEXP_REPLACE(`ENAME`, '[a-zA-Z0-9]+', '<*>')"; verifyPPLToSparkSQL(root, expectedSparkSql); @@ -205,7 +208,7 @@ public void testPatternsAggregationModeForBrainMethod() { String expectedLogical = "LogicalProject(patterns_field=[SAFE_CAST(ITEM($1, 'pattern'))]," + " pattern_count=[SAFE_CAST(ITEM($1, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($1," - + " 'tokens'))])\n" + + " 'tokens'))], sample_logs=[SAFE_CAST(ITEM($1, 'sample_logs'))])\n" + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{0}])\n" + " LogicalAggregate(group=[{}], patterns_field=[pattern($0, $1, $2)])\n" + " LogicalProject(ENAME=[$1], $f8=[10], $f9=[100000])\n" @@ -223,7 +226,8 @@ public void testPatternsAggregationModeForBrainMethod() { "SELECT SAFE_CAST(`t20`.`patterns_field`['pattern'] AS STRING) `patterns_field`," + " SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT) `pattern_count`," + " SAFE_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR, VARCHAR ARRAY >)" - + " `tokens`\n" + + " `tokens`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS VARCHAR ARRAY)" + + " `sample_logs`\n" + "FROM (SELECT `pattern`(`ENAME`, 10, 100000) `patterns_field`\n" + "FROM `scott`.`EMP`) `$cor0`,\n" + "LATERAL UNNEST (SELECT `$cor0`.`patterns_field`\n" @@ -239,7 +243,7 @@ public void testPatternsAggregationModeWithGroupByForBrainMethod() { String expectedLogical = "LogicalProject(DEPTNO=[$0], patterns_field=[SAFE_CAST(ITEM($2, 'pattern'))]," + " pattern_count=[SAFE_CAST(ITEM($2, 'pattern_count'))], tokens=[SAFE_CAST(ITEM($2," - + " 'tokens'))])\n" + + " 'tokens'))], sample_logs=[SAFE_CAST(ITEM($2, 'sample_logs'))])\n" + " LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{1}])\n" + " LogicalAggregate(group=[{1}], patterns_field=[pattern($0, $2, $3)])\n" + " LogicalProject(ENAME=[$1], DEPTNO=[$7], $f8=[10], $f9=[100000])\n" @@ -257,7 +261,8 @@ public void testPatternsAggregationModeWithGroupByForBrainMethod() { "SELECT `$cor0`.`DEPTNO`, SAFE_CAST(`t20`.`patterns_field`['pattern'] AS STRING)" + " `patterns_field`, SAFE_CAST(`t20`.`patterns_field`['pattern_count'] AS BIGINT)" + " `pattern_count`, SAFE_CAST(`t20`.`patterns_field`['tokens'] AS MAP< VARCHAR," - + " VARCHAR ARRAY >) `tokens`\n" + + " VARCHAR ARRAY >) `tokens`, SAFE_CAST(`t20`.`patterns_field`['sample_logs'] AS" + + " VARCHAR ARRAY) `sample_logs`\n" + "FROM (SELECT `DEPTNO`, `pattern`(`ENAME`, 10, 100000) `patterns_field`\n" + "FROM `scott`.`EMP`\n" + "GROUP BY `DEPTNO`) `$cor0`,\n" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java index 31678148c9c..3f7363b0932 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstExpressionBuilderTest.java @@ -2,6 +2,7 @@ * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 */ + package org.opensearch.sql.ppl.parser; import static java.util.Collections.emptyList; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java index c0a14c45cbc..568a771732f 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstStatementBuilderTest.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.ppl.parser; diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 5dfc73f5483..44392cd9f57 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -33,7 +33,7 @@ public class PPLQueryDataAnonymizerTest { @Test public void testSearchCommand() { - assertEquals("source=t a:***", anonymize("search source=t a=1")); + assertEquals("source=table a:***", anonymize("search source=t a=1")); } @Test @@ -45,241 +45,261 @@ public void testTableFunctionCommand() { @Test public void testPrometheusPPLCommand() { - assertEquals( - "source=prometheus.http_requests_process", - anonymize("source=prometheus.http_requests_process")); + assertEquals("source=table", anonymize("source=prometheus.http_requests_process")); } @Test public void testWhereCommand() { - assertEquals("source=t | where a = ***", anonymize("search source=t | where a=1")); + assertEquals("source=table | where identifier = ***", anonymize("search source=t | where a=1")); } // Fields and Table Command Tests @Test public void testFieldsCommandWithoutArguments() { - assertEquals("source=t | fields + f,g", anonymize("source=t | fields f,g")); + assertEquals( + "source=table | fields + identifier,identifier", anonymize("source=t | fields f,g")); } @Test public void testFieldsCommandWithIncludeArguments() { - assertEquals("source=t | fields + f,g", anonymize("source=t | fields + f,g")); + assertEquals( + "source=table | fields + identifier,identifier", anonymize("source=t | fields + f,g")); } @Test public void testFieldsCommandWithExcludeArguments() { - assertEquals("source=t | fields - f,g", anonymize("source=t | fields - f,g")); + assertEquals( + "source=table | fields - identifier,identifier", anonymize("source=t | fields - f,g")); } @Test public void testFieldsCommandWithWildcards() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); - assertEquals("source=t | fields + account*", anonymize("source=t | fields account*")); - assertEquals("source=t | fields + *name", anonymize("source=t | fields *name")); - assertEquals("source=t | fields + *a*", anonymize("source=t | fields *a*")); - assertEquals("source=t | fields - account*", anonymize("source=t | fields - account*")); + assertEquals("source=table | fields + identifier", anonymize("source=t | fields account*")); + assertEquals("source=table | fields + identifier", anonymize("source=t | fields *name")); + assertEquals("source=table | fields + identifier", anonymize("source=t | fields *a*")); + assertEquals("source=table | fields - identifier", anonymize("source=t | fields - account*")); } @Test public void testFieldsCommandWithDelimiters() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=t | fields + firstname,lastname,age", + "source=table | fields + identifier,identifier,identifier", anonymize("source=t | fields firstname lastname age")); assertEquals( - "source=t | fields + firstname,lastname,balance", + "source=table | fields + identifier,identifier,identifier", anonymize("source=t | fields firstname lastname, balance")); assertEquals( - "source=t | fields + account*,*name", anonymize("source=t | fields account*, *name")); + "source=table | fields + identifier,identifier", + anonymize("source=t | fields account*, *name")); } @Test public void testTableCommand() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); - assertEquals("source=t | fields + f,g", anonymize("source=t | table f,g")); - assertEquals("source=t | fields + f,g", anonymize("source=t | table + f,g")); - assertEquals("source=t | fields - f,g", anonymize("source=t | table - f,g")); - assertEquals("source=t | fields + account*", anonymize("source=t | table account*")); assertEquals( - "source=t | fields + firstname,lastname,age", + "source=table | fields + identifier,identifier", anonymize("source=t | table f,g")); + assertEquals( + "source=table | fields + identifier,identifier", anonymize("source=t | table + f,g")); + assertEquals( + "source=table | fields - identifier,identifier", anonymize("source=t | table - f,g")); + assertEquals("source=table | fields + identifier", anonymize("source=t | table account*")); + assertEquals( + "source=table | fields + identifier,identifier,identifier", anonymize("source=t | table firstname lastname age")); } @Test public void anonymizeFieldsNoArg() { assertEquals( - "source=t | fields + f", + "source=table | fields + identifier", anonymize(projectWithArg(relation("t"), Collections.emptyList(), field("f")))); } @Test public void testRenameCommandWithMultiFields() { assertEquals( - "source=t | rename f as g,h as i,j as k", + "source=table | rename identifier as identifier,identifier as identifier,identifier as" + + " identifier", anonymize("source=t | rename f as g,h as i,j as k")); } @Test public void testRenameCommandWithWildcards() { - assertEquals("source=t | rename f* as g*", anonymize("source=t | rename f* as g*")); + assertEquals( + "source=table | rename identifier as identifier", anonymize("source=t | rename f* as g*")); } @Test public void testStatsCommandWithByClause() { - assertEquals("source=t | stats count(a) by b", anonymize("source=t | stats count(a) by b")); + assertEquals( + "source=table | stats count(identifier) by identifier", + anonymize("source=t | stats count(a) by b")); } @Test public void testStatsCommandWithNestedFunctions() { - assertEquals("source=t | stats sum(+(a,b))", anonymize("source=t | stats sum(a+b)")); + assertEquals( + "source=table | stats sum(+(identifier,identifier))", + anonymize("source=t | stats sum(a+b)")); } @Test public void testStatsCommandWithSpanFunction() { assertEquals( - "source=t | stats count(a) by span(b, *** d),c", + "source=table | stats count(identifier) by span(identifier, *** d),identifier", anonymize("source=t | stats count(a) by span(b, 1d), c")); } @Test public void testEventstatsCommandWithByClause() { assertEquals( - "source=t | eventstats count(a) by b", anonymize("source=t | eventstats count(a) by b")); + "source=table | eventstats count(identifier) by identifier", + anonymize("source=t | eventstats count(a) by b")); } @Test public void testEventstatsCommandWithNestedFunctions() { - assertEquals("source=t | eventstats sum(+(a,b))", anonymize("source=t | eventstats sum(a+b)")); + assertEquals( + "source=table | eventstats sum(+(identifier,identifier))", + anonymize("source=t | eventstats sum(a+b)")); } @Test public void testEventstatsCommandWithSpanFunction() { assertEquals( - "source=t | eventstats count(a) by span(b, *** d),c", + "source=table | eventstats count(identifier) by span(identifier, *** d),identifier", anonymize("source=t | eventstats count(a) by span(b, 1d), c")); } @Test public void testBinCommandBasic() { - assertEquals("source=t | bin f span=***", anonymize("source=t | bin f span=10")); + assertEquals("source=table | bin identifier span=***", anonymize("source=t | bin f span=10")); } @Test public void testBinCommandWithAllParameters() { assertEquals( - "source=t | bin f span=*** aligntime=*** as alias", + "source=table | bin identifier span=*** aligntime=*** as identifier", anonymize("source=t | bin f span=10 aligntime=earliest as alias")); } @Test public void testBinCommandWithCountParameters() { assertEquals( - "source=t | bin f bins=*** start=*** end=*** as alias", + "source=table | bin identifier bins=*** start=*** end=*** as identifier", anonymize("source=t | bin f bins=10 start=0 end=100 as alias")); } @Test public void testBinCommandWithMinspanParameters() { assertEquals( - "source=t | bin f minspan=*** start=*** end=*** as alias", + "source=table | bin identifier minspan=*** start=*** end=*** as identifier", anonymize("source=t | bin f minspan=5 start=0 end=100 as alias")); } @Test public void testDedupCommand() { assertEquals( - "source=t | dedup f1,f2 1 keepempty=false consecutive=false", + "source=table | dedup identifier,identifier 1 keepempty=false consecutive=false", anonymize("source=t | dedup f1, f2")); } @Test public void testTrendlineCommand() { assertEquals( - "source=t | trendline sma(2, date) as date_alias sma(3, time) as time_alias", + "source=table | trendline sma(2, identifier) as identifier sma(3, identifier) as" + + " identifier", anonymize("source=t | trendline sma(2, date) as date_alias sma(3, time) as time_alias")); } @Test public void testHeadCommandWithNumber() { - assertEquals("source=t | head 3", anonymize("source=t | head 3")); + assertEquals("source=table | head 3", anonymize("source=t | head 3")); } @Test public void testReverseCommand() { - assertEquals("source=t | reverse", anonymize("source=t | reverse")); + assertEquals("source=table | reverse", anonymize("source=t | reverse")); } @Test public void testTimechartCommand() { assertEquals( - "source=t | timechart span=span(@timestamp, *** m) limit=10 useother=true count() by host", + "source=table | timechart span=span(identifier, *** m) limit=10 useother=true count() by" + + " identifier", anonymize("source=t | timechart count() by host")); } // todo, sort order is ignored, it doesn't impact the log analysis. @Test public void testSortCommandWithOptions() { - assertEquals("source=t | sort f1,f2", anonymize("source=t | sort - f1, + f2")); + assertEquals( + "source=table | sort identifier,identifier", anonymize("source=t | sort - f1, + f2")); } @Test public void testSortCommandWithCount() { - assertEquals("source=t | sort 5 f1", anonymize("source=t | sort 5 f1")); + assertEquals("source=table | sort 5 identifier", anonymize("source=t | sort 5 f1")); } @Test public void testSortCommandWithDesc() { - assertEquals("source=t | sort f1", anonymize("source=t | sort f1 desc")); + assertEquals("source=table | sort identifier", anonymize("source=t | sort f1 desc")); } @Test public void testEvalCommand() { - assertEquals("source=t | eval r=abs(f)", anonymize("source=t | eval r=abs(f)")); + assertEquals( + "source=table | eval identifier=abs(identifier)", anonymize("source=t | eval r=abs(f)")); } @Test public void testEvalCommandWithStrftime() { assertEquals( - "source=t | eval formatted=strftime(timestamp,***)", + "source=table | eval identifier=strftime(identifier,***)", anonymize("source=t | eval formatted=strftime(timestamp, '%Y-%m-%d %H:%M:%S')")); } @Test public void testFillNullSameValue() { assertEquals( - "source=t | fillnull with *** in f1, f2", + "source=table | fillnull with *** in identifier, identifier", anonymize("source=t | fillnull with 0 in f1, f2")); } @Test public void testFillNullVariousValues() { assertEquals( - "source=t | fillnull using f1 = ***, f2 = ***", + "source=table | fillnull using identifier = ***, identifier = ***", anonymize("source=t | fillnull using f1 = 0, f2 = -1")); } @Test public void testFillNullWithoutFields() { - assertEquals("source=t | fillnull with ***", anonymize("source=t | fillnull with 0")); + assertEquals("source=table | fillnull with ***", anonymize("source=t | fillnull with 0")); } @Test public void testRareCommandWithGroupBy() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(false); - assertEquals("source=t | rare 10 a by b", anonymize("source=t | rare a by b")); + assertEquals( + "source=table | rare 10 identifier by identifier", anonymize("source=t | rare a by b")); } @Test public void testTopCommandWithNAndGroupBy() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(false); - assertEquals("source=t | top 1 a by b", anonymize("source=t | top 1 a by b")); + assertEquals( + "source=table | top 1 identifier by identifier", anonymize("source=t | top 1 a by b")); } @Test public void testRareCommandWithGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=t | rare 10 countield='count' showcount=true a by b", + "source=table | rare 10 countield='count' showcount=true identifier by identifier", anonymize("source=t | rare a by b")); } @@ -287,194 +307,220 @@ public void testRareCommandWithGroupByWithCalcite() { public void testTopCommandWithNAndGroupByWithCalcite() { when(settings.getSettingValue(Key.CALCITE_ENGINE_ENABLED)).thenReturn(true); assertEquals( - "source=t | top 1 countield='count' showcount=true a by b", + "source=table | top 1 countield='count' showcount=true identifier by identifier", anonymize("source=t | top 1 a by b")); } @Test public void testAndExpression() { - assertEquals("source=t | where a = *** and b = ***", anonymize("source=t | where a=1 and b=2")); + assertEquals( + "source=table | where identifier = *** and identifier = ***", + anonymize("source=t | where a=1 and b=2")); } @Test public void testOrExpression() { - assertEquals("source=t | where a = *** or b = ***", anonymize("source=t | where a=1 or b=2")); + assertEquals( + "source=table | where identifier = *** or identifier = ***", + anonymize("source=t | where a=1 or b=2")); } @Test public void testXorExpression() { - assertEquals("source=t | where a = *** xor b = ***", anonymize("source=t | where a=1 xor b=2")); + assertEquals( + "source=table | where identifier = *** xor identifier = ***", + anonymize("source=t | where a=1 xor b=2")); } @Test public void testNotExpression() { - assertEquals("source=t | where not a = ***", anonymize("source=t | where not a=1 ")); + assertEquals( + "source=table | where not identifier = ***", anonymize("source=t | where not a=1 ")); } @Test public void testInExpression() { - assertEquals("source=t | where a in (***)", anonymize("source=t | where a in (1, 2, 3) ")); + assertEquals( + "source=table | where identifier in (***)", anonymize("source=t | where a in (1, 2, 3) ")); } @Test public void testQualifiedName() { - assertEquals("source=t | fields + field0", anonymize("source=t | fields field0")); + assertEquals("source=table | fields + identifier", anonymize("source=t | fields field0")); } @Test public void testDateFunction() { assertEquals( - "source=t | eval date=DATE_ADD(DATE(***),INTERVAL *** HOUR)", + "source=table | eval identifier=DATE_ADD(DATE(***),INTERVAL *** HOUR)", anonymize("source=t | eval date=DATE_ADD(DATE('2020-08-26'),INTERVAL 1 HOUR)")); } @Test public void testDescribe() { - assertEquals("describe t", anonymize("describe t")); + assertEquals("describe table", anonymize("describe t")); } @Test public void testExplain() { assertEquals( - "explain standard source=t | fields + a", anonymizeStatement("source=t | fields a", true)); + "explain standard source=table | fields + identifier", + anonymizeStatement("source=t | fields a", true)); } @Test public void testExplainCommand() { assertEquals( - "explain standard source=t | fields + a", + "explain standard source=table | fields + identifier", anonymizeStatement("explain source=t | fields a", false)); assertEquals( - "explain extended source=t | fields + a", + "explain extended source=table | fields + identifier", anonymizeStatement("explain extended source=t | fields a", false)); } @Test public void testQuery() { - assertEquals("source=t | fields + a", anonymizeStatement("source=t | fields a", false)); + assertEquals( + "source=table | fields + identifier", anonymizeStatement("source=t | fields a", false)); } @Test public void testBetween() { assertEquals( - "source=t | where id between *** and *** | fields + id", + "source=table | where identifier between *** and *** | fields + identifier", anonymize("source=t | where id between 1 and 2 | fields id")); assertEquals( - "source=t | where not id between *** and *** | fields + id", + "source=table | where not identifier between *** and *** | fields + identifier", anonymize("source=t | where id not between 1 and 2 | fields id")); } @Test public void testAppendcol() { assertEquals( - "source=t | stats count() by b | appendcol override=false [ stats sum(c) by b ]", + "source=table | stats count() by identifier | appendcol override=false [ stats" + + " sum(identifier) by identifier ]", anonymize("source=t | stats count() by b | appendcol [ stats sum(c) by b ]")); assertEquals( - "source=t | stats count() by b | appendcol override=true [ stats sum(c) by b ]", + "source=table | stats count() by identifier | appendcol override=true [ stats" + + " sum(identifier) by identifier ]", anonymize("source=t | stats count() by b | appendcol override=true [ stats sum(c) by b ]")); assertEquals( - "source=t | appendcol override=false [ where a = *** ]", + "source=table | appendcol override=false [ where identifier = *** ]", anonymize("source=t | appendcol override=false [ where a = 1 ]")); } @Test public void testAppend() { assertEquals( - "source=t | stats count() by b | append [ | stats sum(c) by b ]", + "source=table | stats count() by identifier | append [ | stats sum(identifier) by" + + " identifier ]", anonymize("source=t | stats count() by b | append [ | stats sum(c) by b ]")); assertEquals( - "source=t | stats count() by b | append [ | stats sum(c) by b ]", + "source=table | stats count() by identifier | append [ | stats sum(identifier) by" + + " identifier ]", anonymize("source=t | stats count() by b | append [ | stats sum(c) by b ]")); assertEquals( - "source=t | append [ | where a = *** ]", anonymize("source=t | append [ | where a = 1 ]")); + "source=table | append [ | where identifier = *** ]", + anonymize("source=t | append [ | where a = 1 ]")); assertEquals( - "source=t | stats count() by b | append [source=a | stats sum(c) by b ]", + "source=table | stats count() by identifier | append [source=table | stats sum(identifier)" + + " by identifier ]", anonymize("source=t | stats count() by b | append [source=a | stats sum(c) by b ]")); assertEquals( - "source=t | append [source=b | where a = *** ]", + "source=table | append [source=table | where identifier = *** ]", anonymize("source=t | append [source=b | where a = 1 ]")); assertEquals( - "source=t | stats count() by b | append [source=a ]", + "source=table | stats count() by identifier | append [source=table ]", anonymize("source=t | stats count() by b | append [ source=a ]")); assertEquals( - "source=t | stats count() by b | append [ ]", + "source=table | stats count() by identifier | append [ ]", anonymize("source=t | stats count() by b | append [ ]")); } @Test + // Same as SQL, select * from a as b -> SELECT * FROM table AS identifier public void testSubqueryAlias() { - assertEquals("source=t as t1", anonymize("source=t as t1")); + assertEquals("source=table as identifier", anonymize("source=t as t1")); } @Test public void testJoin() { assertEquals( - "source=t | cross join max=0 on *** = *** s | fields + id", + "source=table | cross join max=*** on *** = *** table | fields + identifier", anonymize("source=t | cross join on 1=1 s | fields id")); assertEquals( - "source=t | inner join max=0 on id = uid s | fields + id", + "source=table | inner join max=*** on identifier = identifier table | fields + identifier", anonymize("source=t | inner join on id = uid s | fields id")); assertEquals( - "source=t as l | inner join max=0 left = l right = r on id = uid s as r | fields + id", + "source=table as identifier | inner join max=*** left = identifier right = identifier on" + + " identifier = identifier table as identifier | fields + identifier", anonymize("source=t | join left = l right = r on id = uid s | fields id")); assertEquals( - "source=t | left join max=0 right = r on id = uid s as r | fields + id", + "source=table | left join max=*** right = identifier on identifier = identifier table as" + + " identifier | fields + identifier", anonymize("source=t | left join right = r on id = uid s | fields id")); assertEquals( - "source=t as t1 | inner join max=0 left = t1 right = t2 on id = uid s as t2 | fields +" - + " t1.id", + "source=table as identifier | inner join max=*** left = identifier right = identifier on" + + " identifier = identifier table as identifier | fields + identifier", anonymize("source=t as t1 | inner join on id = uid s as t2 | fields t1.id")); assertEquals( - "source=t as t1 | right join max=0 left = t1 right = t2 on t1.id = t2.id s as t2 | fields +" - + " t1.id", + "source=table as identifier | right join max=*** left = identifier right = identifier on" + + " identifier = identifier table as identifier | fields + identifier", anonymize("source=t as t1 | right join max=0 on t1.id = t2.id s as t2 | fields t1.id")); assertEquals( - "source=t as t1 | right join max=0 left = t1 right = t2 on t1.id = t2.id [ source=s |" - + " fields + id ] as t2 | fields + t1.id", + "source=table as identifier | right join max=*** left = identifier right = identifier on" + + " identifier = identifier [ source=table | fields + identifier ] as identifier |" + + " fields + identifier", anonymize( "source=t as t1 | right join max=0 on t1.id = t2.id [ source=s | fields id] as t2 |" + " fields t1.id")); assertEquals( - "source=t | inner join max=2 on id = uid s | fields + id", + "source=table | inner join max=*** on identifier = identifier table | fields + identifier", anonymize("source=t | inner join max=2 on id = uid s | fields id")); } @Test public void testJoinWithFieldList() { assertEquals( - "source=t | join type=inner overwrite=true max=0 s | fields + id", + "source=table | join type=inner overwrite=*** max=*** table | fields + identifier", anonymize("source=t | join s | fields id")); assertEquals( - "source=t | join type=inner overwrite=true max=0 id s | fields + id", + "source=table | join type=inner overwrite=*** max=*** identifier table | fields +" + + " identifier", anonymize("source=t | join id s | fields id")); assertEquals( - "source=t | join type=left overwrite=false max=0 id1,id2 s | fields + id1", + "source=table | join type=left overwrite=*** max=*** identifier,identifier table | fields +" + + " identifier", anonymize("source=t | join type=left overwrite=false id1,id2 s | fields id1")); assertEquals( - "source=t | join type=left overwrite=false max=0 id1,id2 s | fields + id1", + "source=table | join type=left overwrite=*** max=*** identifier,identifier table | fields +" + + " identifier", anonymize("source=t | join type=outer overwrite=false id1 id2 s | fields id1")); assertEquals( - "source=t | join type=left overwrite=true max=2 id1,id2 s | fields + id1", + "source=table | join type=left overwrite=*** max=*** identifier,identifier table | fields +" + + " identifier", anonymize("source=t | join type=outer max=2 id1 id2 s | fields id1")); } @Test public void testLookup() { assertEquals( - "source=EMP | lookup DEPT DEPTNO replace LOC", + "source=table | lookup table DEPTNO replace LOC", anonymize("source=EMP | lookup DEPT DEPTNO replace LOC")); assertEquals( - "source=EMP | lookup DEPT DEPTNO replace LOC as JOB", + "source=table | lookup table DEPTNO replace LOC as JOB", anonymize("source=EMP | lookup DEPT DEPTNO replace LOC as JOB")); assertEquals( - "source=EMP | lookup DEPT DEPTNO append LOC", + "source=table | lookup table DEPTNO append LOC", anonymize("source=EMP | lookup DEPT DEPTNO append LOC")); assertEquals( - "source=EMP | lookup DEPT DEPTNO append LOC as JOB", + "source=table | lookup table DEPTNO append LOC as JOB", anonymize("source=EMP | lookup DEPT DEPTNO append LOC as JOB")); - assertEquals("source=EMP | lookup DEPT DEPTNO", anonymize("source=EMP | lookup DEPT DEPTNO")); assertEquals( - "source=EMP | lookup DEPT DEPTNO as EMPNO, ID append ID, LOC as JOB, COUNTRY as COUNTRY2", + "source=table | lookup table DEPTNO", anonymize("source=EMP | lookup DEPT DEPTNO")); + assertEquals( + "source=table | lookup table DEPTNO as EMPNO, ID append ID, LOC as JOB, COUNTRY as" + + " COUNTRY2", anonymize( "source=EMP | lookup DEPT DEPTNO as EMPNO, ID append ID, LOC as JOB, COUNTRY as" + " COUNTRY2")); @@ -483,27 +529,32 @@ public void testLookup() { @Test public void testInSubquery() { assertEquals( - "source=t | where (id) in [ source=s | fields + uid ] | fields + id", + "source=table | where (identifier) in [ source=table | fields + identifier ] | fields +" + + " identifier", anonymize("source=t | where id in [source=s | fields uid] | fields id")); } @Test public void testExistsSubquery() { assertEquals( - "source=t | where exists [ source=s | where id = uid ] | fields + id", + "source=table | where exists [ source=table | where identifier = identifier ] | fields +" + + " identifier", anonymize("source=t | where exists [source=s | where id = uid ] | fields id")); } @Test public void testScalarSubquery() { assertEquals( - "source=t | where id = [ source=s | stats max(b) ] | fields + id", + "source=table | where identifier = [ source=table | stats max(identifier) ] | fields +" + + " identifier", anonymize("source=t | where id = [ source=s | stats max(b) ] | fields id")); assertEquals( - "source=t | eval id=[ source=s | stats max(b) ] | fields + id", + "source=table | eval identifier=[ source=table | stats max(identifier) ] | fields +" + + " identifier", anonymize("source=t | eval id = [ source=s | stats max(b) ] | fields id")); assertEquals( - "source=t | where id > [ source=s | where id = uid | stats max(b) ] | fields + id", + "source=table | where identifier > [ source=table | where identifier = identifier | stats" + + " max(identifier) ] | fields + identifier", anonymize( "source=t | where id > [ source=s | where id = uid | stats max(b) ] | fields id")); } @@ -511,14 +562,14 @@ public void testScalarSubquery() { @Test public void testCaseWhen() { assertEquals( - "source=t | eval level=cast(score >= ***,***,score >= *** and score < ***,*** else ***) |" - + " fields + level", + "source=table | eval identifier=case(identifier >= ***,***,identifier >= *** and identifier" + + " < ***,*** else ***) | fields + identifier", anonymize( "source=t | eval level=CASE(score >= 90, 'A', score >= 80 AND score < 90, 'B' else 'C')" + " | fields level")); assertEquals( - "source=t | eval level=cast(score >= ***,***,score >= *** and score < ***,***) | fields +" - + " level", + "source=table | eval identifier=case(identifier >= ***,***,identifier >= *** and identifier" + + " < ***,***) | fields + identifier", anonymize( "source=t | eval level=CASE(score >= 90, 'A', score >= 80 AND score < 90, 'B')" + " | fields level")); @@ -527,30 +578,30 @@ public void testCaseWhen() { @Test public void testCast() { assertEquals( - "source=t | eval id=cast(a as INTEGER) | fields + id", + "source=table | eval identifier=cast(identifier as INTEGER) | fields + identifier", anonymize("source=t | eval id=CAST(a AS INTEGER) | fields id")); assertEquals( - "source=t | eval id=cast(*** as DOUBLE) | fields + id", + "source=table | eval identifier=cast(*** as DOUBLE) | fields + identifier", anonymize("source=t | eval id=CAST('1' AS DOUBLE) | fields id")); } @Test public void testParse() { assertEquals( - "source=t | parse email '.+@(?.+)'", + "source=table | parse identifier '***'", anonymize("source=t | parse email '.+@(?.+)'")); assertEquals( - "source=t | parse email '.+@(?.+)' | fields + email,host", + "source=table | parse identifier '***' | fields + identifier,identifier", anonymize("source=t | parse email '.+@(?.+)' | fields email, host")); } @Test public void testGrok() { assertEquals( - "source=t | grok email '.+@%{HOSTNAME:host}'", + "source=table | grok identifier '***'", anonymize("source=t | grok email '.+@%{HOSTNAME:host}'")); assertEquals( - "source=t | grok email '.+@%{HOSTNAME:host}' | fields + email,host", + "source=table | grok identifier '***' | fields + identifier,identifier", anonymize("source=t | grok email '.+@%{HOSTNAME:host}' | fields email, host")); } @@ -561,17 +612,17 @@ public void testPatterns() { when(settings.getSettingValue(Key.PATTERN_MAX_SAMPLE_COUNT)).thenReturn(10); when(settings.getSettingValue(Key.PATTERN_BUFFER_LIMIT)).thenReturn(100000); assertEquals( - "source=t | patterns email method=SIMPLE_PATTERN mode=LABEL" - + " max_sample_count=*** buffer_limit=*** new_field=patterns_field", + "source=table | patterns identifier method=SIMPLE_PATTERN mode=LABEL" + + " max_sample_count=*** buffer_limit=*** new_field=identifier", anonymize("source=t | patterns email")); assertEquals( - "source=t | patterns email method=SIMPLE_PATTERN mode=LABEL" - + " max_sample_count=*** buffer_limit=*** new_field=patterns_field |" - + " fields + email,patterns_field", - anonymize("source=t | patterns email | fields email, patterns_field")); + "source=table | patterns identifier method=SIMPLE_PATTERN mode=LABEL" + + " max_sample_count=*** buffer_limit=*** new_field=identifier |" + + " fields + identifier,identifier", + anonymize("source=t | patterns email | fields email, identifier")); assertEquals( - "source=t | patterns email method=BRAIN mode=AGGREGATION" - + " max_sample_count=*** buffer_limit=*** new_field=patterns_field" + "source=table | patterns identifier method=BRAIN mode=AGGREGATION" + + " max_sample_count=*** buffer_limit=*** new_field=identifier" + " variable_count_threshold=***", anonymize( "source=t | patterns email method=BRAIN mode=AGGREGATION" @@ -581,11 +632,11 @@ public void testPatterns() { @Test public void testRegex() { assertEquals( - "source=t | regex fieldname=***", anonymize("source=t | regex fieldname='pattern'")); + "source=table | regex identifier=***", anonymize("source=t | regex fieldname='pattern'")); assertEquals( - "source=t | regex fieldname!=***", anonymize("source=t | regex fieldname!='pattern'")); + "source=table | regex identifier!=***", anonymize("source=t | regex fieldname!='pattern'")); assertEquals( - "source=t | regex email=*** | fields + email", + "source=table | regex identifier=*** | fields + identifier", anonymize("source=t | regex email='.*@domain.com' | fields email")); } @@ -594,15 +645,14 @@ public void testRexCommand() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10); assertEquals( - "source=t | rex field=message mode=extract \"(?[A-Z]+)\" max_match=1", + "source=table | rex field=identifier mode=extract \"***\" max_match=***", anonymize("source=t | rex field=message \"(?[A-Z]+)\"")); assertEquals( - "source=t | rex field=lastname mode=extract \"(?^[A-Z])\" max_match=1 | fields +" - + " lastname,initial", - anonymize( - "source=t | rex field=lastname \"(?^[A-Z])\" | fields lastname, initial")); + "source=table | rex field=identifier mode=extract \"***\" max_match=*** | fields +" + + " identifier,identifier", + anonymize("source=table | rex field=identifier \"***\" | fields identifier, identifier")); assertEquals( - "source=t | rex field=name mode=extract \"(?[A-Z])\" max_match=3", + "source=table | rex field=identifier mode=extract \"***\" max_match=***", anonymize("source=t | rex field=name \"(?[A-Z])\" max_match=3")); } @@ -611,10 +661,10 @@ public void testRexSedMode() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10); assertEquals( - "source=t | rex field=lastname mode=sed \"s/^[A-Z]/X/\" max_match=1", + "source=table | rex field=identifier mode=sed \"***\" max_match=***", anonymize("source=t | rex field=lastname mode=sed \"s/^[A-Z]/X/\"")); assertEquals( - "source=t | rex field=data mode=sed \"s/sensitive/clean/g\" max_match=1 | fields + data", + "source=table | rex field=identifier mode=sed \"***\" max_match=*** | fields + identifier", anonymize("source=t | rex field=data mode=sed \"s/sensitive/clean/g\" | fields data")); } @@ -622,7 +672,7 @@ public void testRexSedMode() { public void testMvjoin() { // Test mvjoin with array of strings assertEquals( - "source=t | eval result=mvjoin(array(***,***,***),***) | fields + result", + "source=table | eval identifier=mvjoin(array(***,***,***),***) | fields + identifier", anonymize("source=t | eval result=mvjoin(array('a', 'b', 'c'), ',') | fields result")); } @@ -631,8 +681,8 @@ public void testRexWithOffsetField() { when(settings.getSettingValue(Key.PPL_REX_MAX_MATCH_LIMIT)).thenReturn(10); assertEquals( - "source=t | rex field=message mode=extract \"(?[a-z]+)\" max_match=1" - + " offset_field=pos", + "source=table | rex field=identifier mode=extract \"***\" max_match=***" + + " offset_field=identifier", anonymize("source=t | rex field=message \"(?[a-z]+)\" offset_field=pos")); } diff --git a/protocol/build.gradle b/protocol/build.gradle index a52cce66c93..fe937648bbe 100644 --- a/protocol/build.gradle +++ b/protocol/build.gradle @@ -26,7 +26,7 @@ plugins { id 'java' id "io.freefair.lombok" id 'jacoco' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } dependencies { @@ -79,10 +79,10 @@ spotless { exclude '**/build/**', '**/build-*/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseWithSanitizer.java b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseWithSanitizer.java index 69a29c5cce7..2c12e9e23be 100644 --- a/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseWithSanitizer.java +++ b/protocol/src/main/java/org/opensearch/sql/protocol/response/format/FlatResponseWithSanitizer.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.protocol.response.format; import com.google.common.collect.ImmutableSet; diff --git a/sql/build.gradle b/sql/build.gradle index bd5092bf4f4..2278496a46d 100644 --- a/sql/build.gradle +++ b/sql/build.gradle @@ -27,7 +27,7 @@ plugins { id "io.freefair.lombok" id 'jacoco' id 'antlr' - id 'com.diffplug.spotless' version '6.22.0' + id 'com.diffplug.spotless' } generateGrammarSource { @@ -66,10 +66,10 @@ spotless { exclude '**/build/**', '**/build-*/**', '**/gen/**' } importOrder() -// licenseHeader("/*\n" + -// " * Copyright OpenSearch Contributors\n" + -// " * SPDX-License-Identifier: Apache-2.0\n" + -// " */\n\n") + licenseHeader("/*\n" + + " * Copyright OpenSearch Contributors\n" + + " * SPDX-License-Identifier: Apache-2.0\n" + + " */\n\n") removeUnusedImports() trimTrailingWhitespace() endWithNewline() diff --git a/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java index 09102b57791..dfb045f345a 100644 --- a/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java +++ b/sql/src/main/java/org/opensearch/sql/sql/parser/AstStatementBuilder.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.sql.parser; diff --git a/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java b/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java index 87f2083774c..1a2e9518ef0 100644 --- a/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java +++ b/sql/src/test/java/org/opensearch/sql/common/antlr/SyntaxParserTestBase.java @@ -1,3 +1,8 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + package org.opensearch.sql.common.antlr; import static org.junit.jupiter.api.Assertions.assertNotNull;