diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index a930e557fd1..7e7b2cc25d5 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -87,6 +87,7 @@ import org.opensearch.sql.ast.tree.Rename; import org.opensearch.sql.ast.tree.Reverse; import org.opensearch.sql.ast.tree.Rex; +import org.opensearch.sql.ast.tree.SPath; import org.opensearch.sql.ast.tree.Search; import org.opensearch.sql.ast.tree.Sort; import org.opensearch.sql.ast.tree.Sort.SortOption; @@ -96,6 +97,7 @@ import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; +import org.opensearch.sql.ast.tree.Window; import org.opensearch.sql.common.antlr.SyntaxCheckException; import org.opensearch.sql.data.model.ExprMissingValue; import org.opensearch.sql.data.type.ExprCoreType; @@ -755,11 +757,21 @@ public LogicalPlan visitReverse(Reverse node, AnalysisContext context) { throw getOnlyForCalciteException("Reverse"); } + @Override + public LogicalPlan visitSpath(SPath node, AnalysisContext context) { + throw getOnlyForCalciteException("Spath"); + } + @Override public LogicalPlan visitTimechart(Timechart node, AnalysisContext context) { throw getOnlyForCalciteException("Timechart"); } + @Override + public LogicalPlan visitWindow(Window node, AnalysisContext context) { + throw getOnlyForCalciteException("Window"); + } + @Override public LogicalPlan visitRegex(Regex node, AnalysisContext context) { throw getOnlyForCalciteException("Regex"); diff --git a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java index 013ee75829a..be2788a547e 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/ExpressionAnalyzer.java @@ -38,6 +38,7 @@ import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; +import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.Literal; import org.opensearch.sql.ast.expression.Not; import org.opensearch.sql.ast.expression.Or; @@ -479,6 +480,11 @@ public Expression visitInSubquery(InSubquery node, AnalysisContext context) { throw getOnlyForCalciteException("Subsearch"); } + @Override + public Expression visitLambdaFunction(LambdaFunction node, AnalysisContext context) { + throw getOnlyForCalciteException("Lambda function"); + } + /** * If QualifiedName is actually a reserved metadata field, return the expr type associated with * the metadata field. diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 85dc1218f3e..4a54ae9b22c 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -6,6 +6,7 @@ package org.opensearch.sql.calcite; import static org.apache.calcite.sql.SqlKind.AS; +import static org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver.INFORMATION_SCHEMA_NAME; import static org.opensearch.sql.ast.tree.Join.JoinType.ANTI; import static org.opensearch.sql.ast.tree.Join.JoinType.SEMI; import static org.opensearch.sql.ast.tree.Sort.NullOrder.NULL_FIRST; @@ -20,6 +21,7 @@ import static org.opensearch.sql.calcite.utils.PlanUtils.getRelation; import static org.opensearch.sql.calcite.utils.PlanUtils.getRexCall; import static org.opensearch.sql.calcite.utils.PlanUtils.transformPlanToAttachChild; +import static org.opensearch.sql.utils.SystemIndexUtils.DATASOURCES_TABLE_NAME; import com.google.common.base.Strings; import com.google.common.collect.ImmutableList; @@ -66,6 +68,7 @@ import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.tuple.Pair; import org.checkerframework.checker.nullness.qual.Nullable; +import org.opensearch.sql.analysis.DataSourceSchemaIdentifierNameResolver; import org.opensearch.sql.ast.AbstractNodeVisitor; import org.opensearch.sql.ast.EmptySourcePropagateVisitor; import org.opensearch.sql.ast.Node; @@ -139,6 +142,7 @@ import org.opensearch.sql.calcite.utils.WildcardUtils; import org.opensearch.sql.common.patterns.PatternUtils; import org.opensearch.sql.common.utils.StringUtils; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.CalciteUnsupportedException; import org.opensearch.sql.exception.SemanticCheckException; import org.opensearch.sql.expression.function.BuiltinFunctionName; @@ -151,10 +155,12 @@ public class CalciteRelNodeVisitor extends AbstractNodeVisitor dedupeFields = diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 1c62e9d7163..25d7af3bb2e 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -48,6 +48,7 @@ import org.opensearch.sql.ast.expression.Compare; import org.opensearch.sql.ast.expression.EqualTo; import org.opensearch.sql.ast.expression.Function; +import org.opensearch.sql.ast.expression.HighlightFunction; import org.opensearch.sql.ast.expression.In; import org.opensearch.sql.ast.expression.Interval; import org.opensearch.sql.ast.expression.LambdaFunction; @@ -57,6 +58,7 @@ import org.opensearch.sql.ast.expression.Or; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.RelevanceFieldList; +import org.opensearch.sql.ast.expression.ScoreFunction; import org.opensearch.sql.ast.expression.Span; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedArgument; @@ -651,6 +653,16 @@ public RexNode visitWhen(When node, CalcitePlanContext context) { throw new CalciteUnsupportedException("CastWhen function is unsupported in Calcite"); } + @Override + public RexNode visitHighlightFunction(HighlightFunction node, CalcitePlanContext context) { + throw new CalciteUnsupportedException("Highlight function is unsupported in Calcite"); + } + + @Override + public RexNode visitScoreFunction(ScoreFunction node, CalcitePlanContext context) { + throw new CalciteUnsupportedException("Score function is unsupported in Calcite"); + } + @Override public RexNode visitRelevanceFieldList(RelevanceFieldList node, CalcitePlanContext context) { List varArgRexNodeList = new ArrayList<>(); diff --git a/core/src/main/java/org/opensearch/sql/executor/QueryService.java b/core/src/main/java/org/opensearch/sql/executor/QueryService.java index 7afbfdf5ba2..995d7d55e0d 100644 --- a/core/src/main/java/org/opensearch/sql/executor/QueryService.java +++ b/core/src/main/java/org/opensearch/sql/executor/QueryService.java @@ -1,9 +1,6 @@ /* + * Copyright OpenSearch Contributors * SPDX-License-Identifier: Apache-2.0 - * - * The OpenSearch Contributors require contributions made to - * this file be licensed under the Apache-2.0 license or a - * compatible open source license. */ package org.opensearch.sql.executor; @@ -12,6 +9,7 @@ import java.security.PrivilegedAction; import java.util.List; import java.util.Optional; +import javax.annotation.Nullable; import lombok.AllArgsConstructor; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -57,13 +55,12 @@ public class QueryService { private final Analyzer analyzer; private final ExecutionEngine executionEngine; private final Planner planner; - - @Getter(lazy = true) - private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor(); - private DataSourceService dataSourceService; private Settings settings; + @Getter(lazy = true) + private final CalciteRelNodeVisitor relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService); + /** Execute the {@link UnresolvedPlan}, using {@link ResponseListener} to get response.
*/ public void execute( UnresolvedPlan plan, @@ -107,7 +104,7 @@ public void executeWithCalcite( return null; }); } catch (Throwable t) { - if (isCalciteFallbackAllowed() && !(t instanceof NonFallbackCalciteException)) { + if (isCalciteFallbackAllowed(t) && !(t instanceof NonFallbackCalciteException)) { log.warn("Fallback to V2 query engine since got exception", t); executeWithLegacy(plan, queryType, listener, Optional.of(t)); } else { @@ -143,7 +140,7 @@ public void explainWithCalcite( return null; }); } catch (Throwable t) { - if (isCalciteFallbackAllowed()) { + if (isCalciteFallbackAllowed(t)) { log.warn("Fallback to V2 query engine since got exception", t); explainWithLegacy(plan, queryType, listener, format, Optional.of(t)); } else { @@ -165,7 +162,7 @@ public void executeWithLegacy( try { executePlan(analyze(plan, queryType), PlanContext.emptyPlanContext(), listener); } catch (Exception e) { - if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed()) { + if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed(null)) { // if there is a failure thrown from Calcite and execution after fallback V2 // keeps failure, we should throw the failure from Calcite. calciteFailure.ifPresentOrElse( @@ -198,7 +195,7 @@ public void explainWithLegacy( } executionEngine.explain(plan(analyze(plan, queryType)), listener); } catch (Exception e) { - if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed()) { + if (shouldUseCalcite(queryType) && isCalciteFallbackAllowed(null)) { // if there is a failure thrown from Calcite and execution after fallback V2 // keeps failure, we should throw the failure from Calcite. calciteFailure.ifPresentOrElse( @@ -263,11 +260,21 @@ public RelNode optimize(RelNode plan, CalcitePlanContext context) { SystemLimitType.QUERY_SIZE_LIMIT, plan, context.relBuilder.literal(context.querySizeLimit)); } - private boolean isCalciteFallbackAllowed() { - if (settings != null) { - return settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED); - } else { + private boolean isCalciteFallbackAllowed(@Nullable Throwable t) { + // We always allow fallback the query failed with CalciteUnsupportedException. + // This is for avoiding breaking changes when enable Calcite by default. + if (t instanceof CalciteUnsupportedException) { return true; + } else { + if (settings != null) { + Boolean fallback_allowed = settings.getSettingValue(Settings.Key.CALCITE_FALLBACK_ALLOWED); + if (fallback_allowed == null) { + return false; + } + return fallback_allowed; + } else { + return true; + } } } diff --git a/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java b/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java index b2de7c050fe..ed878ca622d 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/CalciteRelNodeVisitorSearchSimpleTest.java @@ -11,9 +11,11 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.Mock; import org.opensearch.sql.ast.dsl.AstDSL; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.Search; +import org.opensearch.sql.datasource.DataSourceService; /** * Simple tests for CalciteRelNodeVisitor.visitSearch method. Tests basic functionality without @@ -22,10 +24,11 @@ public class CalciteRelNodeVisitorSearchSimpleTest { private CalciteRelNodeVisitor visitor; + @Mock DataSourceService dataSourceService; @BeforeEach public void setUp() { - visitor = new CalciteRelNodeVisitor(); + visitor = new CalciteRelNodeVisitor(dataSourceService); } @Test diff --git a/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java b/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java index 62f3fadd326..ca9deb77061 100644 --- a/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java +++ b/core/src/test/java/org/opensearch/sql/calcite/CalciteRexNodeVisitorTest.java @@ -29,6 +29,7 @@ import org.opensearch.sql.ast.expression.LambdaFunction; import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.calcite.utils.CalciteToolsHelper; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.executor.QueryType; @ExtendWith(MockitoExtension.class) @@ -44,6 +45,7 @@ public class CalciteRexNodeVisitorTest { @Mock RelDataType accType; @Mock QualifiedName functionArg1; @Mock QualifiedName functionArg2; + @Mock DataSourceService dataSourceService; static CalciteRexNodeVisitor visitor; static CalciteRelNodeVisitor relNodeVisitor; @@ -57,7 +59,7 @@ public class CalciteRexNodeVisitorTest { @BeforeEach public void setUpContext() { - relNodeVisitor = new CalciteRelNodeVisitor(); + relNodeVisitor = new CalciteRelNodeVisitor(dataSourceService); visitor = new CalciteRexNodeVisitor(relNodeVisitor); when(relBuilder.getRexBuilder()).thenReturn(rexBuilder); when(rexBuilder.getTypeFactory()).thenReturn(TYPE_FACTORY); diff --git a/docs/category.json b/docs/category.json index 0d10dc4276f..f52e7ae67f2 100644 --- a/docs/category.json +++ b/docs/category.json @@ -11,7 +11,6 @@ "user/ppl/cmd/dedup.rst", "user/ppl/cmd/describe.rst", "user/ppl/cmd/showdatasources.rst", - "user/ppl/cmd/information_schema.rst", "user/ppl/cmd/eval.rst", "user/ppl/cmd/head.rst", "user/ppl/cmd/rare.rst", diff --git a/docs/dev/intro-v3-engine.md b/docs/dev/intro-v3-engine.md index ecd4f856a8b..9b23cc7ee81 100644 --- a/docs/dev/intro-v3-engine.md +++ b/docs/dev/intro-v3-engine.md @@ -1,4 +1,4 @@ -# PPL Engine V3 (for 3.0.0-beta) +# PPL Engine V3 --- ## 1. Motivations @@ -24,7 +24,7 @@ Find more details in [V3 Architecture](./intro-v3-architecture.md). --- ## 2. What's New -In the initial release of the V3 engine (3.0.0-beta), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior. +In the initial release of the V3 engine (3.0.0), the main new features focus on enhancing the PPL language while maintaining maximum compatibility with V2 behavior. * **[Join](../user/ppl/cmd/join.rst) Command** * **[Lookup](../user/ppl/cmd/lookup.rst) Command** @@ -51,40 +51,22 @@ Because of implementation changed internally, following behaviors are changed fr ### 3.2 Fallback Mechanism -As v3 engine is experimental in 3.0.0-beta, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...". +- As v3 engine is experimental in 3.0.0, not all PPL commands could work under this new engine. Those unsupported queries will be forwarded to V2 engine by fallback mechanism. To avoid impact on your side, normally you won't see any difference in a query response. If you want to check if and why your query falls back to be handled by V2 engine, please check OpenSearch log for "Fallback to V2 query engine since ...". +- Since 3.2.0, the fallback mechanism is disabled by default, to enable fallback, set `plugins.calcite.fallback.allowed=true`. ### 3.3 Limitations For the following functionalities in V3 engine, the query will be forwarded to the V2 query engine and thus you cannot use new features in [2. What's New](#2-whats-new). -#### Unsupported functionalities +#### Unsupported functionalities (up to latest) - All SQL queries -- `trendline` -- `show datasource` -- `explain` -- `describe` -- `top` and `rare` -- `fillnull` -- `patterns` +- PPL queries against non-OpenSearch data sources - `dedup` with `consecutive=true` - Search relevant commands - AD - ML - Kmeans - Commands with `fetch_size` parameter -- query with metadata fields, `_id`, `_doc`, etc. -- Json relevant functions - - cast to json - - json - - json_valid -- Search relevant functions - - match - - match_phrase - - match_bool_prefix - - match_phrase_prefix - - simple_query_string - - query_string - - multi_match - [Existed limitations of V2](intro-v2-engine.md#33-limitations) --- diff --git a/docs/user/admin/settings.rst b/docs/user/admin/settings.rst index e77b66d88cd..ce42f74574c 100644 --- a/docs/user/admin/settings.rst +++ b/docs/user/admin/settings.rst @@ -891,9 +891,10 @@ Description You can enable Calcite as new query optimizer and execution engine to all coming requests. -1. The default value is false since 3.0.0. -2. This setting is node scope. -3. This setting can be updated dynamically. +1. The default value is false in 3.0, 3.1 and 3.2. +2. The default value is true since 3.3.0. +3. This setting is node scope. +4. This setting can be updated dynamically. Check `introduce v3 engine <../../../dev/intro-v3-engine.md>`_ for more details. Check `join doc <../../ppl/cmd/join.rst>`_ for example. diff --git a/docs/user/dql/basics.rst b/docs/user/dql/basics.rst index a59f1930869..5fb8f765a2b 100644 --- a/docs/user/dql/basics.rst +++ b/docs/user/dql/basics.rst @@ -1155,8 +1155,8 @@ Offset position can be given following the OFFSET keyword as well, here is an ex +-----+ -Limitation ----------- +Limitations +----------- Generally, sort plan is pushed down into the OpenSearch DSL in plan optimization, but note that if a query has complex sorting, like sort expression, which would not be pushed down during optimization (see `Optimizations <../optimization/optimization.rst>`_ for details), but computed in local memory. However, the engine fetches the index of a default size that is set in plugin setting (See `Settings <../admin/settings.rst>` plugins.query.size_limit for details). Therefore, the result might not be absolutely correct if the index size is larger than the default size of index scan. For example, the engine has a index scan size of 200 and the index size is 500. Then a query with limit 300 can only fetch 200 rows of the index, compute and return the sorted result with 200 rows, while the rest 300 rows of the index are ignored and would not be fetched into the engine. To get an absolutely correct result, it is suggested to set the query size limit to a larger value before run the query. diff --git a/docs/user/limitations/limitations.rst b/docs/user/limitations/limitations.rst index 22ad3c2a17f..bc55b0045b8 100644 --- a/docs/user/limitations/limitations.rst +++ b/docs/user/limitations/limitations.rst @@ -130,3 +130,25 @@ The response in JSON format is:: }, "status": 400 } + +Limitations on Calcite Engine +============================= + +Since 3.0.0, we introduce Apache Calcite as an experimental query engine. Please see `introduce v3 engine <../../../dev/intro-v3-engine.md>`_. +For the following functionalities, the query will be forwarded to the V2 query engine. It means following functionalities cannot work together with new PPL commands/functions introduced in 3.0.0 and above. + +* All SQL queries + +* PPL queries against non-OpenSearch data sources + +* ``dedup`` with ``consecutive=true`` + +* Search relevant commands + + * AD + * ML + * Kmeans + +* ``show datasources`` command + +* Commands with ``fetch_size`` parameter diff --git a/docs/user/ppl/admin/cross_cluster_search.rst b/docs/user/ppl/admin/cross_cluster_search.rst index 4b267a93406..a94a0dce67e 100644 --- a/docs/user/ppl/admin/cross_cluster_search.rst +++ b/docs/user/ppl/admin/cross_cluster_search.rst @@ -50,8 +50,8 @@ Example PPL query:: +----------------+-----------+----------------------+---------+--------+--------+----------+-------+-----+-----------------------+----------+ -Limitation -========== +Limitations +=========== Since OpenSearch does not support cross cluster index metadata retrieval, field mapping of a remote cluster index is not available to the local cluster. (`[Feature] Cross cluster field mappings query #6573 `_) Therefore, the query engine requires that for any remote cluster index that the users need to search, diff --git a/docs/user/ppl/admin/datasources.rst b/docs/user/ppl/admin/datasources.rst index 90dd43dc5ec..c5f9adfd85a 100644 --- a/docs/user/ppl/admin/datasources.rst +++ b/docs/user/ppl/admin/datasources.rst @@ -24,7 +24,7 @@ Refer below sections for quick setup. Definitions of datasource and connector -==================================== +======================================= * Connector is a component that adapts the query engine to a datastore. For example, Prometheus connector would adapt and help execute the queries to run on Prometheus datastore. connector name is enough in the datasource definition json. * Datasource is a construct to define how to connect to a data store and which connector to adapt by query engine. @@ -56,7 +56,7 @@ Datasource configuration Restrictions. Datasource configuration APIs -====================================== +============================= Datasource configuration can be managed using below REST APIs. All the examples below are for OpenSearch domains enabled with secure domain. we can remove authorization and other details in case of security disabled domains. @@ -158,14 +158,14 @@ Master Key config for encrypting credential information print("Generated master key:", master_key) Datasource URI Hosts Deny Lists Config -======================================================== +====================================== * In the OpenSearch configuration file (opensearch.yml), the parameter "plugins.query.datasources.uri.hosts.denylist" can be utilized to control the permitted host ips within the datasource URI configuration. * By default, the value is set to empty list, which allows any domain to be accepted. * For instance, if you set the value to `127.0.0.0/8`, ppl plugins will deny all the query requests where the datasource URI resolves to the ip range from `127.0.0.0` to `127.255.255.255` Using a datasource in PPL command -==================================== +================================= Datasource is referred in source command as show in the code block below. Based on the abstraction designed by the connector, one can refer the corresponding entity as table in the source command. @@ -201,7 +201,7 @@ Moving from keystore datasource configuration * To port previously configured datasources from the keystore, users can use the `create datasource` REST API mentioned in the above section. Disabling a datasource to block new queries -============================================= +=========================================== * We can disable a datasource using PATCH or PUT API. Below is the example request for disabling a datasource named "my_prometheus" using PATCH API. :: PATCH https://localhost:9200/_plugins/_query/_datasources diff --git a/docs/user/ppl/cmd/ad.rst b/docs/user/ppl/cmd/ad.rst index 5d7a572c968..938e6e79918 100644 --- a/docs/user/ppl/cmd/ad.rst +++ b/docs/user/ppl/cmd/ad.rst @@ -1,6 +1,6 @@ -============= +============================= ad (deprecated by ml command) -============= +============================= .. rubric:: Table of contents @@ -103,3 +103,9 @@ PPL query:: | night | 10844.0 | 0.0 | False | | day | 6526.0 | 0.0 | False | +----------+---------+-------+-----------+ + + +Limitations +=========== +The ``ad`` command can only work with ``plugins.calcite.enabled=false``. +It means ``ad`` command cannot work together with new PPL commands/functions introduced in 3.0.0 and above. diff --git a/docs/user/ppl/cmd/dedup.rst b/docs/user/ppl/cmd/dedup.rst index 362d1637f77..d897c6dcdad 100644 --- a/docs/user/ppl/cmd/dedup.rst +++ b/docs/user/ppl/cmd/dedup.rst @@ -109,6 +109,7 @@ PPL query:: | 18 | M | +----------------+--------+ -Limitation -========== -The ``dedup`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. +Limitations +=========== +The ``dedup`` with ``consecutive=true`` command can only work with ``plugins.calcite.enabled=false``. +It means ``dedup`` with ``consecutive=true`` command cannot work together with new PPL commands/functions introduced in 3.0.0 and above. diff --git a/docs/user/ppl/cmd/head.rst b/docs/user/ppl/cmd/head.rst index cd4aed5a54d..c13a495a77f 100644 --- a/docs/user/ppl/cmd/head.rst +++ b/docs/user/ppl/cmd/head.rst @@ -73,6 +73,6 @@ PPL query:: | Dale | 33 | +-----------+-----+ -Limitation -========== +Limitations +=========== The ``head`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/information_schema.rst b/docs/user/ppl/cmd/information_schema.rst deleted file mode 100644 index 4210502edac..00000000000 --- a/docs/user/ppl/cmd/information_schema.rst +++ /dev/null @@ -1,57 +0,0 @@ -========================================= -Metadata queries using information_schema -========================================= - -.. rubric:: Table of contents - -.. contents:: - :local: - :depth: 2 - - -Description -============ -| Use ``information_schema`` in source command to query tables information under a datasource. -In the current state, ``information_schema`` only support metadata of tables. -This schema will be extended for views, columns and other metadata info in future. - - -Syntax -============ -source = datasource.information_schema.tables; - -Example 1: Fetch tables in prometheus datasource. -============================================== - -The examples fetches tables in the prometheus datasource. - -PPL query for fetching PROMETHEUS TABLES with where clause:: - - os> source = my_prometheus.information_schema.tables | where TABLE_NAME='prometheus_http_requests_total' - fetched rows / total rows = 1/1 - +---------------+--------------+--------------------------------+------------+------+---------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |---------------+--------------+--------------------------------+------------+------+---------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - +---------------+--------------+--------------------------------+------------+------+---------------------------+ - - -Example 2: Search tables in prometheus datasource. -================================================= - -The examples searches tables in the prometheus datasource. - -PPL query for searching PROMETHEUS TABLES:: - - os> source = my_prometheus.information_schema.tables | where LIKE(TABLE_NAME, "%http%"); - fetched rows / total rows = 6/6 - +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ - | TABLE_CATALOG | TABLE_SCHEMA | TABLE_NAME | TABLE_TYPE | UNIT | REMARKS | - |---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------| - | my_prometheus | default | prometheus_http_requests_total | counter | | Counter of HTTP requests. | - | my_prometheus | default | promhttp_metric_handler_requests_in_flight | gauge | | Current number of scrapes being served. | - | my_prometheus | default | prometheus_http_request_duration_seconds | histogram | | Histogram of latencies for HTTP requests. | - | my_prometheus | default | prometheus_sd_http_failures_total | counter | | Number of HTTP service discovery refresh failures. | - | my_prometheus | default | promhttp_metric_handler_requests_total | counter | | Total number of scrapes by HTTP status code. | - | my_prometheus | default | prometheus_http_response_size_bytes | histogram | | Histogram of response size for HTTP requests. | - +---------------+--------------+--------------------------------------------+------------+------+----------------------------------------------------+ diff --git a/docs/user/ppl/cmd/join.rst b/docs/user/ppl/cmd/join.rst index 45c091b5b4e..90f978d056c 100644 --- a/docs/user/ppl/cmd/join.rst +++ b/docs/user/ppl/cmd/join.rst @@ -175,8 +175,8 @@ PPL query:: | 70000.0 | 30 | USA | +-------------+----------+---------+ -Limitation -========== +Limitations +=========== For basic syntax in 3.0.0, if fields in the left outputs and right outputs have the same name. Typically, in the join criteria ``ON t1.id = t2.id``, the names ``id`` in output are ambiguous. To avoid ambiguous, the ambiguous fields in output rename to ``.id``, or else ``.id`` if no alias existing. diff --git a/docs/user/ppl/cmd/kmeans.rst b/docs/user/ppl/cmd/kmeans.rst index faf29d078b8..6d558248ee4 100644 --- a/docs/user/ppl/cmd/kmeans.rst +++ b/docs/user/ppl/cmd/kmeans.rst @@ -1,6 +1,6 @@ -============= +================================= kmeans (deprecated by ml command) -============= +================================= .. rubric:: Table of contents @@ -10,7 +10,7 @@ kmeans (deprecated by ml command) Description -============ +=========== | The ``kmeans`` command applies the kmeans algorithm in the ml-commons plugin on the search result returned by a PPL command. @@ -38,3 +38,9 @@ PPL query:: | 5.6 | 3.0 | 4.1 | 1.3 | 0 | | 6.7 | 2.5 | 5.8 | 1.8 | 2 | +--------------------+-------------------+--------------------+-------------------+-----------+ + + +Limitations +=========== +The ``kmeans`` command can only work with ``plugins.calcite.enabled=false``. +It means ``kmeans`` command cannot work together with new PPL commands/functions introduced in 3.0.0 and above. \ No newline at end of file diff --git a/docs/user/ppl/cmd/ml.rst b/docs/user/ppl/cmd/ml.rst index a48c1ec5896..f38697adbbc 100644 --- a/docs/user/ppl/cmd/ml.rst +++ b/docs/user/ppl/cmd/ml.rst @@ -1,6 +1,6 @@ -============= +== ml -============= +== .. rubric:: Table of contents @@ -134,3 +134,9 @@ PPL query:: | 5.6 | 3.0 | 4.1 | 1.3 | 0 | | 6.7 | 2.5 | 5.8 | 1.8 | 2 | +--------------------+-------------------+--------------------+-------------------+-----------+ + + +Limitations +=========== +The ``ml`` command can only work with ``plugins.calcite.enabled=false``. +It means ``ml`` command cannot work together with new PPL commands/functions introduced in 3.0.0 and above. diff --git a/docs/user/ppl/cmd/rare.rst b/docs/user/ppl/cmd/rare.rst index 456a9360bc9..ad0202b8e88 100644 --- a/docs/user/ppl/cmd/rare.rst +++ b/docs/user/ppl/cmd/rare.rst @@ -96,6 +96,6 @@ PPL query:: | M | 3 | +--------+-----+ -Limitation -========== +Limitations +=========== The ``rare`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/rename.rst b/docs/user/ppl/cmd/rename.rst index f5f129ca8f4..ed7f806aad1 100644 --- a/docs/user/ppl/cmd/rename.rst +++ b/docs/user/ppl/cmd/rename.rst @@ -132,6 +132,6 @@ PPL query:: +---------+ -Limitation -========== +Limitations +=========== The ``rename`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/showdatasources.rst b/docs/user/ppl/cmd/showdatasources.rst index d954ef0c04a..f12622f54da 100644 --- a/docs/user/ppl/cmd/showdatasources.rst +++ b/docs/user/ppl/cmd/showdatasources.rst @@ -34,3 +34,8 @@ PPL query for all PROMETHEUS DATASOURCES:: | my_prometheus | PROMETHEUS | +-----------------+----------------+ + +Limitations +=========== +The ``show datasources`` command can only work with ``plugins.calcite.enabled=false``. +It means ``show datasources`` command cannot work together with new PPL commands/functions introduced in 3.0.0 and above. diff --git a/docs/user/ppl/cmd/top.rst b/docs/user/ppl/cmd/top.rst index e6df0455a2f..ec22c9b5db9 100644 --- a/docs/user/ppl/cmd/top.rst +++ b/docs/user/ppl/cmd/top.rst @@ -108,6 +108,6 @@ PPL query:: | F | 1 | +--------+-----+ -Limitation -========== +Limitations +=========== The ``top`` command is not rewritten to OpenSearch DSL, it is only executed on the coordination node. diff --git a/docs/user/ppl/cmd/trendline.rst b/docs/user/ppl/cmd/trendline.rst index f1229222520..d7fb6544ae6 100644 --- a/docs/user/ppl/cmd/trendline.rst +++ b/docs/user/ppl/cmd/trendline.rst @@ -130,6 +130,6 @@ PPL query:: | 16.333333333333332 | +--------------------------+ -Limitation -========== +Limitations +=========== Starting with version 3.1.0, the ``trendline`` command requires all values in the specified ``field`` to be non-null. Any rows with null values present in the calculation field will be automatically excluded from the command's output. \ No newline at end of file diff --git a/docs/user/ppl/index.rst b/docs/user/ppl/index.rst index 40b9b72666b..2329dc1d0dd 100644 --- a/docs/user/ppl/index.rst +++ b/docs/user/ppl/index.rst @@ -84,8 +84,6 @@ The query start with search command and then flowing a set of command delimited - `lookup command `_ - - `metadata commands `_ - - `ml command `_ - `multisearch command `_ diff --git a/docs/user/ppl/limitations/limitations.rst b/docs/user/ppl/limitations/limitations.rst index ebef537523e..f9c620ff18b 100644 --- a/docs/user/ppl/limitations/limitations.rst +++ b/docs/user/ppl/limitations/limitations.rst @@ -89,10 +89,12 @@ Unsupported Functionalities in Calcite Engine ============================================= Since 3.0.0, we introduce Apache Calcite as an experimental query engine. Please see `introduce v3 engine <../../../dev/intro-v3-engine.md>`_. -For the following functionalities, the query will be forwarded to the V2 query engine. +For the following functionalities, the query will be forwarded to the V2 query engine. It means following functionalities cannot work with new PPL commands/functions introduced in 3.0.0 and above. * All SQL queries +* PPL Queries against non-OpenSearch data sources + * ``dedup`` with ``consecutive=true`` * Search relevant commands @@ -101,4 +103,6 @@ For the following functionalities, the query will be forwarded to the V2 query e * ML * Kmeans +* ``show datasources`` and command + * Commands with ``fetch_size`` parameter diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java index ebf3c600f6e..fc9604d62fd 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalcitePPLDedupIT.java @@ -93,39 +93,34 @@ public void testDedupMultipleFieldsKeepEmpty() throws IOException { } @Test - @SuppressWarnings("ThrowableNotThrown") - public void testConsecutiveThrowException() { - assertThrowsWithReplace( - UnsupportedOperationException.class, - () -> - executeQuery( - String.format( - "source = %s | dedup 1 name CONSECUTIVE=true | fields name", - TEST_INDEX_DUPLICATION_NULLABLE))); + public void testConsecutiveImplicitFallbackV2() throws IOException { + JSONObject actual = + executeQuery( + String.format( + "source = %s | dedup 1 name CONSECUTIVE=true | fields name", + TEST_INDEX_DUPLICATION_NULLABLE)); + verifyNumOfRows(actual, 8); - assertThrowsWithReplace( - UnsupportedOperationException.class, - () -> - executeQuery( - String.format( - "source = %s | dedup 1 name KEEPEMPTY=true CONSECUTIVE=true | fields name", - TEST_INDEX_DUPLICATION_NULLABLE))); + actual = + executeQuery( + String.format( + "source = %s | dedup 1 name KEEPEMPTY=true CONSECUTIVE=true | fields name", + TEST_INDEX_DUPLICATION_NULLABLE)); + verifyNumOfRows(actual, 12); - assertThrowsWithReplace( - UnsupportedOperationException.class, - () -> - executeQuery( - String.format( - "source = %s | dedup 2 name CONSECUTIVE=true | fields name", - TEST_INDEX_DUPLICATION_NULLABLE))); + actual = + executeQuery( + String.format( + "source = %s | dedup 2 name CONSECUTIVE=true | fields name", + TEST_INDEX_DUPLICATION_NULLABLE)); + verifyNumOfRows(actual, 12); - assertThrowsWithReplace( - UnsupportedOperationException.class, - () -> - executeQuery( - String.format( - "source = %s | dedup 2 name KEEPEMPTY=true CONSECUTIVE=true | fields name", - TEST_INDEX_DUPLICATION_NULLABLE))); + actual = + executeQuery( + String.format( + "source = %s | dedup 2 name KEEPEMPTY=true CONSECUTIVE=true | fields name", + TEST_INDEX_DUPLICATION_NULLABLE)); + verifyNumOfRows(actual, 16); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 3c931097259..b45bf978125 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -44,6 +44,7 @@ public abstract class PPLIntegTestCase extends SQLIntegTestCase { protected void init() throws Exception { super.init(); updatePushdownSettings(); + disableCalcite(); // calcite is enabled by default from 3.3.0 } protected JSONObject executeQuery(String query) throws IOException { diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java index b8a23378e2b..aa247234e0a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/setting/OpenSearchSettings.java @@ -148,7 +148,7 @@ public class OpenSearchSettings extends Settings { public static final Setting CALCITE_ENGINE_ENABLED_SETTING = Setting.boolSetting( Key.CALCITE_ENGINE_ENABLED.getKeyValue(), - false, + true, Setting.Property.NodeScope, Setting.Property.Dynamic); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java index 1252aca48a2..56041984c4d 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLAbstractTest.java @@ -8,6 +8,7 @@ import static org.apache.calcite.test.Matchers.hasTree; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doReturn; import static org.mockito.Mockito.mock; import static org.opensearch.sql.executor.QueryType.PPL; @@ -41,6 +42,7 @@ import org.opensearch.sql.calcite.CalciteRelNodeVisitor; import org.opensearch.sql.common.setting.Settings; import org.opensearch.sql.common.setting.Settings.Key; +import org.opensearch.sql.datasource.DataSourceService; import org.opensearch.sql.exception.ExpressionEvaluationException; import org.opensearch.sql.ppl.antlr.PPLSyntaxParser; import org.opensearch.sql.ppl.parser.AstBuilder; @@ -51,11 +53,13 @@ public class CalcitePPLAbstractTest { private final CalciteRelNodeVisitor planTransformer; private final RelToSqlConverter converter; protected final Settings settings; + private final DataSourceService dataSourceService; public PPLSyntaxParser pplParser = new PPLSyntaxParser(); public CalcitePPLAbstractTest(CalciteAssert.SchemaSpec... schemaSpecs) { this.config = config(schemaSpecs); - this.planTransformer = new CalciteRelNodeVisitor(); + this.dataSourceService = mock(DataSourceService.class); + this.planTransformer = new CalciteRelNodeVisitor(dataSourceService); this.converter = new RelToSqlConverter(OpenSearchSparkSqlDialect.DEFAULT); this.settings = mock(Settings.class); } @@ -65,6 +69,7 @@ public void init() { doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_ENGINE_ENABLED); doReturn(true).when(settings).getSettingValue(Settings.Key.CALCITE_SUPPORT_ALL_JOIN_TYPES); doReturn(true).when(settings).getSettingValue(Settings.Key.PPL_SYNTAX_LEGACY_PREFERRED); + doReturn(false).when(dataSourceService).dataSourceExists(any()); } protected Frameworks.ConfigBuilder config(CalciteAssert.SchemaSpec... schemaSpecs) {