From d02ff605edbde0905ea242dd6b4ce1f1d987c2eb Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 14 Jan 2026 14:17:03 -0800 Subject: [PATCH 01/36] fielfformat changes Signed-off-by: Asif Bashar # Conflicts: # ppl/src/main/antlr/OpenSearchPPLLexer.g4 # ppl/src/main/antlr/OpenSearchPPLParser.g4 # Conflicts: # ppl/src/main/antlr/OpenSearchPPLParser.g4 --- .../org/opensearch/sql/analysis/Analyzer.java | 7 + .../sql/ast/AbstractNodeVisitor.java | 5 + .../opensearch/sql/ast/tree/FieldFormat.java | 43 +++++ .../sql/calcite/CalciteRelNodeVisitor.java | 33 ++++ docs/category.json | 1 + docs/user/ppl/cmd/fieldformat.md | 148 ++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 8 +- 8 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java create mode 100644 docs/user/ppl/cmd/fieldformat.md diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 2b2fe3cf1c..80abf724ee 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -70,6 +70,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -526,6 +527,12 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { return new LogicalEval(child, expressionsBuilder.build()); } + /** Build {@link LogicalEval}. */ + @Override + public LogicalPlan visitFieldFormat(FieldFormat node, AnalysisContext context) { + return visitFieldFormat(node, context); + } + @Override public LogicalPlan visitAddTotals(AddTotals node, AnalysisContext context) { throw getOnlyForCalciteException("addtotals"); diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index b1082759a3..fa1a63d850 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -58,6 +58,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -264,6 +265,10 @@ public T visitEval(Eval node, C context) { return visitChildren(node, context); } + public T visitFieldFormat(FieldFormat node, C context) { + return visitChildren(node, context); + } + public T visitParse(Parse node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java new file mode 100644 index 0000000000..cab0346835 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Let; + +/** AST node represent Eval operation. */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class FieldFormat extends UnresolvedPlan { + private final List expressionList; + private UnresolvedPlan child; + + @Override + public FieldFormat attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitFieldFormat(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 5825011f65..d8b1b40f40 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -115,6 +115,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -999,6 +1000,38 @@ public RelNode visitEval(Eval node, CalcitePlanContext context) { return context.relBuilder.peek(); } + @Override + public RelNode visitFieldFormat(FieldFormat node, CalcitePlanContext context) { + visitChildren(node, context); + node.getExpressionList() + .forEach( + expr -> { + boolean containsSubqueryExpression = containsSubqueryExpression(expr); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + if (containsSubqueryExpression) { + context.relBuilder.variable(v::set); + context.pushCorrelVar(v.get()); + } + RexNode eval = rexVisitor.analyze(expr, context); + if (containsSubqueryExpression) { + // RelBuilder.projectPlus doesn't have a parameter with variablesSet: + // projectPlus(Iterable variablesSet, RexNode... nodes) + context.relBuilder.project( + Iterables.concat(context.relBuilder.fields(), ImmutableList.of(eval)), + ImmutableList.of(), + false, + ImmutableList.of(v.get().id)); + context.popCorrelVar(); + } else { + // Overriding the existing field if the alias has the same name with original field. + String alias = + ((RexLiteral) ((RexCall) eval).getOperands().get(1)).getValueAs(String.class); + projectPlusOverriding(List.of(eval), List.of(alias), context); + } + }); + return context.relBuilder.peek(); + } + private void projectPlusOverriding( List newFields, List newNames, CalcitePlanContext context) { List originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); diff --git a/docs/category.json b/docs/category.json index d6c8c9dfb0..bcf73cb1a8 100644 --- a/docs/category.json +++ b/docs/category.json @@ -17,6 +17,7 @@ "user/ppl/cmd/describe.md", "user/ppl/cmd/eventstats.md", "user/ppl/cmd/eval.md", + "user/ppl/cmd/fieldformat.md", "user/ppl/cmd/fields.md", "user/ppl/cmd/fillnull.md", "user/ppl/cmd/grok.md", diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md new file mode 100644 index 0000000000..f898f00a3c --- /dev/null +++ b/docs/user/ppl/cmd/fieldformat.md @@ -0,0 +1,148 @@ + +# fieldformat + +The `fieldformat` command fieldformatuates the specified expression and appends the result of the fieldformatuation to the search results. + +> **Note**: The `fieldformat` command is not rewritten to [query domain-specific language (DSL)](https://docs.opensearch.org/latest/query-dsl/). It is only executed on the coordinating node. + +## Syntax + +The `fieldformat` command has the following syntax: + +```syntax +fieldformat = ["," = ]... +``` + +## Parameters + +The `fieldformat` command supports the following parameters. + +| Parameter | Required/Optional | Description | +| --- | --- | --- | +| `` | Required | The name of the field to create or update. If the field does not exist, a new field is added. If it already exists, its value is overwritten. | +| `` | Required | The expression to fieldformatuate. | + + +## Example 1: Create a new field + +The following query creates a new `doubleAge` field for each document: + +```ppl +source=accounts +| fieldformat doubleAge = age * 2 +| fields age, doubleAge +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++-----+-----------+ +| age | doubleAge | +|-----+-----------| +| 32 | 64 | +| 36 | 72 | +| 28 | 56 | +| 33 | 66 | ++-----+-----------+ +``` + + +## Example 2: Override an existing field + +The following query overrides the `age` field by adding `1` to its value: + +```ppl +source=accounts +| fieldformat age = age + 1 +| fields age +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++-----+ +| age | +|-----| +| 33 | +| 37 | +| 29 | +| 34 | ++-----+ +``` + + +## Example 3: Create a new field using a field defined in fieldformat + +The following query creates a new field based on another field defined in the same `fieldformat` expression. In this example, the new `ddAge` field is calculated by multiplying the `doubleAge` field by `2`. The `doubleAge` field itself is defined earlier in the `fieldformat` command: + +```ppl +source=accounts +| fieldformat doubleAge = age * 2, ddAge = doubleAge * 2 +| fields age, doubleAge, ddAge +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++-----+-----------+-------+ +| age | doubleAge | ddAge | +|-----+-----------+-------| +| 32 | 64 | 128 | +| 36 | 72 | 144 | +| 28 | 56 | 112 | +| 33 | 66 | 132 | ++-----+-----------+-------+ +``` + + +## Example 4: String concatenation + +The following query uses the `+` operator for string concatenation. You can concatenate string literals and field values as follows: + +```ppl +source=accounts +| fieldformat greeting = 'Hello ' + firstname +| fields firstname, greeting +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++-----------+---------------+ +| firstname | greeting | +|-----------+---------------| +| Amber | Hello Amber | +| Hattie | Hello Hattie | +| Nanette | Hello Nanette | +| Dale | Hello Dale | ++-----------+---------------+ +``` + + +## Example 5: Multiple string concatenation with type casting + +The following query performs multiple concatenation operations, including type casting from numeric values to strings: + +```ppl +source=accounts | fieldformat full_info = 'Name: ' + firstname + ', Age: ' + CAST(age AS STRING) | fields firstname, age, full_info +``` + +The query returns the following results: + +```text +fetched rows / total rows = 4/4 ++-----------+-----+------------------------+ +| firstname | age | full_info | +|-----------+-----+------------------------| +| Amber | 32 | Name: Amber, Age: 32 | +| Hattie | 36 | Name: Hattie, Age: 36 | +| Nanette | 28 | Name: Nanette, Age: 28 | +| Dale | 33 | Name: Dale, Age: 33 | ++-----------+-----+------------------------+ +``` + + diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 1939124eed..30250137f7 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -71,6 +71,7 @@ LABEL: 'LABEL'; SHOW_NUMBERED_TOKEN: 'SHOW_NUMBERED_TOKEN'; AGGREGATION: 'AGGREGATION'; APPENDPIPE: 'APPENDPIPE'; +FIELDFORMAT: 'FIELDFORMAT'; COLUMN_NAME: 'COLUMN_NAME'; MVCOMBINE: 'MVCOMBINE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 2131eeae93..b4551ab68e 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -89,7 +89,7 @@ commands | rexCommand | appendPipeCommand | replaceCommand - | mvcombineCommand + | fieldformatCommand ; commandName @@ -133,7 +133,7 @@ commandName | REX | APPENDPIPE | REPLACE - | MVCOMBINE + | FIELDFORMAT | TRANSPOSE ; @@ -545,10 +545,6 @@ expandCommand : EXPAND fieldExpression (AS alias = qualifiedName)? ; -mvcombineCommand - : MVCOMBINE fieldExpression (DELIM EQUAL stringLiteral)? - ; - flattenCommand : FLATTEN fieldExpression (AS aliases = identifierSeq)? ; From 3c62d8689f11a2760a5ad0550f624063a0a3f529 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 23 Jan 2026 17:57:45 -0800 Subject: [PATCH 02/36] added field format with concatenation of string , added more examples Signed-off-by: Asif Bashar --- .../org/opensearch/sql/analysis/Analyzer.java | 5 +- .../sql/ast/AbstractNodeVisitor.java | 3 +- .../opensearch/sql/ast/expression/Let.java | 17 +++++ .../opensearch/sql/ast/tree/FieldFormat.java | 43 ----------- .../sql/calcite/CalciteRelNodeVisitor.java | 33 -------- .../sql/calcite/CalciteRexNodeVisitor.java | 22 ++++++ docs/user/ppl/cmd/fieldformat.md | 76 +++++++------------ .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 20 +++-- .../opensearch/sql/ppl/PPLIntegTestCase.java | 7 +- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 17 ++++- .../opensearch/sql/ppl/parser/AstBuilder.java | 13 ++++ .../sql/ppl/parser/AstExpressionBuilder.java | 48 ++++++++++++ 14 files changed, 170 insertions(+), 137 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 80abf724ee..5015214796 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -70,7 +70,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -529,8 +528,8 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { /** Build {@link LogicalEval}. */ @Override - public LogicalPlan visitFieldFormat(FieldFormat node, AnalysisContext context) { - return visitFieldFormat(node, context); + public LogicalPlan visitFieldFormat(Eval node, AnalysisContext context) { + throw getOnlyForCalciteException("fieldformat"); } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index fa1a63d850..2486b63791 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -58,7 +58,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -265,7 +264,7 @@ public T visitEval(Eval node, C context) { return visitChildren(node, context); } - public T visitFieldFormat(FieldFormat node, C context) { + public T visitFieldFormat(Eval node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/expression/Let.java b/core/src/main/java/org/opensearch/sql/ast/expression/Let.java index ad9843fae1..abd6733e06 100644 --- a/core/src/main/java/org/opensearch/sql/ast/expression/Let.java +++ b/core/src/main/java/org/opensearch/sql/ast/expression/Let.java @@ -20,6 +20,8 @@ public class Let extends UnresolvedExpression { private final Field var; private final UnresolvedExpression expression; + private final Literal concatPrefix; + private final Literal concatSuffix; public Let(Field var, UnresolvedExpression expression) { String varName = var.getField().toString(); @@ -29,6 +31,21 @@ public Let(Field var, UnresolvedExpression expression) { } this.var = var; this.expression = expression; + this.concatPrefix = null; + this.concatSuffix = null; + } + + public Let( + Field var, UnresolvedExpression expression, Literal concatPrefix, Literal concatSuffix) { + String varName = var.getField().toString(); + if (OpenSearchConstants.METADATAFIELD_TYPE_MAP.containsKey(varName)) { + throw new IllegalArgumentException( + String.format("Cannot use metadata field [%s] as the eval field.", varName)); + } + this.var = var; + this.expression = expression; + this.concatPrefix = concatPrefix; + this.concatSuffix = concatSuffix; } @Override diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java deleted file mode 100644 index cab0346835..0000000000 --- a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ast.tree; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import org.opensearch.sql.ast.AbstractNodeVisitor; -import org.opensearch.sql.ast.expression.Let; - -/** AST node represent Eval operation. */ -@Getter -@Setter -@ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor -public class FieldFormat extends UnresolvedPlan { - private final List expressionList; - private UnresolvedPlan child; - - @Override - public FieldFormat attach(UnresolvedPlan child) { - this.child = child; - return this; - } - - @Override - public List getChild() { - return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); - } - - @Override - public T accept(AbstractNodeVisitor nodeVisitor, C context) { - return nodeVisitor.visitFieldFormat(this, context); - } -} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index d8b1b40f40..5825011f65 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -115,7 +115,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -1000,38 +999,6 @@ public RelNode visitEval(Eval node, CalcitePlanContext context) { return context.relBuilder.peek(); } - @Override - public RelNode visitFieldFormat(FieldFormat node, CalcitePlanContext context) { - visitChildren(node, context); - node.getExpressionList() - .forEach( - expr -> { - boolean containsSubqueryExpression = containsSubqueryExpression(expr); - final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); - if (containsSubqueryExpression) { - context.relBuilder.variable(v::set); - context.pushCorrelVar(v.get()); - } - RexNode eval = rexVisitor.analyze(expr, context); - if (containsSubqueryExpression) { - // RelBuilder.projectPlus doesn't have a parameter with variablesSet: - // projectPlus(Iterable variablesSet, RexNode... nodes) - context.relBuilder.project( - Iterables.concat(context.relBuilder.fields(), ImmutableList.of(eval)), - ImmutableList.of(), - false, - ImmutableList.of(v.get().id)); - context.popCorrelVar(); - } else { - // Overriding the existing field if the alias has the same name with original field. - String alias = - ((RexLiteral) ((RexCall) eval).getOperands().get(1)).getValueAs(String.class); - projectPlusOverriding(List.of(eval), List.of(alias), context); - } - }); - return context.relBuilder.peek(); - } - private void projectPlusOverriding( List newFields, List newNames, CalcitePlanContext context) { List originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java index 9354bcc332..04ddd4b767 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRexNodeVisitor.java @@ -321,6 +321,28 @@ public RexNode visitLambdaFunction(LambdaFunction node, CalcitePlanContext conte @Override public RexNode visitLet(Let node, CalcitePlanContext context) { RexNode expr = analyze(node.getExpression(), context); + if (node.getConcatPrefix() != null) { + + expr = + context.rexBuilder.makeCall( + SqlStdOperatorTable.CONCAT, + context.rexBuilder.makeLiteral( + node.getConcatPrefix().getValue(), + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + true), + expr); + } + if (node.getConcatSuffix() != null) { + + expr = + context.rexBuilder.makeCall( + SqlStdOperatorTable.CONCAT, + expr, + context.rexBuilder.makeLiteral( + node.getConcatSuffix().getValue(), + context.rexBuilder.getTypeFactory().createSqlType(SqlTypeName.VARCHAR), + true)); + } return context.relBuilder.alias(expr, node.getVar().getField().toString()); } diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index f898f00a3c..f11f7a4489 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -1,26 +1,30 @@ # fieldformat -The `fieldformat` command fieldformatuates the specified expression and appends the result of the fieldformatuation to the search results. +The `fieldformat` command sets the value to a field with the specified expression and appends the field with evaluated result to the search results. The command is a alias of eval command. +Additionally, it also provides string concatenation dot operator followed by and/or follows a string that will be concatenated to the expression. -> **Note**: The `fieldformat` command is not rewritten to [query domain-specific language (DSL)](https://docs.opensearch.org/latest/query-dsl/). It is only executed on the coordinating node. ## Syntax The `fieldformat` command has the following syntax: ```syntax -fieldformat = ["," = ]... +Following are possible syntaxes: + fieldformat =[(prefix).][.(suffix)] ["," =[(prefix).][.(suffix)] ]... + ``` ## Parameters The `fieldformat` command supports the following parameters. -| Parameter | Required/Optional | Description | -| --- | --- | --- | -| `` | Required | The name of the field to create or update. If the field does not exist, a new field is added. If it already exists, its value is overwritten. | -| `` | Required | The expression to fieldformatuate. | +| Parameter | Required/Optional | Description | +|----------------| |-----------------------------------------------------------------------------------------------------------------------------------------------| +| `` | Required | The name of the field to create or update. If the field does not exist, a new field is added. If it already exists, its value is overwritten. | +| `` | Required | The expression to evaluate. The expression can have a prefix and/or suffix string part that will be concatenated to the expression. | +| `prefix` | Optional | A string before the expression followed by dot operator which will be concatenated as prefix to the evaluated expression value. | +| `suffix` | Optional | A string that follows the expression and dot operator which will be concatenated as suffix to the evaluated expression value. | ## Example 1: Create a new field @@ -73,38 +77,15 @@ fetched rows / total rows = 4/4 ``` -## Example 3: Create a new field using a field defined in fieldformat - -The following query creates a new field based on another field defined in the same `fieldformat` expression. In this example, the new `ddAge` field is calculated by multiplying the `doubleAge` field by `2`. The `doubleAge` field itself is defined earlier in the `fieldformat` command: - -```ppl -source=accounts -| fieldformat doubleAge = age * 2, ddAge = doubleAge * 2 -| fields age, doubleAge, ddAge -``` - -The query returns the following results: - -```text -fetched rows / total rows = 4/4 -+-----+-----------+-------+ -| age | doubleAge | ddAge | -|-----+-----------+-------| -| 32 | 64 | 128 | -| 36 | 72 | 144 | -| 28 | 56 | 112 | -| 33 | 66 | 132 | -+-----+-----------+-------+ -``` -## Example 4: String concatenation +## Example 3: String concatenation with prefix -The following query uses the `+` operator for string concatenation. You can concatenate string literals and field values as follows: +The following query uses the `.` (dot) operator for string concatenation. You can concatenate string literals and field values as follows: ```ppl source=accounts -| fieldformat greeting = 'Hello ' + firstname +| fieldformat greeting = 'Hello '.tostring( firstname) | fields firstname, greeting ``` @@ -123,26 +104,25 @@ fetched rows / total rows = 4/4 ``` -## Example 5: Multiple string concatenation with type casting +## Example 5: string concatenation with dot operator , prefix and suffix + +The following query performs prefix and suffix string concatenation operations using dot operator: -The following query performs multiple concatenation operations, including type casting from numeric values to strings: - ```ppl -source=accounts | fieldformat full_info = 'Name: ' + firstname + ', Age: ' + CAST(age AS STRING) | fields firstname, age, full_info +source=accounts | fieldformat age_info = 'Age: '.CAST(age AS STRING).' years.' | fields firstname, age, age_info ``` - + The query returns the following results: - + ```text fetched rows / total rows = 4/4 -+-----------+-----+------------------------+ -| firstname | age | full_info | -|-----------+-----+------------------------| -| Amber | 32 | Name: Amber, Age: 32 | -| Hattie | 36 | Name: Hattie, Age: 36 | -| Nanette | 28 | Name: Nanette, Age: 28 | -| Dale | 33 | Name: Dale, Age: 33 | -+-----------+-----+------------------------+ -``` ++-----------+-----+----------------+ +| firstname | age | age_info | +|-----------+-----+----------------| +| Amber | 32 | Age: 32 years. | +| Hattie | 36 | Age: 36 years. | +| Nanette | 28 | Age: 28 years. | +| Dale | 33 | Age: 33 years. | ++-----------+-----+----------------+ diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 0010fe13ff..50cdd8d847 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -34,6 +34,7 @@ CalciteDedupCommandIT.class, CalciteDescribeCommandIT.class, CalciteExpandCommandIT.class, + CalciteFieldFormatCommandIT.class, CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 4846480fc1..09936d4846 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2477,13 +2477,23 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { @Test public void testAggFilterOnNestedFields() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_filter_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED))); + + } + @Test + public void testFieldFormatExplain() throws Exception { + // test for issue https://github.com/opensearch-project/sql/issues/4903 enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_field_format.yaml"); assertYamlEqualsIgnoreId( - loadExpectedPlan("agg_filter_nested.yaml"), - explainQueryYaml( - StringUtils.format( - "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", - TEST_INDEX_CASCADED_NESTED))); + expected, + explainQueryYaml("source=opensearch-sql_test_index_account | head 5| fieldformat ")); } @Test diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index 6135d74b2f..a280f88480 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -14,6 +14,7 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Locale; +import org.apache.commons.text.StringEscapeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONException; @@ -142,7 +143,11 @@ protected void failWithMessage(String query, String message) { protected Request buildRequest(String query, String endpoint) { Request request = new Request("POST", endpoint); - request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); + request.setJsonEntity( + String.format( + Locale.ROOT, + "{\n" + " \"query\": \"%s\"\n" + "}", + StringEscapeUtils.escapeJson(query))); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); restOptionsBuilder.addHeader("Content-Type", "application/json"); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 30250137f7..7948f5d754 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -26,6 +26,7 @@ STREAMSTATS: 'STREAMSTATS'; DEDUP: 'DEDUP'; SORT: 'SORT'; EVAL: 'EVAL'; +FIELDFORMAT: 'FIELDFORMAT'; HEAD: 'HEAD'; BIN: 'BIN'; TOP: 'TOP'; @@ -46,7 +47,6 @@ ML: 'ML'; FILLNULL: 'FILLNULL'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; -TRANSPOSE: 'TRANSPOSE'; CHART: 'CHART'; TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index b4551ab68e..df87d789c8 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -107,6 +107,7 @@ commandName | DEDUP | SORT | EVAL + | FIELDFORMAT | HEAD | BIN | TOP @@ -361,6 +362,10 @@ evalCommand : EVAL evalClause (COMMA evalClause)* ; +fieldformatCommand + : FIELDFORMAT fieldFormatEvalClause (COMMA fieldFormatEvalClause)* + ; + headCommand : HEAD (number = integerLiteral)? (FROM from = integerLiteral)? ; @@ -734,6 +739,10 @@ evalClause : fieldExpression EQUAL logicalExpression ; +fieldFormatEvalClause + : fieldExpression EQUAL ffLogicalExpression + ; + eventstatsAggTerm : windowFunction (AS alias = wcFieldExpression)? ; @@ -827,6 +836,13 @@ numericLiteral | floatLiteral ; +ffLogicalExpression + : stringLiteral DOT logicalExpression # stringDotlogicalExpression + | stringLiteral DOT logicalExpression DOT stringLiteral # stringDotlogicalExpressionDotString + | logicalExpression DOT stringLiteral # logicalExpressionDotString + | logicalExpression # ffStandardLogicalExpression + ; + // predicates logicalExpression : NOT logicalExpression # logicalNot @@ -1671,6 +1687,5 @@ searchableKeyWord | FIELDNAME | ROW | COL - | COLUMN_NAME ; diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index d7c725f957..d4426590e0 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -15,6 +15,7 @@ import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DescribeCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.DynamicSourceClauseContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.EvalCommandContext; +import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldformatCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.FieldsCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.HeadCommandContext; import static org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.RenameCommandContext; @@ -822,6 +823,18 @@ public UnresolvedPlan visitEvalCommand(EvalCommandContext ctx) { .collect(Collectors.toList())); } + @Override + public UnresolvedPlan visitFieldformatCommand(FieldformatCommandContext ctx) { + // Use the new fieldFormatEvalClause instead of evalClause + org.opensearch.sql.ast.tree.Eval eval = + new org.opensearch.sql.ast.tree.Eval( + ctx.fieldFormatEvalClause().stream() + .map(ct -> (Let) internalVisitExpression(ct)) + .collect(Collectors.toList())); + + return eval; + } + private List getGroupByList(ByClauseContext ctx) { return ctx.fieldList().fieldExpression().stream() .map(this::internalVisitExpression) diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java index 283297c4ea..471c0c2f1c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstExpressionBuilder.java @@ -105,6 +105,54 @@ public UnresolvedExpression visitEvalClause(EvalClauseContext ctx) { return new Let((Field) visit(ctx.fieldExpression()), visit(ctx.logicalExpression())); } + /** Field format eval clause - similar to evalClause but for fieldformat command. */ + @Override + public UnresolvedExpression visitFieldFormatEvalClause( + OpenSearchPPLParser.FieldFormatEvalClauseContext ctx) { + OpenSearchPPLParser.FfLogicalExpressionContext ffLogicalExpressionCtx = + ctx.ffLogicalExpression(); + OpenSearchPPLParser.LogicalExpressionContext logicalExpression = null; + Literal prefix = null; + Literal suffix = null; + switch (ffLogicalExpressionCtx) { + case OpenSearchPPLParser.FfStandardLogicalExpressionContext + ffStandardLogicalExpressionContext -> { + // Standard logical expression + logicalExpression = ffStandardLogicalExpressionContext.logicalExpression(); + return new Let((Field) visit(ctx.fieldExpression()), visit(logicalExpression)); + } + case OpenSearchPPLParser.StringDotlogicalExpressionContext + stringDotlogicalExpressionContext -> { + // String dot logical expression + logicalExpression = stringDotlogicalExpressionContext.logicalExpression(); + prefix = (Literal) visit(stringDotlogicalExpressionContext.stringLiteral()); + return new Let( + (Field) visit(ctx.fieldExpression()), visit(logicalExpression), prefix, suffix); + } + case OpenSearchPPLParser.LogicalExpressionDotStringContext + logicalExpressionDotStringContext -> { + // Logical expression dot string + logicalExpression = logicalExpressionDotStringContext.logicalExpression(); + suffix = (Literal) visit(logicalExpressionDotStringContext.stringLiteral()); + return new Let( + (Field) visit(ctx.fieldExpression()), visit(logicalExpression), prefix, suffix); + } + case OpenSearchPPLParser.StringDotlogicalExpressionDotStringContext + stringDotlogicalExpressionDotStringContext -> { + // Logical expression dot string + logicalExpression = stringDotlogicalExpressionDotStringContext.logicalExpression(); + prefix = (Literal) visit(stringDotlogicalExpressionDotStringContext.stringLiteral(0)); + + suffix = (Literal) visit(stringDotlogicalExpressionDotStringContext.stringLiteral(1)); + return new Let( + (Field) visit(ctx.fieldExpression()), visit(logicalExpression), prefix, suffix); + } + case null, default -> + throw new IllegalArgumentException( + "Unknown ffLogicalExpression context type: " + ctx.getClass()); + } + } + /** Trendline clause. */ @Override public Trendline.TrendlineComputation visitTrendlineClause( From 553ffa7716d39665c18c305758ada0e1549c2b07 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 23 Jan 2026 20:56:18 -0800 Subject: [PATCH 03/36] added field format with concatenation of string , added more examples Signed-off-by: Asif Bashar --- .../sql/calcite/remote/CalciteExplainIT.java | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 09936d4846..ab8160f392 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2477,15 +2477,15 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { @Test public void testAggFilterOnNestedFields() throws IOException { - enabledOnlyWhenPushdownIsEnabled(); - assertYamlEqualsIgnoreId( - loadExpectedPlan("agg_filter_nested.yaml"), - explainQueryYaml( - StringUtils.format( - "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", - TEST_INDEX_CASCADED_NESTED))); - + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_filter_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED))); } + @Test public void testFieldFormatExplain() throws Exception { // test for issue https://github.com/opensearch-project/sql/issues/4903 From 36085f53f23e8ae0211de7c72a19a78b8a1e4aaa Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 11:16:32 -0800 Subject: [PATCH 04/36] ci failure fix test Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 50cdd8d847..6dec4717e0 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -34,7 +34,7 @@ CalciteDedupCommandIT.class, CalciteDescribeCommandIT.class, CalciteExpandCommandIT.class, - CalciteFieldFormatCommandIT.class, + CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, From c08beda62bd36047c8b769e5c563a272344fddaf Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 11:34:10 -0800 Subject: [PATCH 05/36] ci failure fix test Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 6dec4717e0..0010fe13ff 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -34,7 +34,6 @@ CalciteDedupCommandIT.class, CalciteDescribeCommandIT.class, CalciteExpandCommandIT.class, - CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, From faaf0f91eb93d730f59e1dcc59bebeefb5bf39d9 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 13:55:19 -0800 Subject: [PATCH 06/36] ci failure fix test Signed-off-by: Asif Bashar --- .../opensearch/sql/ast/analysis/FieldResolutionVisitor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java b/core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java index a6f6671084..0b2e05907b 100644 --- a/core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/analysis/FieldResolutionVisitor.java @@ -626,6 +626,12 @@ public Node visitAddColTotals(AddColTotals node, FieldResolutionContext context) return node; } + @Override + public Node visitFieldFormat(Eval node, FieldResolutionContext context) { + visitChildren(node, context); + return node; + } + @Override public Node visitExpand(Expand node, FieldResolutionContext context) { Set expandFields = extractFieldsFromExpression(node.getField()); From b6e896523a9ce2b873727652c948b344efd99632 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 21:31:52 -0800 Subject: [PATCH 07/36] ci failure fix test Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 0010fe13ff..a676e25533 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -108,6 +108,8 @@ CalciteVisualizationFormatIT.class, CalciteWhereCommandIT.class, CalcitePPLTpchIT.class, + CalciteFieldFormatCommandIT.class + CalcitePPLTpchIT.class, CalciteMvCombineCommandIT.class }) public class CalciteNoPushdownIT { From 330cfc02e7118a6bd098cbe155cbdb8c9c88de35 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 10:19:36 -0800 Subject: [PATCH 08/36] added missing test files Signed-off-by: Asif Bashar --- .../remote/CalciteFieldFormatCommandIT.java | 114 ++++++ .../calcite/CalcitePPLFieldFormatTest.java | 358 ++++++++++++++++++ 2 files changed, 472 insertions(+) create mode 100644 integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java new file mode 100644 index 0000000000..5a89f095e6 --- /dev/null +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java @@ -0,0 +1,114 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.calcite.remote; + +import static org.opensearch.sql.util.MatcherUtils.*; + +import java.io.IOException; +import org.json.JSONObject; +import org.junit.jupiter.api.Test; +import org.opensearch.client.Request; +import org.opensearch.sql.ppl.PPLIntegTestCase; + +public class CalciteFieldFormatCommandIT extends PPLIntegTestCase { + + @Override + public void init() throws Exception { + super.init(); + enableCalcite(); + + loadIndex(Index.BANK); + + // Create test data for string concatenation + Request request1 = new Request("PUT", "/test_eval/_doc/1?refresh=true"); + request1.setJsonEntity("{\"name\": \"Alice\", \"age\": 25, \"title\": \"Engineer\"}"); + client().performRequest(request1); + + Request request2 = new Request("PUT", "/test_eval/_doc/2?refresh=true"); + request2.setJsonEntity("{\"name\": \"Bob\", \"age\": 30, \"title\": \"Manager\"}"); + client().performRequest(request2); + + Request request3 = new Request("PUT", "/test_eval/_doc/3?refresh=true"); + request3.setJsonEntity("{\"name\": \"Charlie\", \"age\": null, \"title\": \"Analyst\"}"); + client().performRequest(request3); + } + + @Test + public void testFieldFormatStringConcatenation() throws IOException { + JSONObject result = executeQuery("source=test_eval | fieldformat greeting = 'Hello ' + name"); + verifySchema( + result, + schema("name", "string"), + schema("title", "string"), + schema("age", "bigint"), + schema("greeting", "string")); + verifyDataRows( + result, + rows("Alice", "Engineer", 25, "Hello Alice"), + rows("Bob", "Manager", 30, "Hello Bob"), + rows("Charlie", "Analyst", null, "Hello Charlie")); + } + + @Test + public void testFieldFormatStringConcatenationWithNullFieldToString() throws IOException { + JSONObject result = + executeQuery( + "source=test_eval | fieldformat age_desc = \"Age: \".tostring(age,\"commas\") | fields" + + " name, age, age_desc"); + verifySchema( + result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); + verifyDataRows( + result, + rows("Alice", 25, "Age: 25"), + rows("Bob", 30, "Age: 30"), + rows("Charlie", null, null)); + } + + @Test + public void testFieldFormatStringConcatenationWithNullField() throws IOException { + JSONObject result = + executeQuery( + "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING) | fields name," + + " age, age_desc"); + verifySchema( + result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); + verifyDataRows( + result, + rows("Alice", 25, "Age: 25"), + rows("Bob", 30, "Age: 30"), + rows("Charlie", null, null)); + } + + @Test + public void testFieldFormatStringConcatWithSuffix() throws IOException { + JSONObject result = + executeQuery( + "source=test_eval | fieldformat age_desc = CAST(age AS STRING).\" years\" | fields" + + " name, age, age_desc"); + verifySchema( + result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); + verifyDataRows( + result, + rows("Alice", 25, "25 years"), + rows("Bob", 30, "30 years"), + rows("Charlie", null, null)); + } + + @Test + public void testFieldFormatStringConcatWithPrefixSuffix() throws IOException { + JSONObject result = + executeQuery( + "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING).\" years\" |" + + " fields name, age, age_desc"); + verifySchema( + result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); + verifyDataRows( + result, + rows("Alice", 25, "Age: 25 years"), + rows("Bob", 30, "Age: 30 years"), + rows("Charlie", null, null)); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java new file mode 100644 index 0000000000..64d40eeba6 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -0,0 +1,358 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertThrows; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Test; + +public class CalcitePPLFieldFormatTest extends CalcitePPLAbstractTest { + + public CalcitePPLFieldFormatTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testFieldFormat1() { + String ppl = "source=EMP | sort EMPNO| head 3 | fieldformat a = 1"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[1])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; a=1\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; a=1\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; a=1\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, 1 `a`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormat2() { + String ppl = + "source=EMP | sort EMPNO| head 3 |fieldformat formatted_salary =" + + " \"$\".tostring(SAL,\"commas\")"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], formatted_salary=[||('$':VARCHAR, TOSTRING($5," + + " 'commas':VARCHAR))])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; formatted_salary=$800\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; formatted_salary=$1,600\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; formatted_salary=$1,250\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, '$' ||" + + " TOSTRING(`SAL`, 'commas') `formatted_salary`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormatAndFields() { + String ppl = + "source=EMP | sort EMPNO| head 5|fieldformat formatted_salary =" + + " \"$\".tostring(SAL,\"commas\") |fields EMPNO, JOB, formatted_salary"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], JOB=[$2], formatted_salary=[||('$':VARCHAR, TOSTRING($5," + + " 'commas':VARCHAR))])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[5])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; JOB=CLERK; formatted_salary=$800\n" + + "EMPNO=7499; JOB=SALESMAN; formatted_salary=$1,600\n" + + "EMPNO=7521; JOB=SALESMAN; formatted_salary=$1,250\n" + + "EMPNO=7566; JOB=MANAGER; formatted_salary=$2,975\n" + + "EMPNO=7654; JOB=SALESMAN; formatted_salary=$1,250\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `JOB`, '$' || TOSTRING(`SAL`, 'commas') `formatted_salary`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 5"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormat2Fields() { + String ppl = "source=EMP | sort EMPNO| head 3 | fieldformat a = 1, b = 2"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[1], b=[2])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; a=1; b=2\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; a=1; b=2\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; a=1; b=2\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, 1 `a`, 2 `b`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormat3() { + String ppl = + "source=EMP | sort EMPNO| head 3 | fieldformat a = 1 | fieldformat b = 2 | fieldformat c =" + + " 3"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[1], b=[2], c=[3])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; a=1; b=2; c=3\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; a=1; b=2; c=3\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; a=1; b=2; c=3\n"; + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, 1 `a`, 2 `b`," + + " 3 `c`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormatSum() { + String ppl = + "source=EMP |sort EMPNO | head 3| fieldformat total = sum(1, 2, 3) | fields EMPNO, total"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], total=[+(1, +(2, 3))])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = "EMPNO=7369; total=6\nEMPNO=7499; total=6\nEMPNO=7521; total=6\n"; + verifyResult(root, expectedResult); + + String expectedSparkSql = + "SELECT `EMPNO`, 1 + (2 + 3) `total`\nFROM `scott`.`EMP`\nORDER BY `EMPNO`\nLIMIT 3"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormatWithToNumber() { + String ppl = + "source=EMP | sort EMPNO | head 3| fieldformat total = sum(SAL, COMM, 100) | fieldformat" + + " total = \"$\".cast(total as string) | fields EMPNO, total"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], total=[||('$':VARCHAR, NUMBER_TO_STRING(+($5, +($6, 100))))])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; total=null\nEMPNO=7499; total=$2000.00\nEMPNO=7521; total=$1850.00\n"; + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, '$' || NUMBER_TO_STRING(`SAL` + (`COMM` + 100)) `total`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testComplexFieldFormatCommands4() { + String ppl = + "source=EMP | fieldformat col1 = SAL | sort - col1 | head 3 | fields ENAME, col1 |" + + " fieldformat col2 = col1 | sort + col2 | fields ENAME, col2 | fieldformat col3 =" + + " col2 | head 2 | fields HIREDATE, col3"; + IllegalArgumentException e = + assertThrows( + IllegalArgumentException.class, + () -> { + RelNode root = getRelNode(ppl); + }); + assertThat(e.getMessage(), is("Field [HIREDATE] not found.")); + } + + @Test + public void testFieldFormatMaxOnStrings() { + String ppl = + "source=EMP | sort EMPNO | head 3 |fieldformat a = \"Max String:\".max('banana', 'Door'," + + " ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[||('Max String:':VARCHAR, SCALAR_MAX('banana':VARCHAR," + + " 'Door':VARCHAR, $1))])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; JOB=CLERK; MGR=7902; HIREDATE=1980-12-17; SAL=800.00; COMM=null;" + + " DEPTNO=20; a=Max String:banana\n" + + "EMPNO=7499; ENAME=ALLEN; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-20; SAL=1600.00;" + + " COMM=300.00; DEPTNO=30; a=Max String:banana\n" + + "EMPNO=7521; ENAME=WARD; JOB=SALESMAN; MGR=7698; HIREDATE=1981-02-22; SAL=1250.00;" + + " COMM=500.00; DEPTNO=30; a=Max String:banana\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, 'Max String:'" + + " || SCALAR_MAX('banana', 'Door', `ENAME`) `a`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormatMaxOnStringsWithSuffix() { + String ppl = + "source=EMP | sort EMPNO | head 3 |fields EMPNO, ENAME| fieldformat a = max('banana'," + + " 'Door', ENAME).\" after comparing with provided constant strings and ENAME column" + + " values.\""; + // # + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], a=[||(SCALAR_MAX('banana':VARCHAR, 'Door':VARCHAR," + + " $1), ' after comparing with provided constant strings and ENAME column" + + " values.':VARCHAR)])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; a=banana after comparing with provided constant strings and ENAME" + + " column values.\n" + + "EMPNO=7499; ENAME=ALLEN; a=banana after comparing with provided constant strings and" + + " ENAME column values.\n" + + "EMPNO=7521; ENAME=WARD; a=banana after comparing with provided constant strings and" + + " ENAME column values.\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, SCALAR_MAX('banana', 'Door', `ENAME`) || ' after comparing with" + + " provided constant strings and ENAME column values.' `a`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testFieldFormatMaxOnStringsWithPrefixSuffix() { + String ppl = + "source=EMP | sort EMPNO | head 3 |fields EMPNO, ENAME| fieldformat a = \"Max String:" + + " \\\"\".max('banana', 'Door', ENAME).\"\\\" after comparing with provided constant" + + " strings and ENAME column values.\""; + // # + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], a=[||(||('Max String: \"':VARCHAR," + + " SCALAR_MAX('banana':VARCHAR, 'Door':VARCHAR, $1)), '\" after comparing with" + + " provided constant strings and ENAME column values.':VARCHAR)])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; a=Max String: \"banana\" after comparing with provided constant" + + " strings and ENAME column values.\n" + + "EMPNO=7499; ENAME=ALLEN; a=Max String: \"banana\" after comparing with provided" + + " constant strings and ENAME column values.\n" + + "EMPNO=7521; ENAME=WARD; a=Max String: \"banana\" after comparing with provided" + + " constant strings and ENAME column values.\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, 'Max String: \"' || SCALAR_MAX('banana', 'Door', `ENAME`) || '\"" + + " after comparing with provided constant strings and ENAME column values.' `a`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + + verifyPPLToSparkSQL(root, expectedSparkSql); + } + + @Test + public void testEvalMinOnNumericAndString() { + String ppl = + "source=EMP | sort EMPNO | head 3| fields EMPNO, ENAME | fieldformat a = min(5, 30," + + " DEPTNO, 'banana', 'Door', ENAME)"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[SCALAR_MIN(5, 30, $7, 'banana':VARCHAR, 'Door':VARCHAR," + + " $1)])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," + + " COMM=[$6], DEPTNO=[$7], a=[SCALAR_MIN(5, 30, $7, 'banana':VARCHAR, 'Door':VARCHAR," + + " $1)])\n" + + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n"; + + verifyResult(root, expectedResult); + String expectedSparkSql = + "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, SCALAR_MIN(5," + + " 30, `DEPTNO`, 'banana', 'Door', `ENAME`) `a`\n" + + "FROM `scott`.`EMP`"; + verifyPPLToSparkSQL(root, expectedSparkSql); + } +} From d853f3687a5ec37554365ff07e7c2025337c22ba Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 11:11:27 -0800 Subject: [PATCH 09/36] added missing test files Signed-off-by: Asif Bashar --- .../calcite/CalcitePPLFieldFormatTest.java | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java index 64d40eeba6..98b2917290 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -332,27 +332,35 @@ public void testFieldFormatMaxOnStringsWithPrefixSuffix() { @Test public void testEvalMinOnNumericAndString() { String ppl = - "source=EMP | sort EMPNO | head 3| fields EMPNO, ENAME | fieldformat a = min(5, 30," + "source=EMP | sort EMPNO | head 3| fields EMPNO, ENAME, DEPTNO | fieldformat a = \"Minimum" + + " of DEPTNO, ENAME and Provided list of 5, 30, 'banana', 'Door': \".min(5, 30," + " DEPTNO, 'banana', 'Door', ENAME)"; RelNode root = getRelNode(ppl); String expectedLogical = - "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7], a=[SCALAR_MIN(5, 30, $7, 'banana':VARCHAR, 'Door':VARCHAR," - + " $1)])\n" - + " LogicalTableScan(table=[[scott, EMP]])\n"; - verifyLogical(root, expectedLogical); - String expectedResult = - "LogicalProject(EMPNO=[$0], ENAME=[$1], JOB=[$2], MGR=[$3], HIREDATE=[$4], SAL=[$5]," - + " COMM=[$6], DEPTNO=[$7], a=[SCALAR_MIN(5, 30, $7, 'banana':VARCHAR, 'Door':VARCHAR," - + " $1)])\n" + "LogicalProject(EMPNO=[$0], ENAME=[$1], DEPTNO=[$7], a=[||('Minimum of DEPTNO, ENAME and" + + " Provided list of 5, 30, ''banana'', ''Door'': ':VARCHAR, SCALAR_MIN(5, 30, $7," + + " 'banana':VARCHAR, 'Door':VARCHAR, $1))])\n" + " LogicalSort(sort0=[$0], dir0=[ASC-nulls-first], fetch=[3])\n" + " LogicalTableScan(table=[[scott, EMP]])\n"; + verifyLogical(root, expectedLogical); + String expectedResult = + "EMPNO=7369; ENAME=SMITH; DEPTNO=20; a=Minimum of DEPTNO, ENAME and Provided list of 5," + + " 30, 'banana', 'Door': 5\n" + + "EMPNO=7499; ENAME=ALLEN; DEPTNO=30; a=Minimum of DEPTNO, ENAME and Provided list of " + + " 5, 30, 'banana', 'Door': 5\n" + + "EMPNO=7521; ENAME=WARD; DEPTNO=30; a=Minimum of DEPTNO, ENAME and Provided list of " + + " 5, 30, 'banana', 'Door': 5\n"; + verifyResult(root, expectedResult); String expectedSparkSql = - "SELECT `EMPNO`, `ENAME`, `JOB`, `MGR`, `HIREDATE`, `SAL`, `COMM`, `DEPTNO`, SCALAR_MIN(5," - + " 30, `DEPTNO`, 'banana', 'Door', `ENAME`) `a`\n" - + "FROM `scott`.`EMP`"; + "SELECT `EMPNO`, `ENAME`, `DEPTNO`, 'Minimum of DEPTNO, ENAME and Provided list of 5, 30," + + " ''banana'', ''Door'': ' || SCALAR_MIN(5, 30, `DEPTNO`, 'banana', 'Door', `ENAME`)" + + " `a`\n" + + "FROM `scott`.`EMP`\n" + + "ORDER BY `EMPNO`\n" + + "LIMIT 3"; + verifyPPLToSparkSQL(root, expectedSparkSql); } } From c74151d7c68df4909a310e1d954948c745307446 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 13:34:07 -0800 Subject: [PATCH 10/36] doc fix Signed-off-by: Asif Bashar --- docs/user/ppl/cmd/fieldformat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index f11f7a4489..5271fdc254 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -1,7 +1,7 @@ # fieldformat -The `fieldformat` command sets the value to a field with the specified expression and appends the field with evaluated result to the search results. The command is a alias of eval command. +The `fieldformat` command sets the value to a field with the specified expression and appends the field with evaluated result to the search results. The command is an alias of eval command. Additionally, it also provides string concatenation dot operator followed by and/or follows a string that will be concatenated to the expression. From 6c93ebec28386053af5f1831a58585735ade6195 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 13:38:47 -0800 Subject: [PATCH 11/36] doc fix Signed-off-by: Asif Bashar --- docs/user/ppl/cmd/fieldformat.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index 5271fdc254..428baee5ef 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -19,12 +19,12 @@ Following are possible syntaxes: The `fieldformat` command supports the following parameters. -| Parameter | Required/Optional | Description | -|----------------| |-----------------------------------------------------------------------------------------------------------------------------------------------| -| `` | Required | The name of the field to create or update. If the field does not exist, a new field is added. If it already exists, its value is overwritten. | -| `` | Required | The expression to evaluate. The expression can have a prefix and/or suffix string part that will be concatenated to the expression. | -| `prefix` | Optional | A string before the expression followed by dot operator which will be concatenated as prefix to the evaluated expression value. | -| `suffix` | Optional | A string that follows the expression and dot operator which will be concatenated as suffix to the evaluated expression value. | +| Parameter| Required/Optional | Description | +|----------------|-------------------|-----------------------------------------------------------------------------------------------------------------------------------------------| +| `` | Required | The name of the field to create or update. If the field does not exist, a new field is added. If it already exists, its value is overwritten. | +| `` | Required | The expression to evaluate. The expression can have a prefix and/or suffix string part that will be concatenated to the expression. | +| `prefix` | Optional | A string before the expression followed by dot operator which will be concatenated as prefix to the evaluated expression value. | +| `suffix` | Optional | A string that follows the expression and dot operator which will be concatenated as suffix to the evaluated expression value. | ## Example 1: Create a new field @@ -104,7 +104,7 @@ fetched rows / total rows = 4/4 ``` -## Example 5: string concatenation with dot operator , prefix and suffix +## Example 4: String concatenation with dot operator , prefix and suffix The following query performs prefix and suffix string concatenation operations using dot operator: From 239df5aacf7d40d802ce0bf648784b8066692a2c Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 13:40:07 -0800 Subject: [PATCH 12/36] doc fix Signed-off-by: Asif Bashar --- docs/user/ppl/cmd/fieldformat.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index 428baee5ef..fce367ada4 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -104,7 +104,7 @@ fetched rows / total rows = 4/4 ``` -## Example 4: String concatenation with dot operator , prefix and suffix +## Example 4: String concatenation with dot operator, prefix and suffix The following query performs prefix and suffix string concatenation operations using dot operator: From be0d82bc3a0198d1157492d9269333112cde34af Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 14:10:15 -0800 Subject: [PATCH 13/36] added test Signed-off-by: Asif Bashar --- .../org/opensearch/sql/ppl/NewAddedCommandsIT.java | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index efcda1105a..428e7cdc15 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -202,6 +202,19 @@ public void testAddColTotalCommand() throws IOException { } } + @Test + public void testFieldFormatCommand() throws IOException { + JSONObject result; + try { + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); + } + } + @Test public void testTransposeCommand() throws IOException { JSONObject result; From 090bea77391871559ab1a44764bff2fda908372b Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 18:55:10 -0800 Subject: [PATCH 14/36] [Feature] implement transpose command as in the roadmap #4786 (#5011) * transpose command implementation Signed-off-by: Asif Bashar * transpose rows to columns Signed-off-by: Asif Bashar * added argument type missing map and hashmap Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added more validations Signed-off-by: Asif Bashar * added validation Signed-off-by: Asif Bashar * index.md formatting fix Signed-off-by: Asif Bashar * doc format Signed-off-by: Asif Bashar * coderabbit review fixes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * trim columnName Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * added field resolution Signed-off-by: Asif Bashar * fix by removing metadata field Signed-off-by: Asif Bashar * fixed explain test after removing of metadata fields in transpose result Signed-off-by: Asif Bashar --------- Signed-off-by: Asif Bashar --- .../sql/ppl/NewAddedCommandsIT.java | 27 ++++++++++--------- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 +- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 ++ 3 files changed, 17 insertions(+), 14 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 428e7cdc15..619333e9f1 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -202,19 +202,6 @@ public void testAddColTotalCommand() throws IOException { } } - @Test - public void testFieldFormatCommand() throws IOException { - JSONObject result; - try { - executeQuery( - String.format( - "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - verifyQuery(result); - } - } - @Test public void testTransposeCommand() throws IOException { JSONObject result; @@ -226,6 +213,20 @@ public void testTransposeCommand() throws IOException { } } + @Test + public void testFieldFormatCommand() throws IOException { + JSONObject result; + try { + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); + } + } + + private void verifyQuery(JSONObject result) throws IOException { if (isCalciteEnabled()) { assertFalse(result.getJSONArray("datarows").isEmpty()); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 7948f5d754..cd53d73f3b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -47,6 +47,7 @@ ML: 'ML'; FILLNULL: 'FILLNULL'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; +TRANSPOSE: 'TRANSPOSE'; CHART: 'CHART'; TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; @@ -162,7 +163,6 @@ INPUT: 'INPUT'; OUTPUT: 'OUTPUT'; PATH: 'PATH'; - // COMPARISON FUNCTION KEYWORDS CASE: 'CASE'; ELSE: 'ELSE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index df87d789c8..476dab458f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -860,6 +860,7 @@ expression | expression NOT? BETWEEN expression AND expression # between ; + valueExpression : left = valueExpression binaryOperator = (STAR | DIVIDE | MODULE) right = valueExpression # binaryArithmetic | left = valueExpression binaryOperator = (PLUS | MINUS) right = valueExpression # binaryArithmetic @@ -1687,5 +1688,6 @@ searchableKeyWord | FIELDNAME | ROW | COL + | COLUMN_NAME ; From fcd0f8734570c0c837a813cd666e2a2e8b3011a2 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 14:36:43 -0800 Subject: [PATCH 15/36] added test Signed-off-by: Asif Bashar --- .../sql/ppl/NewAddedCommandsIT.java | 25 +++++++++---------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 619333e9f1..9ea3805d08 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -213,21 +213,20 @@ public void testTransposeCommand() throws IOException { } } - @Test - public void testFieldFormatCommand() throws IOException { - JSONObject result; - try { - executeQuery( - String.format( - "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - verifyQuery(result); - } + @Test + public void testFieldFormatCommand() throws IOException { + JSONObject result; + try { + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); } + } - - private void verifyQuery(JSONObject result) throws IOException { + private void verifyQuery(JSONObject result) throws IOException { if (isCalciteEnabled()) { assertFalse(result.getJSONArray("datarows").isEmpty()); } else { From 1ff53f67ae1b563c354fcefdf0e5ab25f218b331 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 18:55:10 -0800 Subject: [PATCH 16/36] [Feature] implement transpose command as in the roadmap #4786 (#5011) * transpose command implementation Signed-off-by: Asif Bashar * transpose rows to columns Signed-off-by: Asif Bashar * added argument type missing map and hashmap Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added more validations Signed-off-by: Asif Bashar * added validation Signed-off-by: Asif Bashar * index.md formatting fix Signed-off-by: Asif Bashar * doc format Signed-off-by: Asif Bashar * coderabbit review fixes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * trim columnName Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * added field resolution Signed-off-by: Asif Bashar * fix by removing metadata field Signed-off-by: Asif Bashar * fixed explain test after removing of metadata fields in transpose result Signed-off-by: Asif Bashar --------- Signed-off-by: Asif Bashar # Conflicts: # integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java # ppl/src/main/antlr/OpenSearchPPLLexer.g4 --- .../src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java | 1 + 1 file changed, 1 insertion(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 9ea3805d08..017a304454 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -213,6 +213,7 @@ public void testTransposeCommand() throws IOException { } } + @Test public void testFieldFormatCommand() throws IOException { JSONObject result; From 4fc411307ef588202b62d166f456ef83a3528503 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 14 Jan 2026 14:17:03 -0800 Subject: [PATCH 17/36] fielfformat changes Signed-off-by: Asif Bashar # Conflicts: # ppl/src/main/antlr/OpenSearchPPLLexer.g4 # ppl/src/main/antlr/OpenSearchPPLParser.g4 --- .../org/opensearch/sql/analysis/Analyzer.java | 2 + .../sql/ast/AbstractNodeVisitor.java | 3 +- .../opensearch/sql/ast/tree/FieldFormat.java | 43 +++++++++++++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 33 ++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 1 + 6 files changed, 82 insertions(+), 1 deletion(-) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 5015214796..f2dd4c2a4e 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -70,6 +70,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -526,6 +527,7 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { return new LogicalEval(child, expressionsBuilder.build()); } + /** Build {@link LogicalEval}. */ @Override public LogicalPlan visitFieldFormat(Eval node, AnalysisContext context) { diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index 2486b63791..fa1a63d850 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -58,6 +58,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -264,7 +265,7 @@ public T visitEval(Eval node, C context) { return visitChildren(node, context); } - public T visitFieldFormat(Eval node, C context) { + public T visitFieldFormat(FieldFormat node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java new file mode 100644 index 0000000000..cab0346835 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java @@ -0,0 +1,43 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.expression.Let; + +/** AST node represent Eval operation. */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class FieldFormat extends UnresolvedPlan { + private final List expressionList; + private UnresolvedPlan child; + + @Override + public FieldFormat attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor nodeVisitor, C context) { + return nodeVisitor.visitFieldFormat(this, context); + } +} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 5825011f65..d8b1b40f40 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -115,6 +115,7 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; +import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -999,6 +1000,38 @@ public RelNode visitEval(Eval node, CalcitePlanContext context) { return context.relBuilder.peek(); } + @Override + public RelNode visitFieldFormat(FieldFormat node, CalcitePlanContext context) { + visitChildren(node, context); + node.getExpressionList() + .forEach( + expr -> { + boolean containsSubqueryExpression = containsSubqueryExpression(expr); + final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); + if (containsSubqueryExpression) { + context.relBuilder.variable(v::set); + context.pushCorrelVar(v.get()); + } + RexNode eval = rexVisitor.analyze(expr, context); + if (containsSubqueryExpression) { + // RelBuilder.projectPlus doesn't have a parameter with variablesSet: + // projectPlus(Iterable variablesSet, RexNode... nodes) + context.relBuilder.project( + Iterables.concat(context.relBuilder.fields(), ImmutableList.of(eval)), + ImmutableList.of(), + false, + ImmutableList.of(v.get().id)); + context.popCorrelVar(); + } else { + // Overriding the existing field if the alias has the same name with original field. + String alias = + ((RexLiteral) ((RexCall) eval).getOperands().get(1)).getValueAs(String.class); + projectPlusOverriding(List.of(eval), List.of(alias), context); + } + }); + return context.relBuilder.peek(); + } + private void projectPlusOverriding( List newFields, List newNames, CalcitePlanContext context) { List originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index cd53d73f3b..00a2764c0b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -74,6 +74,7 @@ AGGREGATION: 'AGGREGATION'; APPENDPIPE: 'APPENDPIPE'; FIELDFORMAT: 'FIELDFORMAT'; COLUMN_NAME: 'COLUMN_NAME'; +FIELDFORMAT: 'FIELDFORMAT'; MVCOMBINE: 'MVCOMBINE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 476dab458f..689a02753c 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -136,6 +136,7 @@ commandName | REPLACE | FIELDFORMAT | TRANSPOSE + | FIELDFORMAT ; searchCommand From 43ec5c95c41e38f185706ee87a0fea747ad9f6e2 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Fri, 23 Jan 2026 17:57:45 -0800 Subject: [PATCH 18/36] added field format with concatenation of string , added more examples Signed-off-by: Asif Bashar --- .../org/opensearch/sql/analysis/Analyzer.java | 1 - .../sql/ast/AbstractNodeVisitor.java | 3 +- .../opensearch/sql/ast/tree/FieldFormat.java | 43 ------------------- .../sql/calcite/CalciteRelNodeVisitor.java | 33 -------------- .../sql/calcite/CalciteNoPushdownIT.java | 1 + .../sql/calcite/remote/CalciteExplainIT.java | 20 ++++++--- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 - 7 files changed, 17 insertions(+), 85 deletions(-) delete mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index f2dd4c2a4e..c532fd8205 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -70,7 +70,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index fa1a63d850..2486b63791 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -58,7 +58,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -265,7 +264,7 @@ public T visitEval(Eval node, C context) { return visitChildren(node, context); } - public T visitFieldFormat(FieldFormat node, C context) { + public T visitFieldFormat(Eval node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java b/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java deleted file mode 100644 index cab0346835..0000000000 --- a/core/src/main/java/org/opensearch/sql/ast/tree/FieldFormat.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright OpenSearch Contributors - * SPDX-License-Identifier: Apache-2.0 - */ - -package org.opensearch.sql.ast.tree; - -import com.google.common.collect.ImmutableList; -import java.util.List; -import lombok.EqualsAndHashCode; -import lombok.Getter; -import lombok.RequiredArgsConstructor; -import lombok.Setter; -import lombok.ToString; -import org.opensearch.sql.ast.AbstractNodeVisitor; -import org.opensearch.sql.ast.expression.Let; - -/** AST node represent Eval operation. */ -@Getter -@Setter -@ToString -@EqualsAndHashCode(callSuper = false) -@RequiredArgsConstructor -public class FieldFormat extends UnresolvedPlan { - private final List expressionList; - private UnresolvedPlan child; - - @Override - public FieldFormat attach(UnresolvedPlan child) { - this.child = child; - return this; - } - - @Override - public List getChild() { - return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); - } - - @Override - public T accept(AbstractNodeVisitor nodeVisitor, C context) { - return nodeVisitor.visitFieldFormat(this, context); - } -} diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index d8b1b40f40..5825011f65 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -115,7 +115,6 @@ import org.opensearch.sql.ast.tree.Eval; import org.opensearch.sql.ast.tree.Expand; import org.opensearch.sql.ast.tree.FetchCursor; -import org.opensearch.sql.ast.tree.FieldFormat; import org.opensearch.sql.ast.tree.FillNull; import org.opensearch.sql.ast.tree.Filter; import org.opensearch.sql.ast.tree.Flatten; @@ -1000,38 +999,6 @@ public RelNode visitEval(Eval node, CalcitePlanContext context) { return context.relBuilder.peek(); } - @Override - public RelNode visitFieldFormat(FieldFormat node, CalcitePlanContext context) { - visitChildren(node, context); - node.getExpressionList() - .forEach( - expr -> { - boolean containsSubqueryExpression = containsSubqueryExpression(expr); - final Holder<@Nullable RexCorrelVariable> v = Holder.empty(); - if (containsSubqueryExpression) { - context.relBuilder.variable(v::set); - context.pushCorrelVar(v.get()); - } - RexNode eval = rexVisitor.analyze(expr, context); - if (containsSubqueryExpression) { - // RelBuilder.projectPlus doesn't have a parameter with variablesSet: - // projectPlus(Iterable variablesSet, RexNode... nodes) - context.relBuilder.project( - Iterables.concat(context.relBuilder.fields(), ImmutableList.of(eval)), - ImmutableList.of(), - false, - ImmutableList.of(v.get().id)); - context.popCorrelVar(); - } else { - // Overriding the existing field if the alias has the same name with original field. - String alias = - ((RexLiteral) ((RexCall) eval).getOperands().get(1)).getValueAs(String.class); - projectPlusOverriding(List.of(eval), List.of(alias), context); - } - }); - return context.relBuilder.peek(); - } - private void projectPlusOverriding( List newFields, List newNames, CalcitePlanContext context) { List originalFieldNames = context.relBuilder.peek().getRowType().getFieldNames(); diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index a676e25533..92acc16d19 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -34,6 +34,7 @@ CalciteDedupCommandIT.class, CalciteDescribeCommandIT.class, CalciteExpandCommandIT.class, + CalciteFieldFormatCommandIT.class, CalciteFieldsCommandIT.class, CalciteFillNullCommandIT.class, CalciteFlattenCommandIT.class, diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index ab8160f392..897faa7441 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2477,13 +2477,23 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { @Test public void testAggFilterOnNestedFields() throws IOException { + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_filter_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED))); + + } + @Test + public void testFieldFormatExplain() throws Exception { + // test for issue https://github.com/opensearch-project/sql/issues/4903 enabledOnlyWhenPushdownIsEnabled(); + String expected = loadExpectedPlan("explain_field_format.yaml"); assertYamlEqualsIgnoreId( - loadExpectedPlan("agg_filter_nested.yaml"), - explainQueryYaml( - StringUtils.format( - "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", - TEST_INDEX_CASCADED_NESTED))); + expected, + explainQueryYaml("source=opensearch-sql_test_index_account | head 5| fieldformat ")); } @Test diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 00a2764c0b..240d88834a 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -47,7 +47,6 @@ ML: 'ML'; FILLNULL: 'FILLNULL'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; -TRANSPOSE: 'TRANSPOSE'; CHART: 'CHART'; TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; From 90c88fc8ba32ccca0c22f0f4cb2fc8cf80f0dbc1 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 14:10:15 -0800 Subject: [PATCH 19/36] added test Signed-off-by: Asif Bashar --- .../opensearch/sql/ppl/NewAddedCommandsIT.java | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 017a304454..9d7edb2fce 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -201,12 +201,24 @@ public void testAddColTotalCommand() throws IOException { verifyQuery(result); } } + @Test + public void testTransposeCommand() throws IOException { + JSONObject result; + try { + executeQuery(String.format("search source=%s | transpose ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); + } + } @Test - public void testTransposeCommand() throws IOException { + public void testFieldFormatCommand() throws IOException { JSONObject result; try { - executeQuery(String.format("search source=%s | transpose ", TEST_INDEX_BANK)); + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); verifyQuery(result); From 8725e4d66c08426765d0e1867e84537c6b5659e3 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Mon, 26 Jan 2026 18:55:10 -0800 Subject: [PATCH 20/36] [Feature] implement transpose command as in the roadmap #4786 (#5011) * transpose command implementation Signed-off-by: Asif Bashar * transpose rows to columns Signed-off-by: Asif Bashar * added argument type missing map and hashmap Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added tests Signed-off-by: Asif Bashar * added more validations Signed-off-by: Asif Bashar * added validation Signed-off-by: Asif Bashar * index.md formatting fix Signed-off-by: Asif Bashar * doc format Signed-off-by: Asif Bashar * coderabbit review fixes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * added recommended changes Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * for cross cluster failure debugging Signed-off-by: Asif Bashar * trim columnName Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * per review moved to class varialble. Signed-off-by: Asif Bashar * added field resolution Signed-off-by: Asif Bashar * fix by removing metadata field Signed-off-by: Asif Bashar * fixed explain test after removing of metadata fields in transpose result Signed-off-by: Asif Bashar --------- Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 1 + 1 file changed, 1 insertion(+) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 240d88834a..00a2764c0b 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -47,6 +47,7 @@ ML: 'ML'; FILLNULL: 'FILLNULL'; FLATTEN: 'FLATTEN'; TRENDLINE: 'TRENDLINE'; +TRANSPOSE: 'TRANSPOSE'; CHART: 'CHART'; TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; From 3e3908f904383ef6023946bd245547403c709ce0 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 14:36:43 -0800 Subject: [PATCH 21/36] added test Signed-off-by: Asif Bashar --- .../sql/ppl/NewAddedCommandsIT.java | 29 +++++++++---------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 9d7edb2fce..9d3770886c 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -213,31 +213,28 @@ public void testTransposeCommand() throws IOException { } @Test - public void testFieldFormatCommand() throws IOException { + public void testTransposeCommand() throws IOException { JSONObject result; try { - executeQuery( - String.format( - "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + executeQuery(String.format("search source=%s | transpose ", TEST_INDEX_BANK)); } catch (ResponseException e) { result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); verifyQuery(result); } } - - @Test - public void testFieldFormatCommand() throws IOException { - JSONObject result; - try { - executeQuery( - String.format( - "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - verifyQuery(result); + @Test + public void testFieldFormatCommand() throws IOException { + JSONObject result; + try { + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); + } } - } private void verifyQuery(JSONObject result) throws IOException { if (isCalciteEnabled()) { From 99673af86f278f25917e6d54f2a8717a15a8552e Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 14:52:14 -0800 Subject: [PATCH 22/36] merge from main Signed-off-by: Asif Bashar --- .../org/opensearch/sql/analysis/Analyzer.java | 1 - .../sql/calcite/remote/CalciteExplainIT.java | 16 ++++----- .../sql/ppl/NewAddedCommandsIT.java | 34 +++++++------------ 3 files changed, 20 insertions(+), 31 deletions(-) diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index c532fd8205..5015214796 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -526,7 +526,6 @@ public LogicalPlan visitEval(Eval node, AnalysisContext context) { return new LogicalEval(child, expressionsBuilder.build()); } - /** Build {@link LogicalEval}. */ @Override public LogicalPlan visitFieldFormat(Eval node, AnalysisContext context) { diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 897faa7441..1f1a3653a4 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2477,15 +2477,15 @@ public void testFilterOnMultipleCascadedNestedFields() throws IOException { @Test public void testAggFilterOnNestedFields() throws IOException { - enabledOnlyWhenPushdownIsEnabled(); - assertYamlEqualsIgnoreId( - loadExpectedPlan("agg_filter_nested.yaml"), - explainQueryYaml( - StringUtils.format( - "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", - TEST_INDEX_CASCADED_NESTED))); - + enabledOnlyWhenPushdownIsEnabled(); + assertYamlEqualsIgnoreId( + loadExpectedPlan("agg_filter_nested.yaml"), + explainQueryYaml( + StringUtils.format( + "source=%s | stats count(eval(author.name < 'K')) as george_and_jk", + TEST_INDEX_CASCADED_NESTED))); } + @Test public void testFieldFormatExplain() throws Exception { // test for issue https://github.com/opensearch-project/sql/issues/4903 diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java index 9d3770886c..c5a1d08c37 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/NewAddedCommandsIT.java @@ -201,16 +201,6 @@ public void testAddColTotalCommand() throws IOException { verifyQuery(result); } } - @Test - public void testTransposeCommand() throws IOException { - JSONObject result; - try { - executeQuery(String.format("search source=%s | transpose ", TEST_INDEX_BANK)); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - verifyQuery(result); - } - } @Test public void testTransposeCommand() throws IOException { @@ -223,20 +213,20 @@ public void testTransposeCommand() throws IOException { } } - @Test - public void testFieldFormatCommand() throws IOException { - JSONObject result; - try { - executeQuery( - String.format( - "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); - } catch (ResponseException e) { - result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); - verifyQuery(result); - } + @Test + public void testFieldFormatCommand() throws IOException { + JSONObject result; + try { + executeQuery( + String.format( + "search source=%s | fieldformat double_balance = balance * 2 ", TEST_INDEX_BANK)); + } catch (ResponseException e) { + result = new JSONObject(TestUtils.getResponseBody(e.getResponse())); + verifyQuery(result); } + } - private void verifyQuery(JSONObject result) throws IOException { + private void verifyQuery(JSONObject result) throws IOException { if (isCalciteEnabled()) { assertFalse(result.getJSONArray("datarows").isEmpty()); } else { From 8ead775d8c4fea4687f23863660c57405e5ad59d Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 15:10:54 -0800 Subject: [PATCH 23/36] merge conflict issues Signed-off-by: Asif Bashar --- .../sql/calcite/remote/CalciteExplainIT.java | 29 +++++++------------ 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 1f1a3653a4..67390117f9 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2486,15 +2486,6 @@ public void testAggFilterOnNestedFields() throws IOException { TEST_INDEX_CASCADED_NESTED))); } - @Test - public void testFieldFormatExplain() throws Exception { - // test for issue https://github.com/opensearch-project/sql/issues/4903 - enabledOnlyWhenPushdownIsEnabled(); - String expected = loadExpectedPlan("explain_field_format.yaml"); - assertYamlEqualsIgnoreId( - expected, - explainQueryYaml("source=opensearch-sql_test_index_account | head 5| fieldformat ")); - } @Test public void testFieldFormatExplain() throws Exception { @@ -2505,16 +2496,16 @@ public void testFieldFormatExplain() throws Exception { expected, explainQueryYaml("source=opensearch-sql_test_index_account | head 5| fieldformat ")); } + @Test + public void testExplainMvCombine() throws IOException { + String query = + "source=opensearch-sql_test_index_account " + + "| fields state, city, age " + + "| mvcombine age delim=','"; - @Test - public void testExplainMvCombine() throws IOException { - String query = - "source=opensearch-sql_test_index_account " - + "| fields state, city, age " - + "| mvcombine age delim=','"; + String actual = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_mvcombine.yaml"); + assertYamlEqualsIgnoreId(expected, actual); + } - String actual = explainQueryYaml(query); - String expected = loadExpectedPlan("explain_mvcombine.yaml"); - assertYamlEqualsIgnoreId(expected, actual); - } } From 538efac348a9b10b3349c918a3816e2b1ec37163 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 15:15:50 -0800 Subject: [PATCH 24/36] merge conflict issues Signed-off-by: Asif Bashar --- ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 -- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 2 -- 2 files changed, 4 deletions(-) diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 00a2764c0b..9113663e47 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -72,9 +72,7 @@ LABEL: 'LABEL'; SHOW_NUMBERED_TOKEN: 'SHOW_NUMBERED_TOKEN'; AGGREGATION: 'AGGREGATION'; APPENDPIPE: 'APPENDPIPE'; -FIELDFORMAT: 'FIELDFORMAT'; COLUMN_NAME: 'COLUMN_NAME'; -FIELDFORMAT: 'FIELDFORMAT'; MVCOMBINE: 'MVCOMBINE'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 689a02753c..3fbb1fb4a4 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -134,9 +134,7 @@ commandName | REX | APPENDPIPE | REPLACE - | FIELDFORMAT | TRANSPOSE - | FIELDFORMAT ; searchCommand From 8ec8887dc49d50d4a363cc73c849ca34c09e3a6f Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 15:31:25 -0800 Subject: [PATCH 25/36] doc fix Signed-off-by: Asif Bashar --- docs/user/ppl/cmd/fieldformat.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index fce367ada4..1fefa1389a 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -124,5 +124,6 @@ fetched rows / total rows = 4/4 | Nanette | 28 | Age: 28 years. | | Dale | 33 | Age: 33 years. | +-----------+-----+----------------+ - +``` + From c57b38f5284955e71692cf9cb132949857df4bac Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 15:58:18 -0800 Subject: [PATCH 26/36] test fix Signed-off-by: Asif Bashar --- .../opensearch/sql/calcite/remote/CalciteExplainIT.java | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 67390117f9..d5f6a2f5a2 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2489,12 +2489,16 @@ public void testAggFilterOnNestedFields() throws IOException { @Test public void testFieldFormatExplain() throws Exception { - // test for issue https://github.com/opensearch-project/sql/issues/4903 + enabledOnlyWhenPushdownIsEnabled(); String expected = loadExpectedPlan("explain_field_format.yaml"); assertYamlEqualsIgnoreId( expected, - explainQueryYaml("source=opensearch-sql_test_index_account | head 5| fieldformat ")); + explainQueryYaml( + StringUtils.format( + "source=%s | head 5| fieldformat formatted_balance =" + + " \"$\".tostring(balance,\"commas\") ", + TEST_INDEX_ACCOUNT))); } @Test public void testExplainMvCombine() throws IOException { From 2f6cd1acab62daec1a1a37f4e4d53d2a9ebba231 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 15:58:46 -0800 Subject: [PATCH 27/36] test fix Signed-off-by: Asif Bashar --- .../expectedOutput/calcite/explain_field_format.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 integ-test/src/test/resources/expectedOutput/calcite/explain_field_format.yaml diff --git a/integ-test/src/test/resources/expectedOutput/calcite/explain_field_format.yaml b/integ-test/src/test/resources/expectedOutput/calcite/explain_field_format.yaml new file mode 100644 index 0000000000..202f594bef --- /dev/null +++ b/integ-test/src/test/resources/expectedOutput/calcite/explain_field_format.yaml @@ -0,0 +1,10 @@ +calcite: + logical: | + LogicalSystemLimit(fetch=[10000], type=[QUERY_SIZE_LIMIT]) + LogicalProject(account_number=[$0], firstname=[$1], address=[$2], balance=[$3], gender=[$4], city=[$5], employer=[$6], state=[$7], age=[$8], email=[$9], lastname=[$10], formatted_balance=[||('$':VARCHAR, TOSTRING($3, 'commas':VARCHAR))]) + LogicalSort(fetch=[5]) + CalciteLogicalIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]]) + physical: | + EnumerableCalc(expr#0..10=[{inputs}], expr#11=['$':VARCHAR], expr#12=['commas':VARCHAR], expr#13=[TOSTRING($t3, $t12)], expr#14=[||($t11, $t13)], proj#0..10=[{exprs}], formatted_balance=[$t14]) + CalciteEnumerableIndexScan(table=[[OpenSearch, opensearch-sql_test_index_account]], PushDownContext=[[PROJECT->[account_number, firstname, address, balance, gender, city, employer, state, age, email, lastname], LIMIT->5, LIMIT->10000], OpenSearchRequestBuilder(sourceBuilder={"from":0,"size":5,"timeout":"1m","_source":{"includes":["account_number","firstname","address","balance","gender","city","employer","state","age","email","lastname"],"excludes":[]}}, requestedTotalSize=5, pageSize=null, startFrom=0)]) + From 7b05103b5f6a7ab4fe719de793992f28882c882d Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 16:02:10 -0800 Subject: [PATCH 28/36] coderabbit recommendations Signed-off-by: Asif Bashar --- .../opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java index 98b2917290..e20bd1b0e4 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLFieldFormatTest.java @@ -266,7 +266,7 @@ public void testFieldFormatMaxOnStringsWithSuffix() { "source=EMP | sort EMPNO | head 3 |fields EMPNO, ENAME| fieldformat a = max('banana'," + " 'Door', ENAME).\" after comparing with provided constant strings and ENAME column" + " values.\""; - // # + RelNode root = getRelNode(ppl); String expectedLogical = "LogicalProject(EMPNO=[$0], ENAME=[$1], a=[||(SCALAR_MAX('banana':VARCHAR, 'Door':VARCHAR," From ffd33bd9e1317612e1d27ba6bcb2c4b6bfd35999 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 28 Jan 2026 10:28:04 -0800 Subject: [PATCH 29/36] resolve conflicts Signed-off-by: Asif Bashar --- .../sql/calcite/CalciteNoPushdownIT.java | 1 - .../sql/calcite/remote/CalciteExplainIT.java | 23 +++++++++---------- ppl/src/main/antlr/OpenSearchPPLParser.g4 | 6 +++++ 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 92acc16d19..58e6136817 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -109,7 +109,6 @@ CalciteVisualizationFormatIT.class, CalciteWhereCommandIT.class, CalcitePPLTpchIT.class, - CalciteFieldFormatCommandIT.class CalcitePPLTpchIT.class, CalciteMvCombineCommandIT.class }) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index d5f6a2f5a2..3027967b36 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -2486,6 +2486,17 @@ public void testAggFilterOnNestedFields() throws IOException { TEST_INDEX_CASCADED_NESTED))); } + @Test + public void testExplainMvCombine() throws IOException { + String query = + "source=opensearch-sql_test_index_account " + + "| fields state, city, age " + + "| mvcombine age delim=','"; + + String actual = explainQueryYaml(query); + String expected = loadExpectedPlan("explain_mvcombine.yaml"); + assertYamlEqualsIgnoreId(expected, actual); + } @Test public void testFieldFormatExplain() throws Exception { @@ -2500,16 +2511,4 @@ public void testFieldFormatExplain() throws Exception { + " \"$\".tostring(balance,\"commas\") ", TEST_INDEX_ACCOUNT))); } - @Test - public void testExplainMvCombine() throws IOException { - String query = - "source=opensearch-sql_test_index_account " - + "| fields state, city, age " - + "| mvcombine age delim=','"; - - String actual = explainQueryYaml(query); - String expected = loadExpectedPlan("explain_mvcombine.yaml"); - assertYamlEqualsIgnoreId(expected, actual); - } - } diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 3fbb1fb4a4..8cc4ed932d 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -89,6 +89,7 @@ commands | rexCommand | appendPipeCommand | replaceCommand + | mvcombineCommand | fieldformatCommand ; @@ -134,6 +135,7 @@ commandName | REX | APPENDPIPE | REPLACE + | MVCOMBINE | TRANSPOSE ; @@ -549,6 +551,10 @@ expandCommand : EXPAND fieldExpression (AS alias = qualifiedName)? ; +mvcombineCommand + : MVCOMBINE fieldExpression (DELIM EQUAL stringLiteral)? + ; + flattenCommand : FLATTEN fieldExpression (AS aliases = identifierSeq)? ; From 494df1d81a20911d38a574fda1031a046019aed7 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 28 Jan 2026 10:29:24 -0800 Subject: [PATCH 30/36] resolve conflicts Signed-off-by: Asif Bashar --- .../java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java | 1 - 1 file changed, 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java index 58e6136817..50cdd8d847 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/CalciteNoPushdownIT.java @@ -109,7 +109,6 @@ CalciteVisualizationFormatIT.class, CalciteWhereCommandIT.class, CalcitePPLTpchIT.class, - CalcitePPLTpchIT.class, CalciteMvCombineCommandIT.class }) public class CalciteNoPushdownIT { From 75432248d298806806be8b501e5761ee572f6535 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 28 Jan 2026 11:17:06 -0800 Subject: [PATCH 31/36] resolve conflicts Signed-off-by: Asif Bashar --- .../remote/CalciteFieldFormatCommandIT.java | 26 ++++++++++++------- .../opensearch/sql/ppl/PPLIntegTestCase.java | 7 +---- .../opensearch/sql/ppl/SearchCommandIT.java | 2 +- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java index 5a89f095e6..86f87c90c8 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteFieldFormatCommandIT.java @@ -8,6 +8,7 @@ import static org.opensearch.sql.util.MatcherUtils.*; import java.io.IOException; +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONObject; import org.junit.jupiter.api.Test; import org.opensearch.client.Request; @@ -38,7 +39,10 @@ public void init() throws Exception { @Test public void testFieldFormatStringConcatenation() throws IOException { - JSONObject result = executeQuery("source=test_eval | fieldformat greeting = 'Hello ' + name"); + JSONObject result = + executeQuery( + StringEscapeUtils.escapeJson( + "source=test_eval | fieldformat greeting = 'Hello ' + name")); verifySchema( result, schema("name", "string"), @@ -56,8 +60,9 @@ public void testFieldFormatStringConcatenation() throws IOException { public void testFieldFormatStringConcatenationWithNullFieldToString() throws IOException { JSONObject result = executeQuery( - "source=test_eval | fieldformat age_desc = \"Age: \".tostring(age,\"commas\") | fields" - + " name, age, age_desc"); + StringEscapeUtils.escapeJson( + "source=test_eval | fieldformat age_desc = \"Age: \".tostring(age,\"commas\") |" + + " fields name, age, age_desc")); verifySchema( result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); verifyDataRows( @@ -71,8 +76,9 @@ public void testFieldFormatStringConcatenationWithNullFieldToString() throws IOE public void testFieldFormatStringConcatenationWithNullField() throws IOException { JSONObject result = executeQuery( - "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING) | fields name," - + " age, age_desc"); + StringEscapeUtils.escapeJson( + "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING) | fields" + + " name, age, age_desc")); verifySchema( result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); verifyDataRows( @@ -86,8 +92,9 @@ public void testFieldFormatStringConcatenationWithNullField() throws IOException public void testFieldFormatStringConcatWithSuffix() throws IOException { JSONObject result = executeQuery( - "source=test_eval | fieldformat age_desc = CAST(age AS STRING).\" years\" | fields" - + " name, age, age_desc"); + StringEscapeUtils.escapeJson( + "source=test_eval | fieldformat age_desc = CAST(age AS STRING).\" years\" | fields" + + " name, age, age_desc")); verifySchema( result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); verifyDataRows( @@ -101,8 +108,9 @@ public void testFieldFormatStringConcatWithSuffix() throws IOException { public void testFieldFormatStringConcatWithPrefixSuffix() throws IOException { JSONObject result = executeQuery( - "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING).\" years\" |" - + " fields name, age, age_desc"); + StringEscapeUtils.escapeJson( + "source=test_eval | fieldformat age_desc = \"Age: \".CAST(age AS STRING).\" years\"" + + " | fields name, age, age_desc")); verifySchema( result, schema("name", "string"), schema("age", "bigint"), schema("age_desc", "string")); verifyDataRows( diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java index a280f88480..6135d74b2f 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/PPLIntegTestCase.java @@ -14,7 +14,6 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Locale; -import org.apache.commons.text.StringEscapeUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONException; @@ -143,11 +142,7 @@ protected void failWithMessage(String query, String message) { protected Request buildRequest(String query, String endpoint) { Request request = new Request("POST", endpoint); - request.setJsonEntity( - String.format( - Locale.ROOT, - "{\n" + " \"query\": \"%s\"\n" + "}", - StringEscapeUtils.escapeJson(query))); + request.setJsonEntity(String.format(Locale.ROOT, "{\n" + " \"query\": \"%s\"\n" + "}", query)); RequestOptions.Builder restOptionsBuilder = RequestOptions.DEFAULT.toBuilder(); restOptionsBuilder.addHeader("Content-Type", "application/json"); diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java index b8e5b2a572..eb8be689ea 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java @@ -724,7 +724,7 @@ public void testSearchWithTypeMismatch() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s severityNumber=\\\"not-a-number\\\"", TEST_INDEX_OTEL_LOGS)); + "search source=%s severityNumber=\"not-a-number\"", TEST_INDEX_OTEL_LOGS)); verifyDataRows(result); } From 1a5d91662f5e7b8bded5ebffc0ffb298e3926abe Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 28 Jan 2026 11:25:40 -0800 Subject: [PATCH 32/36] resolve conflicts Signed-off-by: Asif Bashar --- .../src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java index eb8be689ea..b8e5b2a572 100644 --- a/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/ppl/SearchCommandIT.java @@ -724,7 +724,7 @@ public void testSearchWithTypeMismatch() throws IOException { JSONObject result = executeQuery( String.format( - "search source=%s severityNumber=\"not-a-number\"", TEST_INDEX_OTEL_LOGS)); + "search source=%s severityNumber=\\\"not-a-number\\\"", TEST_INDEX_OTEL_LOGS)); verifyDataRows(result); } From 54b3a064629fcd1ade4e5a6b9c8e5c376c5ac705 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Wed, 28 Jan 2026 12:26:12 -0800 Subject: [PATCH 33/36] resolve conflicts Signed-off-by: Asif Bashar --- .../sql/calcite/remote/CalciteExplainIT.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java index 3027967b36..6c25242c11 100644 --- a/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/calcite/remote/CalciteExplainIT.java @@ -25,6 +25,7 @@ import java.io.IOException; import java.util.Locale; +import org.apache.commons.text.StringEscapeUtils; import org.junit.Ignore; import org.junit.Test; import org.opensearch.sql.ast.statement.ExplainMode; @@ -2506,9 +2507,10 @@ public void testFieldFormatExplain() throws Exception { assertYamlEqualsIgnoreId( expected, explainQueryYaml( - StringUtils.format( - "source=%s | head 5| fieldformat formatted_balance =" - + " \"$\".tostring(balance,\"commas\") ", - TEST_INDEX_ACCOUNT))); + StringEscapeUtils.escapeJson( + StringUtils.format( + "source=%s | head 5| fieldformat formatted_balance =" + + " \"$\".tostring(balance,\"commas\") ", + TEST_INDEX_ACCOUNT)))); } } From f5703a4a47226b09a29d05098a8fda2e8922ab02 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Thu, 29 Jan 2026 22:24:59 -0800 Subject: [PATCH 34/36] removed ambigous sentence Signed-off-by: Asif Bashar --- docs/user/ppl/cmd/fieldformat.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/user/ppl/cmd/fieldformat.md b/docs/user/ppl/cmd/fieldformat.md index 1fefa1389a..cf03205794 100644 --- a/docs/user/ppl/cmd/fieldformat.md +++ b/docs/user/ppl/cmd/fieldformat.md @@ -10,7 +10,6 @@ Additionally, it also provides string concatenation dot operator followed by and The `fieldformat` command has the following syntax: ```syntax -Following are possible syntaxes: fieldformat =[(prefix).][.(suffix)] ["," =[(prefix).][.(suffix)] ]... ``` From 72bef4b1d82c0fbd677599ff19021143d11f47c9 Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 27 Jan 2026 11:56:29 -0800 Subject: [PATCH 35/36] added missing test files Signed-off-by: Asif Bashar --- .../security/CalciteCrossClusterSearchIT.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java index b89574a4fd..572a84a92a 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java @@ -13,6 +13,8 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; + +import org.apache.commons.text.StringEscapeUtils; import org.json.JSONObject; import org.junit.Test; @@ -401,4 +403,20 @@ public void testCrossClusterMvcombine() throws IOException { rows("Hattie", new org.json.JSONArray().put(36)), rows("Nanette", new org.json.JSONArray().put(28))); } + /** CrossClusterSearchIT Test for fieldformat. */ + @Test + public void testCrossClusterFieldFormat() throws IOException { + // Test fieldformat command with tostring + JSONObject result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | where firstname='Hattie' or firstname ='Nanette'|fields" + + " firstname,age,balance | fieldformat formatted_balance =" + + " \"$\".tostring(balance,\"commas\")", + TEST_INDEX_BANK_REMOTE))); + verifyDataRows( + result, rows("Hattie", 36, 5686, "$5,686"), rows("Nanette", 28, 32838, "$32,838")); + } + } From b9b29089ec9a9805be8e1f8d9ba0ebe675d3e32d Mon Sep 17 00:00:00 2001 From: Asif Bashar Date: Tue, 3 Feb 2026 12:00:23 -0800 Subject: [PATCH 36/36] spotlessApply Signed-off-by: Asif Bashar --- .../security/CalciteCrossClusterSearchIT.java | 31 +++++++++---------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java index 572a84a92a..571d915517 100644 --- a/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/security/CalciteCrossClusterSearchIT.java @@ -13,7 +13,6 @@ import static org.opensearch.sql.util.MatcherUtils.verifySchema; import java.io.IOException; - import org.apache.commons.text.StringEscapeUtils; import org.json.JSONObject; import org.junit.Test; @@ -403,20 +402,20 @@ public void testCrossClusterMvcombine() throws IOException { rows("Hattie", new org.json.JSONArray().put(36)), rows("Nanette", new org.json.JSONArray().put(28))); } - /** CrossClusterSearchIT Test for fieldformat. */ - @Test - public void testCrossClusterFieldFormat() throws IOException { - // Test fieldformat command with tostring - JSONObject result = - executeQuery( - StringEscapeUtils.escapeJson( - String.format( - "search source=%s | where firstname='Hattie' or firstname ='Nanette'|fields" - + " firstname,age,balance | fieldformat formatted_balance =" - + " \"$\".tostring(balance,\"commas\")", - TEST_INDEX_BANK_REMOTE))); - verifyDataRows( - result, rows("Hattie", 36, 5686, "$5,686"), rows("Nanette", 28, 32838, "$32,838")); - } + /** CrossClusterSearchIT Test for fieldformat. */ + @Test + public void testCrossClusterFieldFormat() throws IOException { + // Test fieldformat command with tostring + JSONObject result = + executeQuery( + StringEscapeUtils.escapeJson( + String.format( + "search source=%s | where firstname='Hattie' or firstname ='Nanette'|fields" + + " firstname,age,balance | fieldformat formatted_balance =" + + " \"$\".tostring(balance,\"commas\")", + TEST_INDEX_BANK_REMOTE))); + verifyDataRows( + result, rows("Hattie", 36, 5686, "$5,686"), rows("Nanette", 28, 32838, "$32,838")); + } }