Add tests for nested in select clause

integ-test/src/test/java/org/opensearch/sql/legacy/SQLIntegTestCase.java

@@ -506,6 +506,10 @@ public enum Index {
         "nestedType",
         getNestedTypeIndexMapping(),
         "src/test/resources/nested_objects.json"),
+    NESTED_WITHOUT_ARRAYS(TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS,
+        "nestedTypeWithoutArrays",
+        getNestedTypeIndexMapping(),
+        "src/test/resources/nested_objects_without_arrays.json"),
     NESTED_WITH_QUOTES(TestsConstants.TEST_INDEX_NESTED_WITH_QUOTES,
         "nestedType",
         getNestedTypeIndexMapping(),
@@ -594,9 +598,9 @@ public enum Index {
         "wildcard",
         getMappingFile("wildcard_index_mappings.json"),
         "src/test/resources/wildcard.json"),
-    MULTI_NESTED(TestsConstants.TEST_INDEX_MULTI_NESTED,
+    MULTI_NESTED(TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE,
         "multi_nested",
-        getMappingFile("indexDefinitions/multi_nested.json"),
+        getMappingFile("multi_nested.json"),
         "src/test/resources/multi_nested_objects.json");
 
     private final String name;

integ-test/src/test/java/org/opensearch/sql/legacy/TestsConstants.java

@@ -31,6 +31,8 @@ public class TestsConstants {
   public final static String TEST_INDEX_LOCATION = TEST_INDEX + "_location";
   public final static String TEST_INDEX_LOCATION2 = TEST_INDEX + "_location2";
   public final static String TEST_INDEX_NESTED_TYPE = TEST_INDEX + "_nested_type";
+  public final static String TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS =
+      TEST_INDEX + "_nested_type_without_arrays";
   public final static String TEST_INDEX_NESTED_SIMPLE = TEST_INDEX + "_nested_simple";
   public final static String TEST_INDEX_NESTED_WITH_QUOTES =
       TEST_INDEX + "_nested_type_with_quotes";
@@ -55,7 +57,7 @@ public class TestsConstants {
   public final static String TEST_INDEX_NULL_MISSING = TEST_INDEX + "_null_missing";
   public final static String TEST_INDEX_CALCS = TEST_INDEX + "_calcs";
   public final static String TEST_INDEX_WILDCARD = TEST_INDEX + "_wildcard";
-  public final static String TEST_INDEX_MULTI_NESTED= TEST_INDEX + "_multi_nested";
+  public final static String TEST_INDEX_MULTI_NESTED_TYPE = TEST_INDEX + "_multi_nested";
 
   public final static String DATE_FORMAT = "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'";
   public final static String TS_DATE_FORMAT = "yyyy-MM-dd HH:mm:ss.SSS";

integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java

@@ -5,24 +5,107 @@
 
 package org.opensearch.sql.sql;
 
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE;
+import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS;
+import static org.opensearch.sql.util.MatcherUtils.rows;
+import static org.opensearch.sql.util.MatcherUtils.schema;
+import static org.opensearch.sql.util.MatcherUtils.verifyDataRows;
+import static org.opensearch.sql.util.MatcherUtils.verifySchema;
+
+import java.io.IOException;
+import java.util.List;
+import org.json.JSONArray;
 import org.json.JSONObject;
 import org.junit.Test;
 import org.opensearch.sql.legacy.SQLIntegTestCase;
 
-import java.io.IOException;
-
-import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED;
-
 public class NestedIT extends SQLIntegTestCase {
   @Override
   public void init() throws IOException {
     loadIndex(Index.MULTI_NESTED);
+    loadIndex(Index.NESTED);
+    loadIndex(Index.NESTED_WITHOUT_ARRAYS);
+  }
+
+  @Test
+  public void nested_function_with_array_of_nested_field_test() {
+    String query = "SELECT nested(message.info), nested(comment.data) FROM " + TEST_INDEX_NESTED_TYPE;
+    JSONObject result = executeJdbcRequest(query);
+
+    assertEquals(5, result.getInt("total"));
+    verifyDataRows(result,
+        rows("a", "ab"),
+        rows("b", "aa"),
+        rows("c", "aa"),
+        rows(new JSONArray(List.of("c","a")), "ab"),
+        rows(new JSONArray(List.of("zz")), new JSONArray(List.of("aa", "bb"))));
+  }
+
+  @Test
+  public void nested_function_in_select_test() {
+    String query = "SELECT nested(message.info), nested(comment.data), "
+        + "nested(message.dayOfWeek) FROM "
+        + TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS;
+    JSONObject result = executeJdbcRequest(query);
+
+    assertEquals(5, result.getInt("total"));
+    verifySchema(result,
+        schema("nested(message.info)", null, "keyword"),
+        schema("nested(comment.data)", null, "keyword"),
+        schema("nested(message.dayOfWeek)", null, "long"));
+    verifyDataRows(result,
+        rows("a", "ab", 1),
+        rows("b", "aa", 2),
+        rows("c", "aa", 1),
+        rows("c", "ab", 4),
+        rows("zz", "bb", 6));
   }
 
+  // Has to be tested with JSON format when https://github.com/opensearch-project/sql/issues/1317
+  // gets resolved
   @Test
-  public void nested_string_subfield_test() {
-    String query = "SELECT nested(message.info) FROM " + TEST_INDEX_MULTI_NESTED;
+  public void nested_function_in_an_aggregate_function_in_select_test() {
+    String query = "SELECT sum(nested(message.dayOfWeek)) FROM " +
+        TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS;
     JSONObject result = executeJdbcRequest(query);
-    assertEquals(6, result.getInt("total"));
+    verifyDataRows(result, rows(14));
+  }
+
+  @Test
+  public void nested_function_with_arrays_in_an_aggregate_function_in_select_test() {
+    String query = "SELECT sum(nested(message.dayOfWeek)) FROM " +
+        TEST_INDEX_NESTED_TYPE;
+    JSONObject result = executeJdbcRequest(query);
+    verifyDataRows(result, rows(19));
+  }
+
+  @Test
+  public void nested_function_in_a_function_in_select_test() {
+    String query = "SELECT upper(nested(message.info)) FROM " +
+        TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS;
+    JSONObject result = executeJdbcRequest(query);
+
+    verifyDataRows(result,
+        rows("A"),
+        rows("B"),
+        rows("C"),
+        rows("C"),
+        rows("ZZ"));
+  }
+
+
+  @Test
+  public void nested_function_with_array_of_multi_nested_field_test() {
+    String query = "SELECT nested(message.author.name) FROM " + TEST_INDEX_MULTI_NESTED_TYPE;
+    JSONObject result = executeJdbcRequest(query);
+
+    assertEquals(5, result.getInt("total"));
+    verifyDataRows(result,
+        rows("e"),
+        rows("f"),
+        rows("g"),
+        rows(new JSONArray(List.of("h", "p"))),
+        rows(new JSONArray(List.of("yy"))));
   }
 }

integ-test/src/test/resources/multi_nested_objects.json

@@ -1,9 +1,9 @@
 {"index":{"_id":"1"}}
-{"message":[{"info":"a","author":{"name": "e", "address": {"street": "bc", "number": 1}},"dayOfWeek":1}]}
+{"message":{"info":"a","author":{"name": "e", "address": {"street": "bc", "number": 1}},"dayOfWeek":1}}
 {"index":{"_id":"2"}}
-{"message":[{"info":"b","author":{"name": "f", "address": {"street": "ab", "number": 2}},"dayOfWeek":2}]}
+{"message":{"info":"b","author":{"name": "f", "address": {"street": "ab", "number": 2}},"dayOfWeek":2}}
 {"index":{"_id":"3"}}
-{"message":[{"info":"c","author":{"name": "g", "address": {"street": "sk", "number": 3}},"dayOfWeek":1}]}
+{"message":{"info":"c","author":{"name": "g", "address": {"street": "sk", "number": 3}},"dayOfWeek":1}}
 {"index":{"_id":"4"}}
 {"message":[{"info":"d","author":{"name": "h", "address": {"street": "mb", "number": 4}},"dayOfWeek":4},{"info":"i","author":{"name": "p", "address": {"street": "on", "number": 5}},"dayOfWeek":5}]}
 {"index":{"_id":"5"}}

integ-test/src/test/resources/nested_objects_without_arrays.json

@@ -0,0 +1,10 @@
+{"index":{"_id":"1"}}
+{"message":{"info":"a","author":"e","dayOfWeek":1},"comment":{"data":"ab","likes":3},"myNum":1,"someField":"b"}
+{"index":{"_id":"2"}}
+{"message":{"info":"b","author":"f","dayOfWeek":2},"comment":{"data":"aa","likes":2},"myNum":2,"someField":"a"}
+{"index":{"_id":"3"}}
+{"message":{"info":"c","author":"g","dayOfWeek":1},"comment":{"data":"aa","likes":3},"myNum":3,"someField":"a"}
+{"index":{"_id":"4"}}
+{"message":{"info":"c","author":"h","dayOfWeek":4},"comment":{"data":"ab","likes":1},"myNum":4,"someField":"b"}
+{"index":{"_id":"5"}}
+{"message": {"info":"zz","author":"zz","dayOfWeek":6},"comment":{"data":"bb","likes":10},"myNum":3,"someField":"a"}

sql/src/main/antlr/OpenSearchSQLParser.g4

@@ -317,7 +317,7 @@ nestedFunction
     ;
 
 nestedField
-    : ID DOT ID (DOT ID)*
+    : nestedIdent DOT nestedIdent (DOT nestedIdent)*
     ;
 
 highlightFunction
@@ -624,6 +624,14 @@ ident
     | scalarFunctionName
     ;
 
+
+nestedIdent
+    : ID
+    | BACKTICK_QUOTE_ID
+    | keywordsCanBeId
+    | scalarFunctionName
+    ;
+
 keywordsCanBeId
     : FULL
     | FIELD | D | T | TS // OD SQL and ODBC special

sql/src/test/java/org/opensearch/sql/sql/antlr/SQLSyntaxParserTest.java

@@ -509,6 +509,24 @@ public void can_parse_wildcard_query_relevance_function() {
             + "boost=1.5, case_insensitive=true, rewrite=\"scoring_boolean\")"));
   }
 
+  @Test
+  public void can_parse_nested_function() {
+    assertNotNull(
+        parser.parse("SELECT NESTED(FIELD.DAYOFWEEK) FROM TEST"));
+    assertNotNull(
+        parser.parse("SELECT SUM(NESTED(FIELD.SUBFIELD)) FROM TEST"));
+    assertNotNull(
+        parser.parse("SELECT COUNT(*) FROM TEST GROUP BY NESTED(FIELD.SUBFIELD)"));
+  }
+
+  @Test
+  public void can_not_parse_nested_function_without_dot() {
+    assertThrows(SyntaxCheckException.class,
+        () -> parser.parse("SELECT NESTED(FIELD) FROM TEST"));
+    assertThrows(SyntaxCheckException.class,
+        () -> parser.parse("SELECT COUNT(*) FROM TEST GROUP BY NESTED(FIELD)"));
+  }
+
   @Test
   public void describe_request_accepts_only_quoted_string_literals() {
     assertAll(