diff --git a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java index b1db21a2ff..ce781123d6 100644 --- a/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/legacy/ObjectFieldSelectIT.java @@ -14,7 +14,6 @@ import org.json.JSONArray; import org.json.JSONObject; -import org.junit.Assume; import org.junit.Test; import org.opensearch.sql.legacy.utils.StringUtils; diff --git a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java index 6cb7b7580b..80886fe779 100644 --- a/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java +++ b/integ-test/src/test/java/org/opensearch/sql/sql/NestedIT.java @@ -6,6 +6,7 @@ package org.opensearch.sql.sql; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_MULTI_NESTED_TYPE; +import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_SIMPLE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_TYPE_WITHOUT_ARRAYS; import static org.opensearch.sql.legacy.TestsConstants.TEST_INDEX_NESTED_WITH_NULLS; @@ -31,6 +32,7 @@ public void init() throws IOException { loadIndex(Index.NESTED_WITHOUT_ARRAYS); loadIndex(Index.EMPLOYEE_NESTED); loadIndex(Index.NESTED_WITH_NULLS); + loadIndex(Index.NESTED_SIMPLE); } @Test @@ -366,4 +368,37 @@ public void test_nested_in_where_as_predicate_expression_with_relevance_query() assertEquals(1, result.getInt("total")); verifyDataRows(result, rows(10, "a")); } + + @Test + public void nested_function_with_date_types_as_object_arrays_within_arrays_test() { + String query = "SELECT nested(address.moveInDate) FROM " + TEST_INDEX_NESTED_SIMPLE; + JSONObject result = executeJdbcRequest(query); + + assertEquals(11, result.getInt("total")); + verifySchema(result, + schema("nested(address.moveInDate)", null, "object") + ); + verifyDataRows(result, + rows(new JSONObject(Map.of("dateAndTime","1984-04-12 09:07:42"))), + rows(new JSONArray( + List.of( + Map.of("dateAndTime", "2023-05-03 08:07:42"), + Map.of("dateAndTime", "2001-11-11 04:07:44")) + ) + ), + rows(new JSONObject(Map.of("dateAndTime", "1966-03-19 03:04:55"))), + rows(new JSONObject(Map.of("dateAndTime", "2011-06-01 01:01:42"))), + rows(new JSONObject(Map.of("dateAndTime", "1901-08-11 04:03:33"))), + rows(new JSONObject(Map.of("dateAndTime", "2023-05-03 08:07:42"))), + rows(new JSONObject(Map.of("dateAndTime", "2001-11-11 04:07:44"))), + rows(new JSONObject(Map.of("dateAndTime", "1977-07-13 09:04:41"))), + rows(new JSONObject(Map.of("dateAndTime", "1933-12-12 05:05:45"))), + rows(new JSONObject(Map.of("dateAndTime", "1909-06-17 01:04:21"))), + rows(new JSONArray( + List.of( + Map.of("dateAndTime", "2001-11-11 04:07:44")) + ) + ) + ); + } } diff --git a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json index 2ebc8a50de..7e521cdd44 100644 --- a/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json +++ b/integ-test/src/test/resources/indexDefinitions/nested_simple_index_mapping.json @@ -21,6 +21,14 @@ "ignore_above": 256 } } + }, + "moveInDate" : { + "properties": { + "dateAndTime": { + "type": "date", + "format": "basic_date_time" + } + } } } }, diff --git a/integ-test/src/test/resources/nested_simple.json b/integ-test/src/test/resources/nested_simple.json index d42cc667df..f3cb1a5ebe 100644 --- a/integ-test/src/test/resources/nested_simple.json +++ b/integ-test/src/test/resources/nested_simple.json @@ -1,10 +1,10 @@ {"index":{"_id":"1"}} -{"name":"abbas","age":24,"address":[{"city":"New york city","state":"NY"},{"city":"bellevue","state":"WA"},{"city":"seattle","state":"WA"},{"city":"chicago","state":"IL"}]} +{"name":"abbas","age":24,"address":[{"city":"New york city","state":"NY","moveInDate":{"dateAndTime":"19840412T090742.000Z"}},{"city":"bellevue","state":"WA","moveInDate":[{"dateAndTime":"20230503T080742.000Z"},{"dateAndTime":"20011111T040744.000Z"}]},{"city":"seattle","state":"WA","moveInDate":{"dateAndTime":"19660319T030455.000Z"}},{"city":"chicago","state":"IL","moveInDate":{"dateAndTime":"20110601T010142.000Z"}}]} {"index":{"_id":"2"}} -{"name":"chen","age":32,"address":[{"city":"Miami","state":"Florida"},{"city":"los angeles","state":"CA"}]} +{"name":"chen","age":32,"address":[{"city":"Miami","state":"Florida","moveInDate":{"dateAndTime":"19010811T040333.000Z"}},{"city":"los angeles","state":"CA","moveInDate":{"dateAndTime":"20230503T080742.000Z"}}]} {"index":{"_id":"3"}} -{"name":"peng","age":26,"address":[{"city":"san diego","state":"CA"},{"city":"austin","state":"TX"}]} +{"name":"peng","age":26,"address":[{"city":"san diego","state":"CA","moveInDate":{"dateAndTime":"20011111T040744.000Z"}},{"city":"austin","state":"TX","moveInDate":{"dateAndTime":"19770713T090441.000Z"}}]} {"index":{"_id":"4"}} -{"name":"andy","age":19,"id":4,"address":[{"city":"houston","state":"TX"}]} +{"name":"andy","age":19,"id":4,"address":[{"city":"houston","state":"TX","moveInDate":{"dateAndTime":"19331212T050545.000Z"}}]} {"index":{"_id":"5"}} -{"name":"david","age":25,"address":[{"city":"raleigh","state":"NC"},{"city":"charlotte","state":"SC"}]} +{"name":"david","age":25,"address":[{"city":"raleigh","state":"NC","moveInDate":{"dateAndTime":"19090617T010421.000Z"}},{"city":"charlotte","state":"SC","moveInDate":[{"dateAndTime":"20011111T040744.000Z"}]}]} diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java index 94cd9d93ca..992689a186 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/Content.java @@ -29,11 +29,36 @@ public interface Content { */ boolean isNumber(); + /** + * Is float value. + */ + boolean isFloat(); + + /** + * Is double value. + */ + boolean isDouble(); + + /** + * Is long value. + */ + boolean isLong(); + + /** + * Is boolean value. + */ + boolean isBoolean(); + /** * Is string value. */ boolean isString(); + /** + * Is array value. + */ + boolean isArray(); + /** * Get integer value. */ diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java index 15e2e959a4..e8875d19ba 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/ObjectContent.java @@ -6,6 +6,7 @@ package org.opensearch.sql.opensearch.data.utils; +import com.fasterxml.jackson.databind.node.ArrayNode; import java.util.AbstractMap; import java.util.Iterator; import java.util.List; @@ -103,6 +104,31 @@ public boolean isNumber() { return value instanceof Number; } + @Override + public boolean isFloat() { + return value instanceof Float; + } + + @Override + public boolean isDouble() { + return value instanceof Double; + } + + @Override + public boolean isLong() { + return value instanceof Long; + } + + @Override + public boolean isBoolean() { + return value instanceof Boolean; + } + + @Override + public boolean isArray() { + return value instanceof ArrayNode; + } + @Override public boolean isString() { return value instanceof String; diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java index 13a1fbf6a4..61da7c3b74 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/utils/OpenSearchJsonContent.java @@ -88,11 +88,36 @@ public boolean isNumber() { return value().isNumber(); } + @Override + public boolean isLong() { + return value().isLong(); + } + + @Override + public boolean isFloat() { + return value().isFloat(); + } + + @Override + public boolean isDouble() { + return value().isDouble(); + } + @Override public boolean isString() { return value().isTextual(); } + @Override + public boolean isBoolean() { + return value().isBoolean(); + } + + @Override + public boolean isArray() { + return value().isArray(); + } + @Override public Object objectValue() { return value(); @@ -126,11 +151,10 @@ public Pair geoValue() { } /** - * Return the first element if is OpenSearch Array. - * https://www.elastic.co/guide/en/elasticsearch/reference/current/array.html. + * Getter for value. If value is array the whole array is returned. */ private JsonNode value() { - return value.isArray() ? value.get(0) : value; + return value; } /** diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java index 1ff5af7304..abad197bd4 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactory.java @@ -6,8 +6,14 @@ package org.opensearch.sql.opensearch.data.value; +import static org.opensearch.sql.data.type.ExprCoreType.ARRAY; +import static org.opensearch.sql.data.type.ExprCoreType.BOOLEAN; import static org.opensearch.sql.data.type.ExprCoreType.DATE; import static org.opensearch.sql.data.type.ExprCoreType.DATETIME; +import static org.opensearch.sql.data.type.ExprCoreType.DOUBLE; +import static org.opensearch.sql.data.type.ExprCoreType.FLOAT; +import static org.opensearch.sql.data.type.ExprCoreType.INTEGER; +import static org.opensearch.sql.data.type.ExprCoreType.LONG; import static org.opensearch.sql.data.type.ExprCoreType.STRING; import static org.opensearch.sql.data.type.ExprCoreType.STRUCT; import static org.opensearch.sql.data.type.ExprCoreType.TIME; @@ -18,10 +24,10 @@ import static org.opensearch.sql.utils.DateTimeUtils.UTC_ZONE_ID; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterators; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; @@ -55,8 +61,11 @@ import org.opensearch.sql.data.model.ExprTupleValue; import org.opensearch.sql.data.model.ExprValue; import org.opensearch.sql.data.type.ExprType; +import org.opensearch.sql.opensearch.data.type.OpenSearchBinaryType; import org.opensearch.sql.opensearch.data.type.OpenSearchDataType; import org.opensearch.sql.opensearch.data.type.OpenSearchDateType; +import org.opensearch.sql.opensearch.data.type.OpenSearchGeoPointType; +import org.opensearch.sql.opensearch.data.type.OpenSearchIpType; import org.opensearch.sql.opensearch.data.utils.Content; import org.opensearch.sql.opensearch.data.utils.ObjectContent; import org.opensearch.sql.opensearch.data.utils.OpenSearchJsonContent; @@ -149,10 +158,10 @@ public OpenSearchExprValueFactory(Map typeMapping) { * { "employ.id", "INTEGER" } * { "employ.state", "STRING" } */ - public ExprValue construct(String jsonString) { + public ExprValue construct(String jsonString, boolean supportArrays) { try { return parse(new OpenSearchJsonContent(OBJECT_MAPPER.readTree(jsonString)), TOP_PATH, - Optional.of(STRUCT)); + Optional.of(STRUCT), supportArrays); } catch (JsonProcessingException e) { throw new IllegalStateException(String.format("invalid json: %s.", jsonString), e); } @@ -167,21 +176,27 @@ public ExprValue construct(String jsonString) { * @param value value object * @return ExprValue */ - public ExprValue construct(String field, Object value) { - return parse(new ObjectContent(value), field, type(field)); + public ExprValue construct(String field, Object value, boolean supportArrays) { + return parse(new ObjectContent(value), field, type(field), supportArrays); } - private ExprValue parse(Content content, String field, Optional fieldType) { + private ExprValue parse( + Content content, + String field, + Optional fieldType, + boolean supportArrays + ) { if (content.isNull() || !fieldType.isPresent()) { return ExprNullValue.of(); } ExprType type = fieldType.get(); - if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) - || type == STRUCT) { - return parseStruct(content, field); - } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested))) { - return parseArray(content, field); + if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + || content.isArray()) { + return parseArray(content, field, type, supportArrays); + } else if (type.equals(OpenSearchDataType.of(OpenSearchDataType.MappingType.Object)) + || type == STRUCT) { + return parseStruct(content, field, supportArrays); } else { if (typeActionMap.containsKey(type)) { return typeActionMap.get(type).apply(content, type); @@ -338,39 +353,97 @@ private ExprValue createOpenSearchDateType(Content value, ExprType type) { return new ExprTimestampValue((Instant) value.objectValue()); } - private ExprValue parseStruct(Content content, String prefix) { + /** + * Parse struct content. + * @param content Content to parse. + * @param prefix Prefix for Level of object depth to parse. + * @param supportArrays Parsing the whole array if array is type nested. + * @return Value parsed from content. + */ + private ExprValue parseStruct(Content content, String prefix, boolean supportArrays) { LinkedHashMap result = new LinkedHashMap<>(); content.map().forEachRemaining(entry -> result.put(entry.getKey(), parse(entry.getValue(), makeField(prefix, entry.getKey()), - type(makeField(prefix, entry.getKey()))))); + type(makeField(prefix, entry.getKey())), supportArrays))); return new ExprTupleValue(result); } /** - * Todo. ARRAY is not completely supported now. In OpenSearch, there is no dedicated array type. - * docs - * The similar data type is nested, but it can only allow a list of objects. + * Parse array content. Can also parse nested which isn't necessarily an array. + * @param content Content to parse. + * @param prefix Prefix for Level of object depth to parse. + * @param type Type of content parsing. + * @param supportArrays Parsing the whole array if array is type nested. + * @return Value parsed from content. */ - private ExprValue parseArray(Content content, String prefix) { + private ExprValue parseArray( + Content content, + String prefix, + ExprType type, + boolean supportArrays + ) { List result = new ArrayList<>(); - // ExprCoreType.ARRAY does not indicate inner elements type. - if (Iterators.size(content.array()) == 1 && content.objectValue() instanceof JsonNode) { - result.add(parse(content, prefix, Optional.of(STRUCT))); + + // ARRAY is mapped to nested but can take the json structure of an Object. + if (content.objectValue() instanceof ObjectNode) { + result.add(parseStruct(content, prefix, supportArrays)); + // non-object type arrays are only supported when parsing inner_hits of OS response. + } else if ( + !(type instanceof OpenSearchDataType + && ((OpenSearchDataType) type).getExprType().equals(ARRAY)) + && !supportArrays) { + return parseInnerArrayValue(content.array().next(), prefix, type, supportArrays); } else { content.array().forEachRemaining(v -> { - // ExprCoreType.ARRAY does not indicate inner elements type. OpenSearch nested will be an - // array of structs, otherwise parseArray currently only supports array of strings. - if (v.isString()) { - result.add(parse(v, prefix, Optional.of(OpenSearchDataType.of(STRING)))); - } else { - result.add(parse(v, prefix, Optional.of(STRUCT))); - } + result.add(parseInnerArrayValue(v, prefix, type, supportArrays)); }); } return new ExprCollectionValue(result); } + /** + * Parse inner array value. Can be object type and recurse continues. + * @param content Array index being parsed. + * @param prefix Prefix for value. + * @param type Type of inner array value. + * @param supportArrays Parsing the whole array if array is type nested. + * @return Inner array value. + */ + private ExprValue parseInnerArrayValue( + Content content, + String prefix, + ExprType type, + boolean supportArrays + ) { + if (type instanceof OpenSearchIpType + || type instanceof OpenSearchBinaryType + || type instanceof OpenSearchDateType + || type instanceof OpenSearchGeoPointType) { + return parse(content, prefix, Optional.of(type), supportArrays); + } else if (content.isString()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(STRING)), supportArrays); + } else if (content.isLong()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(LONG)), supportArrays); + } else if (content.isFloat()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(FLOAT)), supportArrays); + } else if (content.isDouble()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(DOUBLE)), supportArrays); + } else if (content.isNumber()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(INTEGER)), supportArrays); + } else if (content.isBoolean()) { + return parse(content, prefix, Optional.of(OpenSearchDataType.of(BOOLEAN)), supportArrays); + } else { + return parse(content, prefix, Optional.of(STRUCT), supportArrays); + } + } + + /** + * Make complete path string for field. + * @param path Path of field. + * @param field Field to append to path. + * @return Field appended to path level. + */ private String makeField(String path, String field) { return path.equalsIgnoreCase(TOP_PATH) ? field : String.join(".", path, field); } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java index 733fad6203..973624d19a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/response/OpenSearchResponse.java @@ -15,15 +15,15 @@ import com.google.common.collect.ImmutableMap; import java.util.Arrays; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.stream.Collectors; import lombok.EqualsAndHashCode; import lombok.ToString; import org.opensearch.action.search.SearchResponse; +import org.opensearch.common.text.Text; +import org.opensearch.search.SearchHit; import org.opensearch.search.SearchHits; import org.opensearch.search.aggregations.Aggregations; import org.opensearch.sql.data.model.ExprFloatValue; @@ -107,61 +107,108 @@ public boolean isAggregationResponse() { */ public Iterator iterator() { if (isAggregationResponse()) { - return exprValueFactory.getParser().parse(aggregations).stream().map(entry -> { - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - for (Map.Entry value : entry.entrySet()) { - builder.put(value.getKey(), exprValueFactory.construct(value.getKey(), value.getValue())); - } - return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); - }).iterator(); + return handleAggregationResponse(); } else { - List metaDataFieldSet = includes.stream() - .filter(include -> METADATAFIELD_TYPE_MAP.containsKey(include)) - .collect(Collectors.toList()); - ExprFloatValue maxScore = Float.isNaN(hits.getMaxScore()) - ? null : new ExprFloatValue(hits.getMaxScore()); return Arrays.stream(hits.getHits()) .map(hit -> { - String source = hit.getSourceAsString(); - ExprValue docData = exprValueFactory.construct(source); - ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); - if (hit.getInnerHits() == null || hit.getInnerHits().isEmpty()) { - builder.putAll(docData.tupleValue()); - } else { - Map rowSource = hit.getSourceAsMap(); - builder.putAll(ExprValueUtils.tupleValue(rowSource).tupleValue()); - } - - metaDataFieldSet.forEach(metaDataField -> { - if (metaDataField.equals(METADATA_FIELD_INDEX)) { - builder.put(METADATA_FIELD_INDEX, new ExprStringValue(hit.getIndex())); - } else if (metaDataField.equals(METADATA_FIELD_ID)) { - builder.put(METADATA_FIELD_ID, new ExprStringValue(hit.getId())); - } else if (metaDataField.equals(METADATA_FIELD_SCORE)) { - if (!Float.isNaN(hit.getScore())) { - builder.put(METADATA_FIELD_SCORE, new ExprFloatValue(hit.getScore())); - } - } else if (metaDataField.equals(METADATA_FIELD_MAXSCORE)) { - if (maxScore != null) { - builder.put(METADATA_FIELD_MAXSCORE, maxScore); - } - } else { // if (metaDataField.equals(METADATA_FIELD_SORT)) { - builder.put(METADATA_FIELD_SORT, new ExprLongValue(hit.getSeqNo())); - } - }); - - if (!hit.getHighlightFields().isEmpty()) { - var hlBuilder = ImmutableMap.builder(); - for (var es : hit.getHighlightFields().entrySet()) { - hlBuilder.put(es.getKey(), ExprValueUtils.collectionValue( - Arrays.stream(es.getValue().fragments()).map( - t -> (t.toString())).collect(Collectors.toList()))); - } - builder.put("_highlight", ExprTupleValue.fromExprValueMap(hlBuilder.build())); - } + addParsedHitsToBuilder(builder, hit); + addMetaDataFieldsToBuilder(builder, hit); + addHighlightsToBuilder(builder, hit); return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); }).iterator(); } } + + /** + * Parse response for all hits to add to builder. Inner_hits supports arrays of objects + * with nested type. + * @param builder builder to build values from response. + * @param hit Search hit from response. + */ + private void addParsedHitsToBuilder( + ImmutableMap.Builder builder, + SearchHit hit + ) { + builder.putAll( + exprValueFactory.construct( + hit.getSourceAsString(), + !(hit.getInnerHits() == null || hit.getInnerHits().isEmpty()) + ).tupleValue()); + } + + /** + * If highlight fields are present in response add the fields to the builder. + * @param builder builder to build values from response. + * @param hit Search hit from response. + */ + private void addHighlightsToBuilder( + ImmutableMap.Builder builder, + SearchHit hit + ) { + if (!hit.getHighlightFields().isEmpty()) { + var hlBuilder = ImmutableMap.builder(); + for (var es : hit.getHighlightFields().entrySet()) { + hlBuilder.put(es.getKey(), ExprValueUtils.collectionValue( + Arrays.stream(es.getValue().fragments()).map( + Text::toString).collect(Collectors.toList()))); + } + builder.put("_highlight", ExprTupleValue.fromExprValueMap(hlBuilder.build())); + } + } + + /** + * Add metadata fields to builder from response. + * @param builder builder to build values from response. + * @param hit Search hit from response. + */ + private void addMetaDataFieldsToBuilder( + ImmutableMap.Builder builder, + SearchHit hit + ) { + List metaDataFieldSet = includes.stream() + .filter(METADATAFIELD_TYPE_MAP::containsKey) + .collect(Collectors.toList()); + ExprFloatValue maxScore = Float.isNaN(hits.getMaxScore()) + ? null : new ExprFloatValue(hits.getMaxScore()); + + metaDataFieldSet.forEach(metaDataField -> { + if (metaDataField.equals(METADATA_FIELD_INDEX)) { + builder.put(METADATA_FIELD_INDEX, new ExprStringValue(hit.getIndex())); + } else if (metaDataField.equals(METADATA_FIELD_ID)) { + builder.put(METADATA_FIELD_ID, new ExprStringValue(hit.getId())); + } else if (metaDataField.equals(METADATA_FIELD_SCORE)) { + if (!Float.isNaN(hit.getScore())) { + builder.put(METADATA_FIELD_SCORE, new ExprFloatValue(hit.getScore())); + } + } else if (metaDataField.equals(METADATA_FIELD_MAXSCORE)) { + if (maxScore != null) { + builder.put(METADATA_FIELD_MAXSCORE, maxScore); + } + } else { // if (metaDataField.equals(METADATA_FIELD_SORT)) { + builder.put(METADATA_FIELD_SORT, new ExprLongValue(hit.getSeqNo())); + } + }); + } + + /** + * Handle an aggregation response. + * @return Parsed and built return values from response. + */ + private Iterator handleAggregationResponse() { + return exprValueFactory.getParser().parse(aggregations).stream().map(entry -> { + ImmutableMap.Builder builder = new ImmutableMap.Builder<>(); + for (Map.Entry value : entry.entrySet()) { + builder.put( + value.getKey(), + exprValueFactory.construct( + value.getKey(), + value.getValue(), + false + ) + ); + } + return (ExprValue) ExprTupleValue.fromExprValueMap(builder.build()); + }).iterator(); + } } diff --git a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java index b327b73b86..9bdb15d63a 100644 --- a/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java +++ b/opensearch/src/main/java/org/opensearch/sql/opensearch/storage/script/core/ExpressionScript.java @@ -118,7 +118,11 @@ private Environment buildValueEnv( Map valueEnv = new HashMap<>(); for (ReferenceExpression field : fields) { String fieldName = field.getAttr(); - ExprValue exprValue = valueFactory.construct(fieldName, getDocValue(field, docProvider)); + ExprValue exprValue = valueFactory.construct( + fieldName, + getDocValue(field, docProvider), + false + ); valueEnv.put(field, exprValue); } // Encapsulate map data structure into anonymous Environment class diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java index 2de0214baa..6c400ff3ca 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchNodeClientTest.java @@ -315,7 +315,7 @@ void search() { 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); when(searchHit.getInnerHits()).thenReturn(null); - when(factory.construct(any())).thenReturn(exprTupleValue); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed SearchResponse scrollResponse = mock(SearchResponse.class); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java index 2877e04c14..e6b6be117a 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/client/OpenSearchRestClientTest.java @@ -13,6 +13,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Answers.RETURNS_DEEP_STUBS; +import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.Mockito.any; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.never; @@ -298,7 +299,7 @@ void search() throws IOException { 1.0F)); when(searchHit.getSourceAsString()).thenReturn("{\"id\", 1}"); when(searchHit.getInnerHits()).thenReturn(null); - when(factory.construct(any())).thenReturn(exprTupleValue); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue); // Mock second scroll request followed SearchResponse scrollResponse = mock(SearchResponse.class); diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java index 81ac39ede0..a7e3531e8b 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/data/value/OpenSearchExprValueFactoryTest.java @@ -12,6 +12,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.opensearch.sql.data.model.ExprValueUtils.booleanValue; import static org.opensearch.sql.data.model.ExprValueUtils.byteValue; +import static org.opensearch.sql.data.model.ExprValueUtils.collectionValue; import static org.opensearch.sql.data.model.ExprValueUtils.doubleValue; import static org.opensearch.sql.data.model.ExprValueUtils.floatValue; import static org.opensearch.sql.data.model.ExprValueUtils.integerValue; @@ -37,12 +38,12 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.ObjectMapper; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.time.Instant; import java.time.LocalDate; import java.time.LocalTime; import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import lombok.EqualsAndHashCode; import lombok.ToString; @@ -50,7 +51,6 @@ import org.opensearch.sql.data.model.ExprCollectionValue; import org.opensearch.sql.data.model.ExprDateValue; import org.opensearch.sql.data.model.ExprDatetimeValue; -import org.opensearch.sql.data.model.ExprStringValue; import org.opensearch.sql.data.model.ExprTimeValue; import org.opensearch.sql.data.model.ExprTimestampValue; import org.opensearch.sql.data.model.ExprTupleValue; @@ -92,6 +92,17 @@ class OpenSearchExprValueFactoryTest { .put("arrayV", OpenSearchDataType.of(ARRAY)) .put("arrayV.info", OpenSearchDataType.of(STRING)) .put("arrayV.author", OpenSearchDataType.of(STRING)) + .put("deepNestedV", OpenSearchDataType.of( + OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + ) + .put("deepNestedV.year", OpenSearchDataType.of( + OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + ) + .put("deepNestedV.year.timeV", OpenSearchDateType.of(TIME)) + .put("nestedV", OpenSearchDataType.of( + OpenSearchDataType.of(OpenSearchDataType.MappingType.Nested)) + ) + .put("nestedV.count", OpenSearchDataType.of(INTEGER)) .put("textV", OpenSearchDataType.of(OpenSearchDataType.MappingType.Text)) .put("textKeywordV", OpenSearchTextType.of(Map.of("words", OpenSearchDataType.of(OpenSearchDataType.MappingType.Keyword)))) @@ -381,7 +392,7 @@ public void constructDateFromUnsupportedFormat_ThrowIllegalArgumentException() { @Test public void constructArray() { assertEquals( - new ExprCollectionValue(ImmutableList.of(new ExprTupleValue( + new ExprCollectionValue(List.of(new ExprTupleValue( new LinkedHashMap() { { put("info", stringValue("zz")); @@ -390,22 +401,252 @@ public void constructArray() { }))), tupleValue("{\"arrayV\":[{\"info\":\"zz\",\"author\":\"au\"}]}").get("arrayV")); assertEquals( - new ExprCollectionValue(ImmutableList.of(new ExprTupleValue( + new ExprCollectionValue(List.of(new ExprTupleValue( new LinkedHashMap() { { put("info", stringValue("zz")); put("author", stringValue("au")); } }))), - constructFromObject("arrayV", ImmutableList.of( + constructFromObject("arrayV", List.of( ImmutableMap.of("info", "zz", "author", "au")))); } @Test public void constructArrayOfStrings() { assertEquals(new ExprCollectionValue( - ImmutableList.of(new ExprStringValue("zz"), new ExprStringValue("au"))), - constructFromObject("arrayV", ImmutableList.of("zz", "au"))); + List.of(stringValue("zz"), stringValue("au"))), + constructFromObject("arrayV", List.of("zz", "au"))); + } + + @Test + public void constructNestedArraysOfStrings() { + assertEquals( + new ExprCollectionValue( + List.of( + collectionValue( + List.of("zz", "au") + ), + collectionValue( + List.of("ss") + ) + ) + ), + tupleValueWithArraySupport( + "{\"stringV\":[" + + "[\"zz\", \"au\"]," + + "[\"ss\"]" + + "]}" + ).get("stringV")); + } + + @Test + public void constructNestedArraysOfStringsReturnsFirstIndex() { + assertEquals( + stringValue("zz"), + tupleValue( + "{\"stringV\":[" + + "[\"zz\", \"au\"]," + + "[\"ss\"]" + + "]}" + ).get("stringV")); + } + + @Test + public void constructMultiNestedArraysOfStringsReturnsFirstIndex() { + assertEquals( + stringValue("z"), + tupleValue( + "{\"stringV\":" + + "[\"z\"," + + "[\"s\"]," + + "[\"zz\", \"au\"]" + + "]}" + ).get("stringV")); + } + + @Test + public void constructArrayOfInts() { + assertEquals(new ExprCollectionValue( + List.of(integerValue(1), integerValue(2))), + constructFromObject("arrayV", List.of(1, 2))); + } + + @Test + public void constructArrayOfShorts() { + // Shorts are treated same as integer + assertEquals(new ExprCollectionValue( + List.of(shortValue((short)3), shortValue((short)4))), + constructFromObject("arrayV", List.of(3, 4))); + } + + @Test + public void constructArrayOfLongs() { + assertEquals(new ExprCollectionValue( + List.of(longValue(123456789L), longValue(987654321L))), + constructFromObject("arrayV", List.of(123456789L, 987654321L))); + } + + @Test + public void constructArrayOfFloats() { + assertEquals(new ExprCollectionValue( + List.of(floatValue(3.14f), floatValue(4.13f))), + constructFromObject("arrayV", List.of(3.14f, 4.13f))); + } + + @Test + public void constructArrayOfDoubles() { + assertEquals(new ExprCollectionValue( + List.of(doubleValue(9.1928374756D), doubleValue(4.987654321D))), + constructFromObject("arrayV", List.of(9.1928374756D, 4.987654321D))); + } + + @Test + public void constructArrayOfBooleans() { + assertEquals(new ExprCollectionValue( + List.of(booleanValue(true), booleanValue(false))), + constructFromObject("arrayV", List.of(true, false))); + } + + @Test + public void constructNestedObjectArrayNode() { + assertEquals(collectionValue( + List.of( + Map.of("count", 1), + Map.of("count", 2) + )), + tupleValueWithArraySupport("{\"nestedV\":[{\"count\":1},{\"count\":2}]}") + .get("nestedV")); + } + + @Test + public void constructNestedObjectArrayOfObjectArraysNode() { + assertEquals( + collectionValue( + List.of( + Map.of("year", + List.of( + Map.of("timeV", new ExprTimeValue("09:07:42")), + Map.of("timeV", new ExprTimeValue("09:07:42")) + ) + ), + Map.of("year", + List.of( + Map.of("timeV", new ExprTimeValue("09:07:42")), + Map.of("timeV", new ExprTimeValue("09:07:42")) + ) + ) + ) + ), + tupleValueWithArraySupport( + "{\"deepNestedV\":" + + "[" + + "{\"year\":" + + "[" + + "{\"timeV\":\"09:07:42\"}," + + "{\"timeV\":\"09:07:42\"}" + + "]" + + "}," + + "{\"year\":" + + "[" + + "{\"timeV\":\"09:07:42\"}," + + "{\"timeV\":\"09:07:42\"}" + + "]" + + "}" + + "]" + + "}") + .get("deepNestedV")); + } + + @Test + public void constructNestedArrayNode() { + assertEquals(collectionValue( + List.of( + 1969, + 2011 + )), + tupleValueWithArraySupport("{\"nestedV\":[1969,2011]}") + .get("nestedV")); + } + + @Test + public void constructNestedObjectNode() { + assertEquals(collectionValue( + List.of( + Map.of("count", 1969) + )), + tupleValue("{\"nestedV\":{\"count\":1969}}") + .get("nestedV")); + } + + @Test + public void constructArrayOfGeoPoints() { + assertEquals(new ExprCollectionValue( + List.of( + new OpenSearchExprGeoPointValue(42.60355556, -97.25263889), + new OpenSearchExprGeoPointValue(-33.6123556, 66.287449)) + ), + tupleValueWithArraySupport( + "{\"geoV\":[" + + "{\"lat\":42.60355556,\"lon\":-97.25263889}," + + "{\"lat\":-33.6123556,\"lon\":66.287449}" + + "]}" + ).get("geoV") + ); + } + + @Test + public void constructArrayOfIPsReturnsFirstIndex() { + assertEquals( + new OpenSearchExprIpValue("192.168.0.1"), + tupleValue("{\"ipV\":[\"192.168.0.1\",\"192.168.0.2\"]}") + .get("ipV") + ); + } + + @Test + public void constructBinaryArrayReturnsFirstIndex() { + assertEquals( + new OpenSearchExprBinaryValue("U29tZSBiaWsdfsdfgYmxvYg=="), + tupleValue("{\"binaryV\":[\"U29tZSBiaWsdfsdfgYmxvYg==\",\"U987yuhjjiy8jhk9vY+98jjdf\"]}") + .get("binaryV") + ); + } + + @Test + public void constructArrayOfCustomEpochMillisReturnsFirstIndex() { + assertEquals( + new ExprDatetimeValue("2015-01-01 12:10:30"), + tupleValue("{\"customAndEpochMillisV\":[\"2015-01-01 12:10:30\",\"1999-11-09 01:09:44\"]}") + .get("customAndEpochMillisV") + ); + } + + @Test + public void constructArrayOfDateStringsReturnsFirstIndex() { + assertEquals( + new ExprDateValue("1984-04-12"), + tupleValue("{\"dateStringV\":[\"1984-04-12\",\"2033-05-03\"]}") + .get("dateStringV") + ); + } + + @Test + public void constructArrayOfTimeStringsReturnsFirstIndex() { + assertEquals( + new ExprTimeValue("12:10:30"), + tupleValue("{\"timeStringV\":[\"12:10:30.000Z\",\"18:33:55.000Z\"]}") + .get("timeStringV") + ); + } + + @Test + public void constructArrayOfEpochMillis() { + assertEquals( + new ExprTimestampValue(Instant.ofEpochMilli(1420070400001L)), + tupleValue("{\"dateOrEpochMillisV\":[\"1420070400001\",\"1454251113333\"]}") + .get("dateOrEpochMillisV") + ); } @Test @@ -517,13 +758,19 @@ public void noTypeFoundForMapping() { @Test public void constructUnsupportedTypeThrowException() { OpenSearchExprValueFactory exprValueFactory = - new OpenSearchExprValueFactory(ImmutableMap.of("type", new TestType())); + new OpenSearchExprValueFactory(Map.of("type", new TestType())); IllegalStateException exception = - assertThrows(IllegalStateException.class, () -> exprValueFactory.construct("{\"type\":1}")); + assertThrows( + IllegalStateException.class, + () -> exprValueFactory.construct("{\"type\":1}", false) + ); assertEquals("Unsupported type: TEST_TYPE for value: 1.", exception.getMessage()); exception = - assertThrows(IllegalStateException.class, () -> exprValueFactory.construct("type", 1)); + assertThrows( + IllegalStateException.class, + () -> exprValueFactory.construct("type", 1, false) + ); assertEquals( "Unsupported type: TEST_TYPE for value: 1.", exception.getMessage()); @@ -553,12 +800,21 @@ public void factoryMappingsAreExtendableWithoutOverWrite() } public Map tupleValue(String jsonString) { - final ExprValue construct = exprValueFactory.construct(jsonString); + final ExprValue construct = exprValueFactory.construct(jsonString, false); + return construct.tupleValue(); + } + + public Map tupleValueWithArraySupport(String jsonString) { + final ExprValue construct = exprValueFactory.construct(jsonString, true); return construct.tupleValue(); } private ExprValue constructFromObject(String fieldName, Object value) { - return exprValueFactory.construct(fieldName, value); + return exprValueFactory.construct(fieldName, value, false); + } + + private ExprValue constructFromObjectWithArraySupport(String fieldName, Object value) { + return exprValueFactory.construct(fieldName, value, true); } @EqualsAndHashCode(callSuper = false) diff --git a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java index 079a82b783..05e5d80c39 100644 --- a/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java +++ b/opensearch/src/test/java/org/opensearch/sql/opensearch/response/OpenSearchResponseTest.java @@ -12,7 +12,11 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyBoolean; +import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.ArgumentMatchers.eq; +import static org.mockito.Mockito.lenient; import static org.mockito.Mockito.when; import com.google.common.collect.ImmutableMap; @@ -114,7 +118,8 @@ void iterator() { when(searchHit2.getSourceAsString()).thenReturn("{\"id1\", 2}"); when(searchHit1.getInnerHits()).thenReturn(null); when(searchHit2.getInnerHits()).thenReturn(null); - when(factory.construct(any())).thenReturn(exprTupleValue1).thenReturn(exprTupleValue2); + when(factory.construct(any(), anyBoolean())) + .thenReturn(exprTupleValue1).thenReturn(exprTupleValue2); int i = 0; for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, List.of("id1"))) { @@ -149,7 +154,7 @@ void iterator_metafields() { when(searchHit1.getScore()).thenReturn(3.75F); when(searchHit1.getSeqNo()).thenReturn(123456L); - when(factory.construct(any())).thenReturn(exprTupleHit); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleHit); ExprTupleValue exprTupleResponse = ExprTupleValue.fromExprValueMap(ImmutableMap.of( "id1", new ExprIntegerValue(1), @@ -187,7 +192,7 @@ void iterator_metafields_withoutIncludes() { when(searchHit1.getSourceAsString()).thenReturn("{\"id1\", 1}"); - when(factory.construct(any())).thenReturn(exprTupleHit); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleHit); List includes = List.of("id1"); ExprTupleValue exprTupleResponse = ExprTupleValue.fromExprValueMap(ImmutableMap.of( @@ -224,7 +229,7 @@ void iterator_metafields_scoreNaN() { when(searchHit1.getScore()).thenReturn(Float.NaN); when(searchHit1.getSeqNo()).thenReturn(123456L); - when(factory.construct(any())).thenReturn(exprTupleHit); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleHit); List includes = List.of("id1", "_index", "_id", "_sort", "_score", "_maxscore"); ExprTupleValue exprTupleResponse = ExprTupleValue.fromExprValueMap(ImmutableMap.of( @@ -252,8 +257,6 @@ void iterator_with_inner_hits() { new SearchHit[] {searchHit1}, new TotalHits(2L, TotalHits.Relation.EQUAL_TO), 1.0F)); - when(searchHit1.getSourceAsString()).thenReturn("{\"id1\", 1}"); - when(searchHit1.getSourceAsMap()).thenReturn(Map.of("id1", 1)); when(searchHit1.getInnerHits()).thenReturn( Map.of( "innerHit", @@ -262,7 +265,7 @@ void iterator_with_inner_hits() { new TotalHits(2L, TotalHits.Relation.EQUAL_TO), 1.0F))); - when(factory.construct(any())).thenReturn(exprTupleValue1); + when(factory.construct(any(), anyBoolean())).thenReturn(exprTupleValue1); for (ExprValue hit : new OpenSearchResponse(searchResponse, factory, includes)) { assertEquals(exprTupleValue1, hit); @@ -293,7 +296,7 @@ void aggregation_iterator() { .thenReturn(Arrays.asList(ImmutableMap.of("id1", 1), ImmutableMap.of("id2", 2))); when(searchResponse.getAggregations()).thenReturn(aggregations); when(factory.getParser()).thenReturn(parser); - when(factory.construct(anyString(), any())) + when(factory.construct(anyString(), anyInt(), anyBoolean())) .thenReturn(new ExprIntegerValue(1)) .thenReturn(new ExprIntegerValue(2)); @@ -329,7 +332,7 @@ void highlight_iterator() { 1.0F)); when(searchHit1.getHighlightFields()).thenReturn(highlightMap); - when(factory.construct(any())).thenReturn(resultTuple); + when(factory.construct(any(), anyBoolean())).thenReturn(resultTuple); for (ExprValue resultHit : new OpenSearchResponse(searchResponse, factory, includes)) { var expected = ExprValueUtils.collectionValue(