From 220dc24e6f2e689b9444c39b024778a78ad0be81 Mon Sep 17 00:00:00 2001 From: Mingshi Liu Date: Fri, 9 Jun 2023 16:06:04 -0700 Subject: [PATCH] Enable Partial Flat Object Signed-off-by: Mingshi Liu --- .../test/index/100_partial_flat_object.yml | 539 ++++++++++++++++++ .../index/mapper/FlatObjectFieldMapper.java | 57 +- 2 files changed, 583 insertions(+), 13 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml new file mode 100644 index 0000000000000..9dd26d922737d --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/100_partial_flat_object.yml @@ -0,0 +1,539 @@ +--- +# The test setup includes: +# - Create flat_object mapping for test_partial_flat_object index +# - Index two example documents +# - Refresh the index so it is ready for search tests + +setup: + - do: + indices.create: + index: test_partial_flat_object + body: + mappings: + properties: + issue: + properties: + number: + type : "integer" + labels: + type : "flat_object" + - do: + index: + index: test_partial_flat_object + id: 1 + body: { + "issue": { + "number": 123, + "labels": { + "version": "2.2", + "backport": [ + "2.0", + "1.9" + ], + "category": { + "type": "API", + "level": "bug" + }, + "createdDate": "2023-01-01", + "comment" : [["Doe","Shipped"],["John","Approved"]], + "views": 288, + "priority": 5.00 + } + } + } + + - do: + index: + index: test_partial_flat_object + id: 2 + body: { + "issue": { + "number": 456, + "labels": { + "version": "2.1", + "backport": [ + "2.0", + "1.3" + ], + "category": { + "type": "API", + "level": "enhancement" + }, + "createdDate": "2023-02-01", + "comment" : [["Mike","LGTM"],["John","Approved"]], + "views": 3333, + "priority": 1.50 + } + } + } + - do: + indices.refresh: + index: test_partial_flat_object +--- +# Delete Index when connection is teardown +teardown: + - do: + indices.delete: + index: test_partial_flat_object + + +--- +# Verify that mappings under the catalog field did not expand +# and no dynamic fields were created. +"Mappings": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + - do: + indices.get_mapping: + index: test_partial_flat_object + - is_true: test_partial_flat_object.mappings + - match: { test_partial_flat_object.mappings.properties.issue.properties.number.type: integer } + - match: { test_partial_flat_object.mappings.properties.issue.properties.labels.type: flat_object } + # https://github.com/opensearch-project/OpenSearch/tree/main/rest-api-spec/src/main/resources/rest-api-spec/test#length + - length: { test_partial_flat_object.mappings.properties.issue.properties: 2 } + - length: { test_partial_flat_object.mappings.properties.issue.properties.labels: 1 } + + +--- +"Supported queries": + - skip: + version: " - 2.99.99" + reason: "flat_object is introduced in 3.0.0 in main branch" + + + # Verify Document Count + - do: + search: + body: { + query: { + match_all: {} + } + } + + - length: { hits.hits: 2 } + + # Match Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + match: { "issue.labels.version": "2.1" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.version: "2.1" } + + # Match Query without exact dot path. + - do: + search: + body: { + _source: true, + query: { + match: { issue.labels: "2.1" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.version: "2.1" } + + # Multi Match Query with exact dot path. + - do: + search: + body: { + _source: true, + query: { + multi_match: { + "query": "2.0", + "fields": [ "issue.labels.version", "issue.labels.backport" ] + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9"] } + + # Term Query1 with exact dot path for date + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.createdDate: "2023-01-01" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.createdDate: "2023-01-01" } + + # Term Query1 without exact dot path for date + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "2023-01-01" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.createdDate: "2023-01-01" } + + + # Term Query2 with dot path for string + - do: + search: + body: { + _source: true, + query: { + term: { "issue.labels.category.type": "API" } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.category.type: "API" } + + # Term Query2 without exact dot path. + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "API" } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.category.type: "API" } + + # Term Query3 with dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.backport: "1.9" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9"]} + + # Term Query3 without dot path for array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "1.9" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.backport: [ "2.0", "1.9"]} + + # Term Query4 with dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels.comment: "LGTM" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]]} + + # Term Query4 without dot path for nested-array + - do: + search: + body: { + _source: true, + query: { + term: { issue.labels: "LGTM" } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]] } + + # Terms Query without dot path. + - do: + search: + body: { + _source: true, + query: { + terms: { issue.labels: [ "John","Mike" ] } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]] } + + # Terms Query with dot path. + - do: + search: + body: { + _source: true, + query: { + terms: { issue.labels.comment: ["John","Mike"] } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]] } + + # Prefix Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "prefix": { + "issue.labels.comment": { + "value": "Mi" + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]] } + + # Prefix Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "prefix": { + "issue.labels": { + "value": "Mi" + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]] } + + # Range Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.version": { + "gte": "2.1", + "lte": "3.0" + } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.version: "2.2" } + + # Range Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": "2.1", + "lte": "3.0" + } + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.version: "2.2" } + + # Range Query with integer input with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.views": { + "gte": 3000, + "lte": 4000 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.views: 3333 } + + # Range Query with integer input without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": 3000, + "lte": 4000 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.views: 3333 } + + + # Range Query with double input with dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels.priority": { + "gte": 4.1234, + "lte": 5.1234 + } + } + } + } + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.priority: 5.00 } + + # Range Query with double input without dot path. + - do: + search: + body: { + _source: true, + query: { + "range": { + "issue.labels": { + "gte": 4.1234, + "lte": 5.1234 + } + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.priority: 5.00 } + + + # Exists Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.priority + } + } + } + + - length: { hits.hits: 2 } + + # Exists Query with nested dot path, use the flat_object_field_name.last_key + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels.type + } + } + } + + - length: { hits.hits: 2 } + + # Exists Query without dot path for the flat_object_field_name + - do: + search: + body: { + _source: true, + query: { + "exists": { + "field": issue.labels + } + } + } + + - length: { hits.hits: 2 } + + # Query_string Query without dot path. + - do: + search: + body: { + _source: true, + query: { + "query_string": { + "fields": [ "issue.labels"], + "query": "Doe OR Mike" + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]]} + - match: { hits.hits.1._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]]} + + # Query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "query_string": { + "fields": [ "issue.labels.comment" ], + "query": "Doe OR Mike" + } + } + } + + - length: { hits.hits: 2 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]]} + - match: { hits.hits.1._source.issue.labels.comment: [["Mike","LGTM"],["John","Approved"]]} + + # Simple_query_string Query without full dot path. + - do: + search: + body: { + _source: true, + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels" ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]] } + + # Simple_query_string Query with dot path. + - do: + search: + body: { + _source: true, + query: { + "simple_query_string": { + "query": "Doe", + "fields": [ "issue.labels.comment" ] + } + } + } + + - length: { hits.hits: 1 } + - match: { hits.hits.0._source.issue.labels.comment: [["Doe","Shipped"],["John","Approved"]] } diff --git a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java index e0b37df5c1734..c30b16d69f72c 100644 --- a/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/FlatObjectFieldMapper.java @@ -85,7 +85,7 @@ public static class Defaults { @Override public MappedFieldType keyedFieldType(String key) { - return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key); + return new FlatObjectFieldType(this.name() + DOT_SYMBOL + key, this.name()); } /** @@ -186,6 +186,8 @@ public static final class FlatObjectFieldType extends StringFieldType { private final int ignoreAbove; private final String nullValue; + private final String mappedFieldTypeName; + private KeywordFieldMapper.KeywordFieldType valueFieldType; private KeywordFieldMapper.KeywordFieldType valueAndPathFieldType; @@ -195,10 +197,7 @@ public FlatObjectFieldType(String name, boolean isSearchable, boolean hasDocValu setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; - } - - public FlatObjectFieldType(String name) { - this(name, true, true, Collections.emptyMap()); + this.mappedFieldTypeName = null; } public FlatObjectFieldType(String name, FieldType fieldType) { @@ -212,12 +211,28 @@ public FlatObjectFieldType(String name, FieldType fieldType) { ); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; + this.mappedFieldTypeName = null; } public FlatObjectFieldType(String name, NamedAnalyzer analyzer) { super(name, true, false, true, new TextSearchInfo(Defaults.FIELD_TYPE, null, analyzer, analyzer), Collections.emptyMap()); this.ignoreAbove = Integer.MAX_VALUE; this.nullValue = null; + this.mappedFieldTypeName = null; + } + + public FlatObjectFieldType(String name, String mappedFieldTypeName) { + super( + name, + true, + false, + true, + new TextSearchInfo(Defaults.FIELD_TYPE, null, Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER), + Collections.emptyMap() + ); + this.ignoreAbove = Integer.MAX_VALUE; + this.nullValue = null; + this.mappedFieldTypeName = mappedFieldTypeName; } void setValueFieldType(KeywordFieldMapper.KeywordFieldType valueFieldType) { @@ -356,11 +371,10 @@ public Query termsQuery(List values, QueryShardContext context) { * @return directedSubFieldName */ public String directSubfield() { - if (name().contains(DOT_SYMBOL)) { - String[] dotPathList = name().split("\\."); - return dotPathList[0] + VALUE_AND_PATH_SUFFIX; + if (mappedFieldTypeName == null) { + return new StringBuilder().append(this.name()).append(VALUE_SUFFIX).toString(); } else { - return this.valueFieldType.name(); + return new StringBuilder().append(this.mappedFieldTypeName).append(VALUE_AND_PATH_SUFFIX).toString(); } } @@ -371,7 +385,7 @@ public String directSubfield() { * @return rewriteSearchValue */ public String rewriteValue(String searchValueString) { - if (!name().contains(DOT_SYMBOL)) { + if (!hasDotPath(name())) { return searchValueString; } else { String rewriteSearchValue = new StringBuilder().append(name()).append(EQUAL_SYMBOL).append(searchValueString).toString(); @@ -380,6 +394,23 @@ public String rewriteValue(String searchValueString) { } + private boolean hasDotPath(String input) { + String prefix = this.mappedFieldTypeName; + if (prefix == null) { + return false; + } + if (!input.startsWith(prefix)) { + return false; + } + String rest = input.substring(prefix.length()); + + if (rest.isEmpty()) { + return false; + } else { + return true; + } + } + private String inputToString(Object inputValue) { if (inputValue instanceof Integer) { String inputToString = Integer.toString((Integer) inputValue); @@ -460,15 +491,15 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } /** - * if there is dot path. query the field name in flatObject parent field. + * if there is dot path. query the field name in flatObject parent field (mappedFieldTypeName). * else query in _field_names system field */ @Override public Query existsQuery(QueryShardContext context) { String searchKey; String searchField; - if (name().contains(DOT_SYMBOL)) { - searchKey = name().split("\\.")[0]; + if (hasDotPath(name())) { + searchKey = this.mappedFieldTypeName; searchField = name(); } else { searchKey = FieldNamesFieldMapper.NAME;