From aa9ab72636efab72c48ce0f4563a9b0a08f08e37 Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Wed, 26 Jun 2024 14:13:33 +0800 Subject: [PATCH 1/9] The dynamic mapping parameter supports strict_allow_templates Signed-off-by: Gao Binlong --- CHANGELOG.md | 1 + .../test/index/110_strict_allow_templates.yml | 155 ++++++++ .../indices.put_mapping/all_path_options.yml | 31 ++ .../index/mapper/DocumentParser.java | 63 ++-- .../opensearch/index/mapper/ObjectMapper.java | 20 +- .../index/mapper/RootObjectMapper.java | 36 +- .../mapper/StrictDynamicMappingException.java | 4 +- .../index/mapper/CopyToMapperTests.java | 40 +++ .../index/mapper/DocumentParserTests.java | 334 ++++++++++++++++++ 9 files changed, 650 insertions(+), 34 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index f71ba46745ef1..664fb06d28b91 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865)) - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782)) - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445)) +- The dynamic mapping parameter supports strict_allow_templates ### Dependencies - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442)) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml new file mode 100644 index 0000000000000..623cb97c37728 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml @@ -0,0 +1,155 @@ +--- +"Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index": + - skip: + version: " - 2.15.99" + reason: "introduced in 2.16.0" + + - do: + indices.create: + index: test_1 + body: + mappings: + dynamic: strict_allow_templates + dynamic_templates: [ + { + strings: { + "match": "stringField*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + }, + { + object: { + "match": "objectField*", + "match_mapping_type": "object", + "mapping": { + "type": "object", + "properties": { + "bar1": { + "type": "keyword" + }, + "bar2": { + "type": "text" + } + } + } + } + }, + { + boolean: { + "match": "booleanField*", + "match_mapping_type": "boolean", + "mapping": { + "type": "boolean" + } + } + }, + { + double: { + "match": "doubleField*", + "match_mapping_type": "double", + "mapping": { + "type": "double" + } + } + }, + { + long: { + "match": "longField*", + "match_mapping_type": "long", + "mapping": { + "type": "long" + } + } + }, + { + array: { + "match": "arrayField*", + "mapping": { + "type": "keyword" + } + } + }, + { + date: { + "match": "dateField*", + "match_mapping_type": "date", + "mapping": { + "type": "date" + } + } + } + ] + properties: + test1: + type: text + + - do: + catch: /mapping set to strict_allow_templates, dynamic introduction of \[test2\] within \[\_doc\] is not allowed/ + index: + index: test_1 + id: 1 + body: { + stringField: bar, + objectField: { + bar1: "bar1", + bar2: "bar2" + }, + test1: test1, + test2: test2 + } + + - do: + index: + index: test_1 + id: 1 + body: { + stringField: bar, + objectField: { + bar1: "bar1", + bar2: "bar2" + }, + booleanField: true, + doubleField: 1.0, + longField: 100, + arrayField: ["1","2"], + dateField: "2024-06-25T05:11:51.243Z", + test1: test1 + } + + - do: + get: + index: test_1 + id: 1 + - match: { _source: { + stringField: bar, + objectField: { + bar1: "bar1", + bar2: "bar2" + }, + booleanField: true, + doubleField: 1.0, + longField: 100, + arrayField: [ "1","2" ], + dateField: "2024-06-25T05:11:51.243Z", + test1: test1 + } + } + + - do: + indices.get_mapping: { + index: test_1 + } + + - match: {test_1.mappings.dynamic: strict_allow_templates} + - match: {test_1.mappings.properties.stringField.type: keyword} + - match: {test_1.mappings.properties.objectField.properties.bar1.type: keyword} + - match: {test_1.mappings.properties.objectField.properties.bar2.type: text} + - match: {test_1.mappings.properties.booleanField.type: boolean} + - match: {test_1.mappings.properties.doubleField.type: double} + - match: {test_1.mappings.properties.longField.type: long} + - match: {test_1.mappings.properties.arrayField.type: keyword} + - match: {test_1.mappings.properties.dateField.type: date} + - match: {test_1.mappings.properties.test1.type: text} diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml index ca7a21df20ea4..89b47fde2a72c 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml @@ -159,3 +159,34 @@ setup: indices.get_mapping: {} - match: {test_index1.mappings.properties.text.type: text} + +--- +"post a mapping with setting dynamic to strict_allow_templates": + - skip: + version: " - 2.15.99" + reason: "introduced in 2.16.0" + - do: + indices.put_mapping: + index: test_index1 + body: + dynamic: strict_allow_templates + dynamic_templates: [ + { + strings: { + "match": "foo*", + "match_mapping_type": "string", + "mapping": { + "type": "keyword" + } + } + } + ] + properties: + test1: + type: text + + - do: + indices.get_mapping: {} + + - match: {test_index1.mappings.dynamic: strict_allow_templates} + - match: {test_index1.mappings.properties.test1.type: text} diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index f276d6ee2e579..91e94ff534c97 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -546,9 +546,11 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, ObjectMapper parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(mapper.fullPath(), currentFieldName); - } else if (dynamic == ObjectMapper.Dynamic.TRUE) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT); + throw new StrictDynamicMappingException(dynamic.toString(), mapper.fullPath(), currentFieldName); + } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT, dynamic, mapper.fullPath()); + if (builder == null) { builder = new ObjectMapper.Builder(currentFieldName).enabled(true); } @@ -592,9 +594,10 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(parentMapper.fullPath(), arrayFieldName); - } else if (dynamic == ObjectMapper.Dynamic.TRUE) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT); + throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), arrayFieldName); + } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT, dynamic, parentMapper.fullPath()); if (builder == null) { parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); } else { @@ -695,9 +698,10 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp if (mapper != null) { // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); - } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(parentMapper.fullPath(), lastFieldName); - } + } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT + || parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + throw new StrictDynamicMappingException(parentMapper.dynamic().toString(), parentMapper.fullPath(), lastFieldName); + } } private static Mapper.Builder newLongBuilder(String name, Settings settings) { @@ -711,7 +715,9 @@ private static Mapper.Builder newFloatBuilder(String name, Settings settings) private static Mapper.Builder createBuilderFromDynamicValue( final ParseContext context, XContentParser.Token token, - String currentFieldName + String currentFieldName, + ObjectMapper.Dynamic dynamic, + String fullPath ) throws IOException { if (token == XContentParser.Token.VALUE_STRING) { String text = context.parser().text(); @@ -733,13 +739,15 @@ private static Mapper.Builder createBuilderFromDynamicValue( } if (parseableAsLong && context.root().numericDetection()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } else if (parseableAsDouble && context.root().numericDetection()) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); } @@ -755,7 +763,8 @@ private static Mapper.Builder createBuilderFromDynamicValue( // failure to parse this, continue continue; } - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, dateTimeFormatter); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath); if (builder == null) { boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); builder = new DateFieldMapper.Builder( @@ -771,7 +780,8 @@ private static Mapper.Builder createBuilderFromDynamicValue( } } - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder == null) { builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) @@ -783,7 +793,8 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG || numberType == XContentParser.NumberType.BIG_INTEGER) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } @@ -791,7 +802,8 @@ private static Mapper.Builder createBuilderFromDynamicValue( } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE || numberType == XContentParser.NumberType.BIG_DECIMAL) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of @@ -801,19 +813,22 @@ private static Mapper.Builder createBuilderFromDynamicValue( return builder; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath); if (builder == null) { builder = new BooleanFieldMapper.Builder(currentFieldName); } return builder; } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath); if (builder == null) { builder = new BinaryFieldMapper.Builder(currentFieldName); } return builder; } else { - Mapper.Builder builder = context.root().findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder != null) { return builder; } @@ -832,13 +847,13 @@ private static void parseDynamicValue( ) throws IOException { ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(parentMapper.fullPath(), currentFieldName); + throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), currentFieldName); } if (dynamic == ObjectMapper.Dynamic.FALSE) { return; } final Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); - final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName); + final Mapper.Builder builder = createBuilderFromDynamicValue(context, token, currentFieldName, dynamic, parentMapper.fullPath()); Mapper mapper = builder.build(builderContext); context.addDynamicMapper(mapper); @@ -926,9 +941,11 @@ private static Tuple getDynamicParentMapper( switch (dynamic) { case STRICT: - throw new StrictDynamicMappingException(parent.fullPath(), paths[i]); + throw new StrictDynamicMappingException(dynamic.toString(), parent.fullPath(), paths[i]); + case STRICT_ALLOW_TEMPLATES: case TRUE: - Mapper.Builder builder = context.root().findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT); + Mapper.Builder builder = context.root() + .findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT, dynamic, parent.fullPath()); if (builder == null) { builder = new ObjectMapper.Builder(paths[i]).enabled(true); } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 92ffdb60e6cde..1d0234b8e07b8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -88,9 +88,21 @@ public static class Defaults { */ @PublicApi(since = "1.0.0") public enum Dynamic { - TRUE, - FALSE, - STRICT + TRUE("true"), + FALSE("false"), + STRICT("strict"), + STRICT_ALLOW_TEMPLATES("strict_allow_templates"); + + private final String name; + + Dynamic(String name) { + this.name = name; + } + + @Override + public String toString() { + return this.name; + } } /** @@ -283,6 +295,8 @@ protected static boolean parseObjectOrDocumentTypeProperties( String value = fieldNode.toString(); if (value.equalsIgnoreCase("strict")) { builder.dynamic(Dynamic.STRICT); + } else if (value.equalsIgnoreCase("strict_allow_templates")) { + builder.dynamic(Dynamic.STRICT_ALLOW_TEMPLATES); } else { boolean dynamic = XContentMapValues.nodeBooleanValue(fieldNode, fieldName + ".dynamic"); builder.dynamic(dynamic ? Dynamic.TRUE : Dynamic.FALSE); diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 9504e6eafc046..6f72d38574fa2 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -301,12 +301,24 @@ public DynamicTemplate[] dynamicTemplates() { } @SuppressWarnings("rawtypes") - public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { - return findTemplateBuilder(context, name, matchType, null); + public Mapper.Builder findTemplateBuilder( + ParseContext context, + String name, + XContentFieldType matchType, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { + return findTemplateBuilder(context, name, matchType, null, dynamic, fieldFullPath); } - public Mapper.Builder findTemplateBuilder(ParseContext context, String name, DateFormatter dateFormatter) { - return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter); + public Mapper.Builder findTemplateBuilder( + ParseContext context, + String name, + DateFormatter dateFormatter, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { + return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter, dynamic, fieldFullPath); } /** @@ -314,14 +326,26 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat * @param name the field name * @param matchType the type of the field in the json document or null if unknown * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format + * @param dynamic the type of dynamic mapping + * @param fieldFullPath the field's full path * @return a mapper builder, or null if there is no template for such a field */ @SuppressWarnings("rawtypes") - private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { + public Mapper.Builder findTemplateBuilder( + ParseContext context, + String name, + XContentFieldType matchType, + DateFormatter dateFormat, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); - if (dynamicTemplate == null) { + if (dynamicTemplate == null && dynamic == Dynamic.STRICT_ALLOW_TEMPLATES) { + throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); + } else if (dynamicTemplate == null) { return null; } + String dynamicType = matchType.defaultMappingType(); Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(dateFormat); String mappingType = dynamicTemplate.mappingType(dynamicType); diff --git a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java index 9127641128dad..0524c672011c5 100644 --- a/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java +++ b/server/src/main/java/org/opensearch/index/mapper/StrictDynamicMappingException.java @@ -43,8 +43,8 @@ */ public class StrictDynamicMappingException extends MapperParsingException { - public StrictDynamicMappingException(String path, String fieldName) { - super("mapping set to strict, dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed"); + public StrictDynamicMappingException(String dynamic, String path, String fieldName) { + super("mapping set to " + dynamic + ", dynamic introduction of [" + fieldName + "] within [" + path + "] is not allowed"); } public StrictDynamicMappingException(StreamInput in) throws IOException { diff --git a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java index b274cf28429e8..7a8c4ffe35021 100644 --- a/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/CopyToMapperTests.java @@ -247,6 +247,46 @@ public void testCopyToStrictDynamicInnerObjectParsing() throws Exception { assertThat(e.getMessage(), startsWith("mapping set to strict, dynamic introduction of [very] within [_doc] is not allowed")); } + public void testCopyToStrictAllowTemplatesDynamicInnerObjectParsing() throws Exception { + DocumentMapper docMapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + b.startObject("properties"); + { + b.startObject("copy_test"); + { + b.field("type", "text"); + b.field("copy_to", "very.inner.field"); + } + b.endObject(); + } + b.endObject(); + })); + + MapperParsingException e = expectThrows( + MapperParsingException.class, + () -> docMapper.parse(source(b -> b.field("copy_test", "foo"))) + ); + + assertThat( + e.getMessage(), + startsWith("mapping set to strict_allow_templates, dynamic introduction of [very] within [_doc] is not allowed") + ); + } + public void testCopyToInnerStrictDynamicInnerObjectParsing() throws Exception { DocumentMapper docMapper = createDocumentMapper(mapping(b -> { diff --git a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java index ecab9da8c6b6c..15e2b6649b0be 100644 --- a/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java +++ b/server/src/test/java/org/opensearch/index/mapper/DocumentParserTests.java @@ -878,6 +878,340 @@ public void testDynamicStrictDottedFieldNameLong() throws Exception { assertEquals("mapping set to strict, dynamic introduction of [foo] within [_doc] is not allowed", exception.getMessage()); } + public void testDynamicStrictAllowTemplatesDottedFieldNameLong() throws Exception { + DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates"))); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapper.parse(source(b -> b.field("foo.bar.baz", 0))) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("path_match", "foo.bar.baz"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapperWithDynamicTemplates.parse(source(b -> b.field("foo.bar.baz", 0))) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "bar"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test2"); + { + b.field("path_match", "foo.bar.baz"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + ParsedDocument doc = mapper.parse(source(b -> b.field("foo.bar.baz", 0))); + assertEquals(2, doc.rootDoc().getFields("foo.bar.baz").length); + } + + public void testDynamicAllowTemplatesStrictLongArray() throws Exception { + DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates"))); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapperWithDynamicTemplates.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + ParsedDocument doc = mapper.parse(source(b -> b.startArray("foo").value(0).value(1).endArray())); + assertEquals(4, doc.rootDoc().getFields("foo").length); + } + + public void testDynamicStrictAllowTemplatesDottedFieldNameObject() throws Exception { + DocumentMapper documentMapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates"))); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper documentMapperWithDynamicTemplates = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + exception = expectThrows( + StrictDynamicMappingException.class, + () -> documentMapperWithDynamicTemplates.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "foo"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "bar"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test2"); + { + b.field("match", "baz"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test3"); + { + b.field("path_match", "foo.bar.baz.a"); + b.startObject("mapping").field("type", "long").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + })); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); + assertEquals(2, doc.rootDoc().getFields("foo.bar.baz.a").length); + } + + public void testDynamicStrictAllowTemplatesObject() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test"); + b.field("match_mapping_type", "object"); + b.startObject("mapping").field("type", "object").endObject(); + } + b.endObject(); + } + b.endObject(); + } + { + b.startObject(); + { + b.startObject("test1"); + { + b.field("match", "test1"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + } + + )); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> mapper.parse(source(b -> b.startObject("foo").field("bar", "baz").endObject())) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [foo] within [_doc] is not allowed", + exception.getMessage() + ); + + ParsedDocument doc = mapper.parse(source(b -> b.startObject("test").field("test1", "baz").endObject())); + assertEquals(2, doc.rootDoc().getFields("test.test1").length); + } + + public void testDynamicStrictAllowTemplatesValue() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> { + b.field("dynamic", "strict_allow_templates"); + b.startArray("dynamic_templates"); + { + b.startObject(); + { + b.startObject("test"); + { + b.field("match", "test*"); + b.field("match_mapping_type", "string"); + b.startObject("mapping").field("type", "keyword").endObject(); + } + b.endObject(); + } + b.endObject(); + } + b.endArray(); + } + + )); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> mapper.parse(source(b -> b.field("bar", "baz"))) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [bar] within [_doc] is not allowed", + exception.getMessage() + ); + + ParsedDocument doc = mapper.parse(source(b -> b.field("test1", "baz"))); + assertEquals(2, doc.rootDoc().getFields("test1").length); + } + + public void testDynamicStrictAllowTemplatesNull() throws Exception { + DocumentMapper mapper = createDocumentMapper(topMapping(b -> b.field("dynamic", "strict_allow_templates"))); + StrictDynamicMappingException exception = expectThrows( + StrictDynamicMappingException.class, + () -> mapper.parse(source(b -> b.nullField("bar"))) + ); + assertEquals( + "mapping set to strict_allow_templates, dynamic introduction of [bar] within [_doc] is not allowed", + exception.getMessage() + ); + } + public void testDynamicDottedFieldNameObject() throws Exception { DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); ParsedDocument doc = mapper.parse(source(b -> b.startObject("foo.bar.baz").field("a", 0).endObject())); From ebbc317a72713ee569364fe71bdce50d5a85b50b Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Wed, 26 Jun 2024 17:21:29 +0800 Subject: [PATCH 2/9] Modify change log Signed-off-by: Gao Binlong --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 664fb06d28b91..c1aafee74cbed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - Apply the date histogram rewrite optimization to range aggregation ([#13865](https://github.com/opensearch-project/OpenSearch/pull/13865)) - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782)) - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445)) -- The dynamic mapping parameter supports strict_allow_templates +- The dynamic mapping parameter supports strict_allow_templates ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555)) ### Dependencies - Bump `org.gradle.test-retry` from 1.5.8 to 1.5.9 ([#13442](https://github.com/opensearch-project/OpenSearch/pull/13442)) From bb0e1f8caf32d3a3d2af38b455ccddacfa451ffe Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Wed, 26 Jun 2024 22:18:36 +0800 Subject: [PATCH 3/9] Modify skip version in yml test file Signed-off-by: Gao Binlong --- .../rest-api-spec/test/index/110_strict_allow_templates.yml | 4 ++-- .../test/indices.put_mapping/all_path_options.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml index 623cb97c37728..b3899e295eb61 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/index/110_strict_allow_templates.yml @@ -1,8 +1,8 @@ --- "Index documents with setting dynamic parameter to strict_allow_templates in the mapping of the index": - skip: - version: " - 2.15.99" - reason: "introduced in 2.16.0" + version: " - 2.99.99" + reason: "introduced in 3.0.0" - do: indices.create: diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml index 89b47fde2a72c..f579891478b19 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/indices.put_mapping/all_path_options.yml @@ -163,8 +163,8 @@ setup: --- "post a mapping with setting dynamic to strict_allow_templates": - skip: - version: " - 2.15.99" - reason: "introduced in 2.16.0" + version: " - 2.99.99" + reason: "introduced in 3.0.0" - do: indices.put_mapping: index: test_index1 From 7c91f91e588efb05889e093de5bad2f9b94eee67 Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Fri, 5 Jul 2024 16:23:58 +0800 Subject: [PATCH 4/9] Refactor some code Signed-off-by: Gao Binlong --- .../index/mapper/DocumentParser.java | 116 +++++++++++++----- .../index/mapper/RootObjectMapper.java | 36 +----- 2 files changed, 92 insertions(+), 60 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 91e94ff534c97..df9d629ca22de 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -548,8 +548,13 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, if (dynamic == ObjectMapper.Dynamic.STRICT) { throw new StrictDynamicMappingException(dynamic.toString(), mapper.fullPath(), currentFieldName); } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.OBJECT, dynamic, mapper.fullPath()); + Mapper.Builder builder = findTemplateBuilder( + context, + currentFieldName, + XContentFieldType.OBJECT, + dynamic, + mapper.fullPath() + ); if (builder == null) { builder = new ObjectMapper.Builder(currentFieldName).enabled(true); @@ -596,8 +601,13 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, if (dynamic == ObjectMapper.Dynamic.STRICT) { throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), arrayFieldName); } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, arrayFieldName, XContentFieldType.OBJECT, dynamic, parentMapper.fullPath()); + Mapper.Builder builder = findTemplateBuilder( + context, + arrayFieldName, + XContentFieldType.OBJECT, + dynamic, + parentMapper.fullPath() + ); if (builder == null) { parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); } else { @@ -695,13 +705,13 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp throws IOException { // we can only handle null values if we have mappings for them Mapper mapper = getMapper(context, parentMapper, lastFieldName, paths); + ObjectMapper.Dynamic dynamic = parentMapper.dynamic(); if (mapper != null) { // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); - } else if (parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT - || parentMapper.dynamic() == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - throw new StrictDynamicMappingException(parentMapper.dynamic().toString(), parentMapper.fullPath(), lastFieldName); - } + } else if (dynamic == ObjectMapper.Dynamic.STRICT || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), lastFieldName); + } } private static Mapper.Builder newLongBuilder(String name, Settings settings) { @@ -739,15 +749,13 @@ private static Mapper.Builder createBuilderFromDynamicValue( } if (parseableAsLong && context.root().numericDetection()) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } return builder; } else if (parseableAsDouble && context.root().numericDetection()) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { builder = newFloatBuilder(currentFieldName, context.indexSettings().getSettings()); } @@ -763,8 +771,14 @@ private static Mapper.Builder createBuilderFromDynamicValue( // failure to parse this, continue continue; } - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder( + context, + currentFieldName, + XContentFieldType.DATE, + dateTimeFormatter, + dynamic, + fullPath + ); if (builder == null) { boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); builder = new DateFieldMapper.Builder( @@ -780,8 +794,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( } } - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder == null) { builder = new TextFieldMapper.Builder(currentFieldName, context.mapperService().getIndexAnalyzers()).addMultiField( new KeywordFieldMapper.Builder("keyword").ignoreAbove(256) @@ -793,8 +806,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( if (numberType == XContentParser.NumberType.INT || numberType == XContentParser.NumberType.LONG || numberType == XContentParser.NumberType.BIG_INTEGER) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.LONG, dynamic, fullPath); if (builder == null) { builder = newLongBuilder(currentFieldName, context.indexSettings().getSettings()); } @@ -802,8 +814,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( } else if (numberType == XContentParser.NumberType.FLOAT || numberType == XContentParser.NumberType.DOUBLE || numberType == XContentParser.NumberType.BIG_DECIMAL) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.DOUBLE, dynamic, fullPath); if (builder == null) { // no templates are defined, we use float by default instead of double // since this is much more space-efficient and should be enough most of @@ -813,22 +824,19 @@ private static Mapper.Builder createBuilderFromDynamicValue( return builder; } } else if (token == XContentParser.Token.VALUE_BOOLEAN) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BOOLEAN, dynamic, fullPath); if (builder == null) { builder = new BooleanFieldMapper.Builder(currentFieldName); } return builder; } else if (token == XContentParser.Token.VALUE_EMBEDDED_OBJECT) { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.BINARY, dynamic, fullPath); if (builder == null) { builder = new BinaryFieldMapper.Builder(currentFieldName); } return builder; } else { - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, XContentFieldType.STRING, dynamic, fullPath); if (builder != null) { return builder; } @@ -944,8 +952,13 @@ private static Tuple getDynamicParentMapper( throw new StrictDynamicMappingException(dynamic.toString(), parent.fullPath(), paths[i]); case STRICT_ALLOW_TEMPLATES: case TRUE: - Mapper.Builder builder = context.root() - .findTemplateBuilder(context, paths[i], XContentFieldType.OBJECT, dynamic, parent.fullPath()); + Mapper.Builder builder = findTemplateBuilder( + context, + paths[i], + XContentFieldType.OBJECT, + dynamic, + parent.fullPath() + ); if (builder == null) { builder = new ObjectMapper.Builder(paths[i]).enabled(true); } @@ -1027,4 +1040,51 @@ private static Mapper getMapper(final ParseContext context, ObjectMapper objectM } return objectMapper.getMapper(subfields[subfields.length - 1]); } + + @SuppressWarnings("rawtypes") + private static Mapper.Builder findTemplateBuilder( + ParseContext context, + String name, + XContentFieldType matchType, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { + DynamicTemplate dynamicTemplate = findTemplate(context, name, matchType, dynamic, fieldFullPath); + if (dynamicTemplate == null) { + return null; + } + + return context.root().findTemplateBuilder(context, name, matchType, null, dynamicTemplate); + } + + @SuppressWarnings("rawtypes") + private static Mapper.Builder findTemplateBuilder( + ParseContext context, + String name, + XContentFieldType matchType, + DateFormatter dateFormat, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { + DynamicTemplate dynamicTemplate = findTemplate(context, name, matchType, dynamic, fieldFullPath); + if (dynamicTemplate == null) { + return null; + } + + return context.root().findTemplateBuilder(context, name, matchType, dateFormat, dynamicTemplate); + } + + private static DynamicTemplate findTemplate( + ParseContext context, + String name, + XContentFieldType matchType, + ObjectMapper.Dynamic dynamic, + String fieldFullPath + ) { + DynamicTemplate dynamicTemplate = context.root().findTemplate(context.path(), name, matchType); + if (dynamicTemplate == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); + } + return dynamicTemplate; + } } diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 8fe059349f944..fe30425b73398 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -55,6 +55,8 @@ import java.util.Locale; import java.util.Map; +import reactor.util.annotation.NonNull; + import static org.opensearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.opensearch.index.mapper.TypeParsers.parseDateTimeFormatter; @@ -311,34 +313,12 @@ public DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } - @SuppressWarnings("rawtypes") - public Mapper.Builder findTemplateBuilder( - ParseContext context, - String name, - XContentFieldType matchType, - ObjectMapper.Dynamic dynamic, - String fieldFullPath - ) { - return findTemplateBuilder(context, name, matchType, null, dynamic, fieldFullPath); - } - - public Mapper.Builder findTemplateBuilder( - ParseContext context, - String name, - DateFormatter dateFormatter, - ObjectMapper.Dynamic dynamic, - String fieldFullPath - ) { - return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter, dynamic, fieldFullPath); - } - /** * Find a template. Returns {@code null} if no template could be found. * @param name the field name * @param matchType the type of the field in the json document or null if unknown * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format - * @param dynamic the type of dynamic mapping - * @param fieldFullPath the field's full path + * @param dynamicTemplate the dynamic template * @return a mapper builder, or null if there is no template for such a field */ @SuppressWarnings("rawtypes") @@ -347,16 +327,8 @@ public Mapper.Builder findTemplateBuilder( String name, XContentFieldType matchType, DateFormatter dateFormat, - ObjectMapper.Dynamic dynamic, - String fieldFullPath + @NonNull DynamicTemplate dynamicTemplate ) { - DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); - if (dynamicTemplate == null && dynamic == Dynamic.STRICT_ALLOW_TEMPLATES) { - throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); - } else if (dynamicTemplate == null) { - return null; - } - String dynamicType = matchType.defaultMappingType(); Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(dateFormat); String mappingType = dynamicTemplate.mappingType(dynamicType); From 321e1d66db86a6491fdcead570c4eae4f0e284c0 Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Fri, 5 Jul 2024 22:32:15 +0800 Subject: [PATCH 5/9] Keep the old methods Signed-off-by: Gao Binlong --- .../index/mapper/DocumentParser.java | 4 +-- .../index/mapper/RootObjectMapper.java | 32 ++++++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index df9d629ca22de..1cc4df3960f46 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -1054,7 +1054,7 @@ private static Mapper.Builder findTemplateBuilder( return null; } - return context.root().findTemplateBuilder(context, name, matchType, null, dynamicTemplate); + return context.root().findTemplateBuilderByTemplate(context, name, matchType, null, dynamicTemplate); } @SuppressWarnings("rawtypes") @@ -1071,7 +1071,7 @@ private static Mapper.Builder findTemplateBuilder( return null; } - return context.root().findTemplateBuilder(context, name, matchType, dateFormat, dynamicTemplate); + return context.root().findTemplateBuilderByTemplate(context, name, matchType, dateFormat, dynamicTemplate); } private static DynamicTemplate findTemplate( diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index fe30425b73398..4d565cc8558e8 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -55,8 +55,6 @@ import java.util.Locale; import java.util.Map; -import reactor.util.annotation.NonNull; - import static org.opensearch.common.xcontent.support.XContentMapValues.nodeBooleanValue; import static org.opensearch.index.mapper.TypeParsers.parseDateTimeFormatter; @@ -313,6 +311,29 @@ public DynamicTemplate[] dynamicTemplates() { return dynamicTemplates.value(); } + @SuppressWarnings("rawtypes") + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType) { + return findTemplateBuilder(context, name, matchType, null); + } + + @SuppressWarnings("rawtypes") + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, DateFormatter dateFormatter) { + return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter); + } + + /** + * Find a template. Returns {@code null} if no template could be found. + * @param name the field name + * @param matchType the type of the field in the json document or null if unknown + * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format + * @return a mapper builder, or null if there is no template for such a field + */ + @SuppressWarnings("rawtypes") + public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { + DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); + return findTemplateBuilderByTemplate(context, name, matchType, dateFormat, dynamicTemplate); + } + /** * Find a template. Returns {@code null} if no template could be found. * @param name the field name @@ -322,13 +343,16 @@ public DynamicTemplate[] dynamicTemplates() { * @return a mapper builder, or null if there is no template for such a field */ @SuppressWarnings("rawtypes") - public Mapper.Builder findTemplateBuilder( + public Mapper.Builder findTemplateBuilderByTemplate( ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat, - @NonNull DynamicTemplate dynamicTemplate + DynamicTemplate dynamicTemplate ) { + if (dynamicTemplate == null) { + return null; + } String dynamicType = matchType.defaultMappingType(); Mapper.TypeParser.ParserContext parserContext = context.docMapperParser().parserContext(dateFormat); String mappingType = dynamicTemplate.mappingType(dynamicType); From a0efcd7723077ffcc3dfdac93a1985a23f453ec9 Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Fri, 5 Jul 2024 22:36:41 +0800 Subject: [PATCH 6/9] change public to private Signed-off-by: Gao Binlong --- .../main/java/org/opensearch/index/mapper/RootObjectMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 4d565cc8558e8..0dd1c892444ab 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -329,7 +329,7 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat * @return a mapper builder, or null if there is no template for such a field */ @SuppressWarnings("rawtypes") - public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { + private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); return findTemplateBuilderByTemplate(context, name, matchType, dateFormat, dynamicTemplate); } From 7bae50310d8d76258dbf36b8c1933a69105c049f Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Sat, 6 Jul 2024 00:07:16 +0800 Subject: [PATCH 7/9] Optimize some code Signed-off-by: Gao Binlong --- .../index/mapper/DocumentParser.java | 41 +++++-------------- .../index/mapper/RootObjectMapper.java | 20 --------- 2 files changed, 10 insertions(+), 51 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 1cc4df3960f46..55ef1400b93ba 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -771,14 +771,7 @@ private static Mapper.Builder createBuilderFromDynamicValue( // failure to parse this, continue continue; } - Mapper.Builder builder = findTemplateBuilder( - context, - currentFieldName, - XContentFieldType.DATE, - dateTimeFormatter, - dynamic, - fullPath - ); + Mapper.Builder builder = findTemplateBuilder(context, currentFieldName, dateTimeFormatter, dynamic, fullPath); if (builder == null) { boolean ignoreMalformed = IGNORE_MALFORMED_SETTING.get(context.indexSettings().getSettings()); builder = new DateFieldMapper.Builder( @@ -1041,6 +1034,7 @@ private static Mapper getMapper(final ParseContext context, ObjectMapper objectM return objectMapper.getMapper(subfields[subfields.length - 1]); } + // Throws exception if no dynamic templates found but `dynamic` is set to strict_allow_templates @SuppressWarnings("rawtypes") private static Mapper.Builder findTemplateBuilder( ParseContext context, @@ -1049,42 +1043,27 @@ private static Mapper.Builder findTemplateBuilder( ObjectMapper.Dynamic dynamic, String fieldFullPath ) { - DynamicTemplate dynamicTemplate = findTemplate(context, name, matchType, dynamic, fieldFullPath); - if (dynamicTemplate == null) { - return null; + Mapper.Builder builder = context.root().findTemplateBuilder(context, name, matchType); + if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); } - return context.root().findTemplateBuilderByTemplate(context, name, matchType, null, dynamicTemplate); + return builder; } + // Throws exception if no dynamic templates found but `dynamic` is set to strict_allow_templates @SuppressWarnings("rawtypes") private static Mapper.Builder findTemplateBuilder( ParseContext context, String name, - XContentFieldType matchType, DateFormatter dateFormat, ObjectMapper.Dynamic dynamic, String fieldFullPath ) { - DynamicTemplate dynamicTemplate = findTemplate(context, name, matchType, dynamic, fieldFullPath); - if (dynamicTemplate == null) { - return null; - } - - return context.root().findTemplateBuilderByTemplate(context, name, matchType, dateFormat, dynamicTemplate); - } - - private static DynamicTemplate findTemplate( - ParseContext context, - String name, - XContentFieldType matchType, - ObjectMapper.Dynamic dynamic, - String fieldFullPath - ) { - DynamicTemplate dynamicTemplate = context.root().findTemplate(context.path(), name, matchType); - if (dynamicTemplate == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { + Mapper.Builder builder = context.root().findTemplateBuilder(context, name, dateFormat); + if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); } - return dynamicTemplate; + return builder; } } diff --git a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java index 0dd1c892444ab..e06e5be4633f9 100644 --- a/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/RootObjectMapper.java @@ -316,7 +316,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, XCo return findTemplateBuilder(context, name, matchType, null); } - @SuppressWarnings("rawtypes") public Mapper.Builder findTemplateBuilder(ParseContext context, String name, DateFormatter dateFormatter) { return findTemplateBuilder(context, name, XContentFieldType.DATE, dateFormatter); } @@ -331,25 +330,6 @@ public Mapper.Builder findTemplateBuilder(ParseContext context, String name, Dat @SuppressWarnings("rawtypes") private Mapper.Builder findTemplateBuilder(ParseContext context, String name, XContentFieldType matchType, DateFormatter dateFormat) { DynamicTemplate dynamicTemplate = findTemplate(context.path(), name, matchType); - return findTemplateBuilderByTemplate(context, name, matchType, dateFormat, dynamicTemplate); - } - - /** - * Find a template. Returns {@code null} if no template could be found. - * @param name the field name - * @param matchType the type of the field in the json document or null if unknown - * @param dateFormat a dateformatter to use if the type is a date, null if not a date or is using the default format - * @param dynamicTemplate the dynamic template - * @return a mapper builder, or null if there is no template for such a field - */ - @SuppressWarnings("rawtypes") - public Mapper.Builder findTemplateBuilderByTemplate( - ParseContext context, - String name, - XContentFieldType matchType, - DateFormatter dateFormat, - DynamicTemplate dynamicTemplate - ) { if (dynamicTemplate == null) { return null; } From 84c5db521a422adba755e9a6d37bab8147c3694b Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Tue, 9 Jul 2024 11:20:27 +0800 Subject: [PATCH 8/9] Do not override toString method for Dynamic Signed-off-by: Gao Binlong --- .../index/mapper/DocumentParser.java | 19 ++++++++++++------- .../opensearch/index/mapper/ObjectMapper.java | 19 ++++--------------- 2 files changed, 16 insertions(+), 22 deletions(-) diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index 55ef1400b93ba..d8c3c88fc4d52 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -54,6 +54,7 @@ import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Locale; import static org.opensearch.index.mapper.FieldMapper.IGNORE_MALFORMED_SETTING; @@ -546,7 +547,7 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, ObjectMapper parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(dynamic.toString(), mapper.fullPath(), currentFieldName); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { Mapper.Builder builder = findTemplateBuilder( context, @@ -599,7 +600,11 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), arrayFieldName); + throw new StrictDynamicMappingException( + dynamic.name().toLowerCase(Locale.ROOT), + parentMapper.fullPath(), + arrayFieldName + ); } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { Mapper.Builder builder = findTemplateBuilder( context, @@ -710,7 +715,7 @@ private static void parseNullValue(ParseContext context, ObjectMapper parentMapp // TODO: passing null to an object seems bogus? parseObjectOrField(context, mapper); } else if (dynamic == ObjectMapper.Dynamic.STRICT || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), lastFieldName); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), lastFieldName); } } @@ -848,7 +853,7 @@ private static void parseDynamicValue( ) throws IOException { ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(dynamic.toString(), parentMapper.fullPath(), currentFieldName); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parentMapper.fullPath(), currentFieldName); } if (dynamic == ObjectMapper.Dynamic.FALSE) { return; @@ -942,7 +947,7 @@ private static Tuple getDynamicParentMapper( switch (dynamic) { case STRICT: - throw new StrictDynamicMappingException(dynamic.toString(), parent.fullPath(), paths[i]); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), parent.fullPath(), paths[i]); case STRICT_ALLOW_TEMPLATES: case TRUE: Mapper.Builder builder = findTemplateBuilder( @@ -1045,7 +1050,7 @@ private static Mapper.Builder findTemplateBuilder( ) { Mapper.Builder builder = context.root().findTemplateBuilder(context, name, matchType); if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), fieldFullPath, name); } return builder; @@ -1062,7 +1067,7 @@ private static Mapper.Builder findTemplateBuilder( ) { Mapper.Builder builder = context.root().findTemplateBuilder(context, name, dateFormat); if (builder == null && dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - throw new StrictDynamicMappingException(dynamic.toString(), fieldFullPath, name); + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), fieldFullPath, name); } return builder; } diff --git a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java index 2bac54e1f92f6..533e6ca73d737 100644 --- a/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java +++ b/server/src/main/java/org/opensearch/index/mapper/ObjectMapper.java @@ -90,21 +90,10 @@ public static class Defaults { */ @PublicApi(since = "1.0.0") public enum Dynamic { - TRUE("true"), - FALSE("false"), - STRICT("strict"), - STRICT_ALLOW_TEMPLATES("strict_allow_templates"); - - private final String name; - - Dynamic(String name) { - this.name = name; - } - - @Override - public String toString() { - return this.name; - } + TRUE, + FALSE, + STRICT, + STRICT_ALLOW_TEMPLATES } /** From 37117f555dcf14c959c4314587601a78e21f5d79 Mon Sep 17 00:00:00 2001 From: Gao Binlong Date: Thu, 11 Jul 2024 12:09:13 +0800 Subject: [PATCH 9/9] Optimize some code and modify the changelog Signed-off-by: Gao Binlong --- CHANGELOG.md | 2 +- .../index/mapper/DocumentParser.java | 114 +++++++++--------- 2 files changed, 61 insertions(+), 55 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 30dda532d19db..5f6eae9647e75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), - [Writable Warm] Add composite directory implementation and integrate it with FileCache ([12782](https://github.com/opensearch-project/OpenSearch/pull/12782)) - Add batching supported processor base type AbstractBatchingProcessor ([#14554](https://github.com/opensearch-project/OpenSearch/pull/14554)) - Fix race condition while parsing derived fields from search definition ([14445](https://github.com/opensearch-project/OpenSearch/pull/14445)) -- The dynamic mapping parameter supports strict_allow_templates ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555)) +- Add `strict_allow_templates` dynamic mapping option ([#14555](https://github.com/opensearch-project/OpenSearch/pull/14555)) - Add allowlist setting for ingest-common and search-pipeline-common processors ([#14439](https://github.com/opensearch-project/OpenSearch/issues/14439)) ### Dependencies diff --git a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java index d8c3c88fc4d52..c6815ebe8d91a 100644 --- a/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java +++ b/server/src/main/java/org/opensearch/index/mapper/DocumentParser.java @@ -546,29 +546,32 @@ private static void parseObject(final ParseContext context, ObjectMapper mapper, Tuple parentMapperTuple = getDynamicParentMapper(context, paths, mapper); ObjectMapper parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); - if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); - } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - Mapper.Builder builder = findTemplateBuilder( - context, - currentFieldName, - XContentFieldType.OBJECT, - dynamic, - mapper.fullPath() - ); + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException(dynamic.name().toLowerCase(Locale.ROOT), mapper.fullPath(), currentFieldName); + case TRUE: + case STRICT_ALLOW_TEMPLATES: + Mapper.Builder builder = findTemplateBuilder( + context, + currentFieldName, + XContentFieldType.OBJECT, + dynamic, + mapper.fullPath() + ); - if (builder == null) { - builder = new ObjectMapper.Builder(currentFieldName).enabled(true); - } - Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); - objectMapper = builder.build(builderContext); - context.addDynamicMapper(objectMapper); - context.path().add(currentFieldName); - parseObjectOrField(context, objectMapper); - context.path().remove(); - } else { - // not dynamic, read everything up to end object - context.parser().skipChildren(); + if (builder == null) { + builder = new ObjectMapper.Builder(currentFieldName).enabled(true); + } + Mapper.BuilderContext builderContext = new Mapper.BuilderContext(context.indexSettings().getSettings(), context.path()); + objectMapper = builder.build(builderContext); + context.addDynamicMapper(objectMapper); + context.path().add(currentFieldName); + parseObjectOrField(context, objectMapper); + context.path().remove(); + break; + case FALSE: + // not dynamic, read everything up to end object + context.parser().skipChildren(); } for (int i = 0; i < parentMapperTuple.v1(); i++) { context.path().remove(); @@ -599,41 +602,44 @@ private static void parseArray(ParseContext context, ObjectMapper parentMapper, Tuple parentMapperTuple = getDynamicParentMapper(context, paths, parentMapper); parentMapper = parentMapperTuple.v2(); ObjectMapper.Dynamic dynamic = dynamicOrDefault(parentMapper, context); - if (dynamic == ObjectMapper.Dynamic.STRICT) { - throw new StrictDynamicMappingException( - dynamic.name().toLowerCase(Locale.ROOT), - parentMapper.fullPath(), - arrayFieldName - ); - } else if (dynamic == ObjectMapper.Dynamic.TRUE || dynamic == ObjectMapper.Dynamic.STRICT_ALLOW_TEMPLATES) { - Mapper.Builder builder = findTemplateBuilder( - context, - arrayFieldName, - XContentFieldType.OBJECT, - dynamic, - parentMapper.fullPath() - ); - if (builder == null) { - parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); - } else { - Mapper.BuilderContext builderContext = new Mapper.BuilderContext( - context.indexSettings().getSettings(), - context.path() + switch (dynamic) { + case STRICT: + throw new StrictDynamicMappingException( + dynamic.name().toLowerCase(Locale.ROOT), + parentMapper.fullPath(), + arrayFieldName ); - mapper = builder.build(builderContext); - assert mapper != null; - if (parsesArrayValue(mapper)) { - context.addDynamicMapper(mapper); - context.path().add(arrayFieldName); - parseObjectOrField(context, mapper); - context.path().remove(); - } else { + case TRUE: + case STRICT_ALLOW_TEMPLATES: + Mapper.Builder builder = findTemplateBuilder( + context, + arrayFieldName, + XContentFieldType.OBJECT, + dynamic, + parentMapper.fullPath() + ); + if (builder == null) { parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); + } else { + Mapper.BuilderContext builderContext = new Mapper.BuilderContext( + context.indexSettings().getSettings(), + context.path() + ); + mapper = builder.build(builderContext); + assert mapper != null; + if (parsesArrayValue(mapper)) { + context.addDynamicMapper(mapper); + context.path().add(arrayFieldName); + parseObjectOrField(context, mapper); + context.path().remove(); + } else { + parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); + } } - } - } else { - // TODO: shouldn't this skip, not parse? - parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); + break; + case FALSE: + // TODO: shouldn't this skip, not parse? + parseNonDynamicArray(context, parentMapper, lastFieldName, arrayFieldName); } for (int i = 0; i < parentMapperTuple.v1(); i++) { context.path().remove();