From 797a42a4f3bb88f03bfee4891dc74551c5ef2675 Mon Sep 17 00:00:00 2001 From: Julie Tibshirani Date: Wed, 31 Oct 2018 11:52:35 -0700 Subject: [PATCH] Enforce a limit on the depth of the JSON object. (#35063) --- .../index/mapper/ContentPath.java | 4 ++ .../index/mapper/JsonFieldMapper.java | 30 +++++++++++--- .../index/mapper/JsonFieldParser.java | 11 +++++ .../index/mapper/JsonFieldMapperTests.java | 29 +++++++++++++ .../index/mapper/JsonFieldParserTests.java | 41 ++++++++++++++----- 5 files changed, 100 insertions(+), 15 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java index 3c67d3ee7f31c..7f3e26312ad87 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java @@ -66,4 +66,8 @@ public String pathAsText(String name) { sb.append(name); return sb.toString(); } + + public int length() { + return index; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java index 62cbeebeafe84..5ed98746fbf59 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java @@ -97,10 +97,12 @@ private static class Defaults { FIELD_TYPE.freeze(); } + public static final int DEPTH_LIMIT = 20; public static final int IGNORE_ABOVE = Integer.MAX_VALUE; } public static class Builder extends FieldMapper.Builder { + private int depthLimit = Defaults.DEPTH_LIMIT; private int ignoreAbove = Defaults.IGNORE_ABOVE; public Builder(String name) { @@ -123,6 +125,14 @@ public Builder indexOptions(IndexOptions indexOptions) { return super.indexOptions(indexOptions); } + public Builder depthLimit(int depthLimit) { + if (depthLimit < 0) { + throw new IllegalArgumentException("[depth_limit] must be positive, got " + depthLimit); + } + this.depthLimit = depthLimit; + return this; + } + public Builder ignoreAbove(int ignoreAbove) { if (ignoreAbove < 0) { throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove); @@ -153,7 +163,7 @@ public JsonFieldMapper build(BuilderContext context) { fieldType().setSearchAnalyzer(WHITESPACE_ANALYZER); } return new JsonFieldMapper(name, fieldType, defaultFieldType, - ignoreAbove, context.indexSettings()); + ignoreAbove, depthLimit, context.indexSettings()); } } @@ -166,7 +176,10 @@ public Mapper.Builder parse(String name, Map node, ParserCo Map.Entry entry = iterator.next(); String propName = entry.getKey(); Object propNode = entry.getValue(); - if (propName.equals("ignore_above")) { + if (propName.equals("depth_limit")) { + builder.depthLimit(XContentMapValues.nodeIntegerValue(propNode, -1)); + iterator.remove(); + } else if (propName.equals("ignore_above")) { builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1)); iterator.remove(); } else if (propName.equals("null_value")) { @@ -367,19 +380,22 @@ public Query wildcardQuery(String value, } private final JsonFieldParser fieldParser; + private int depthLimit; private int ignoreAbove; private JsonFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, int ignoreAbove, + int depthLimit, Settings indexSettings) { super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty()); assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0; + this.depthLimit = depthLimit; this.ignoreAbove = ignoreAbove; this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(), - ignoreAbove, fieldType.nullValueAsString()); + depthLimit, ignoreAbove, fieldType.nullValueAsString()); } @Override @@ -453,14 +469,18 @@ protected void parseCreateField(ParseContext context, List field protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { super.doXContentBody(builder, includeDefaults, params); - if (includeDefaults || fieldType().nullValue() != null) { - builder.field("null_value", fieldType().nullValue()); + if (includeDefaults || depthLimit != Defaults.DEPTH_LIMIT) { + builder.field("depth_limit", depthLimit); } if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) { builder.field("ignore_above", ignoreAbove); } + if (includeDefaults || fieldType().nullValue() != null) { + builder.field("null_value", fieldType().nullValue()); + } + if (includeDefaults || fieldType().splitQueriesOnWhitespace()) { builder.field("split_queries_on_whitespace", fieldType().splitQueriesOnWhitespace()); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java index fd05c07538c5b..d3109e8975197 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java @@ -40,15 +40,18 @@ public class JsonFieldParser { private final String rootFieldName; private final String keyedFieldName; + private final int depthLimit; private final int ignoreAbove; private final String nullValueAsString; JsonFieldParser(String rootFieldName, String keyedFieldName, + int depthLimit, int ignoreAbove, String nullValueAsString) { this.rootFieldName = rootFieldName; this.keyedFieldName = keyedFieldName; + this.depthLimit = depthLimit; this.ignoreAbove = ignoreAbove; this.nullValueAsString = nullValueAsString; } @@ -104,6 +107,7 @@ private void parseFieldValue(XContentParser.Token token, List fields) throws IOException { if (token == XContentParser.Token.START_OBJECT) { path.add(currentName); + validateDepthLimit(path); parseObject(parser, path, fields); path.remove(); } else if (token == XContentParser.Token.START_ARRAY) { @@ -141,6 +145,13 @@ private void addField(ContentPath path, fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO)); } + private void validateDepthLimit(ContentPath path) { + if (path.length() + 1 > depthLimit) { + throw new IllegalArgumentException("The provided JSON field [" + rootFieldName + "] exceeds" + + " the maximum depth limit of [" + depthLimit + "]."); + } + } + public static String createKeyedValue(String key, String value) { return key + SEPARATOR + value; } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java index 36dd6999eab8b..cee87f3a81e34 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java @@ -300,6 +300,35 @@ public void testFieldMultiplicity() throws Exception { assertEquals(new BytesRef("key3\0false"), keyedFields[2].binaryValue()); } + public void testDepthLimit() throws IOException { + String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() + .startObject("type") + .startObject("properties") + .startObject("field") + .field("type", "json") + .field("depth_limit", 2) + .endObject() + .endObject() + .endObject() + .endObject()); + + DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping)); + assertEquals(mapping, mapper.mappingSource().toString()); + + BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject() + .startObject("field") + .startObject("key1") + .startObject("key2") + .field("key3", "value") + .endObject() + .endObject() + .endObject() + .endObject()); + + expectThrows(MapperParsingException.class, () -> + mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON))); + } + public void testIgnoreAbove() throws IOException { String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject() .startObject("type") diff --git a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java index 715397167fd75..ea0f3a150db5f 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java @@ -27,7 +27,6 @@ import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.common.xcontent.json.JsonXContent; -import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.test.XContentTestUtils; import org.junit.Before; @@ -41,7 +40,8 @@ public class JsonFieldParserTests extends ESTestCase { @Before public void setUp() throws Exception { super.setUp(); - parser = new JsonFieldParser("field", "field._keyed", Integer.MAX_VALUE, null); + parser = new JsonFieldParser("field", "field._keyed", + Integer.MAX_VALUE, Integer.MAX_VALUE, null); } public void testTextValues() throws Exception { @@ -213,15 +213,36 @@ public void testNestedObjects() throws Exception { assertEquals(new BytesRef("parent2.key\0value"), keyedField2.binaryValue()); } + public void testDepthLimit() throws Exception { + String input = "{ \"parent1\": { \"key\" : \"value\" }," + + "\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}"; + XContentParser xContentParser = createXContentParser(input); + JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed", + 2, Integer.MAX_VALUE, null); + + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, + () -> configuredParser.parse(xContentParser)); + assertEquals("The provided JSON field [field] exceeds the maximum depth limit of [2].", e.getMessage()); + } + + public void testDepthLimitBoundary() throws Exception { + String input = "{ \"parent1\": { \"key\" : \"value\" }," + + "\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}"; + XContentParser xContentParser = createXContentParser(input); + JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed", + 3, Integer.MAX_VALUE, null); + + List fields = configuredParser.parse(xContentParser); + assertEquals(4, fields.size()); + } + public void testIgnoreAbove() throws Exception { String input = "{ \"key\": \"a longer field than usual\" }"; XContentParser xContentParser = createXContentParser(input); + JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed", + Integer.MAX_VALUE, 10, null); - RootJsonFieldType fieldType = new RootJsonFieldType(); - fieldType.setName("field"); - JsonFieldParser parserWithIgnoreAbove = new JsonFieldParser("field", "field._keyed", 10, null); - - List fields = parserWithIgnoreAbove.parse(xContentParser); + List fields = configuredParser.parse(xContentParser); assertEquals(0, fields.size()); } @@ -233,10 +254,10 @@ public void testNullValues() throws Exception { assertEquals(0, fields.size()); xContentParser = createXContentParser(input); - JsonFieldParser parserWithNullValue = new JsonFieldParser("field", "field._keyed", - Integer.MAX_VALUE, "placeholder"); + JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed", + Integer.MAX_VALUE, Integer.MAX_VALUE, "placeholder"); - fields = parserWithNullValue.parse(xContentParser); + fields = configuredParser.parse(xContentParser); assertEquals(2, fields.size()); IndexableField field = fields.get(0);