Skip to content

Commit

Permalink
Enforce a limit on the depth of the JSON object. (#35063)
Browse files Browse the repository at this point in the history
  • Loading branch information
@jtibshirani
jtibshirani committed May 1, 2019
1 parent d03f777 commit b91b948
Show file tree
Hide file tree
Showing 5 changed files with 100 additions and 15 deletions.
4 changes: 4 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/ContentPath.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,4 +66,8 @@ public String pathAsText(String name) {
sb.append(name);
return sb.toString();
}

public int length() {
return index;
}
}
30 changes: 25 additions & 5 deletions server/src/main/java/org/elasticsearch/index/mapper/JsonFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -97,10 +97,12 @@ private static class Defaults {
FIELD_TYPE.freeze();
}

public static final int DEPTH_LIMIT = 20;
public static final int IGNORE_ABOVE = Integer.MAX_VALUE;
}

public static class Builder extends FieldMapper.Builder<Builder, JsonFieldMapper> {
private int depthLimit = Defaults.DEPTH_LIMIT;
private int ignoreAbove = Defaults.IGNORE_ABOVE;

public Builder(String name) {
Expand All @@ -123,6 +125,14 @@ public Builder indexOptions(IndexOptions indexOptions) {
return super.indexOptions(indexOptions);
}

public Builder depthLimit(int depthLimit) {
if (depthLimit < 0) {
throw new IllegalArgumentException("[depth_limit] must be positive, got " + depthLimit);
}
this.depthLimit = depthLimit;
return this;
}

public Builder ignoreAbove(int ignoreAbove) {
if (ignoreAbove < 0) {
throw new IllegalArgumentException("[ignore_above] must be positive, got " + ignoreAbove);
Expand Down Expand Up @@ -153,7 +163,7 @@ public JsonFieldMapper build(BuilderContext context) {
fieldType().setSearchAnalyzer(WHITESPACE_ANALYZER);
}
return new JsonFieldMapper(name, fieldType, defaultFieldType,
ignoreAbove, context.indexSettings());
ignoreAbove, depthLimit, context.indexSettings());
}
}

Expand All @@ -166,7 +176,10 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node, ParserCo
Map.Entry<String, Object> entry = iterator.next();
String propName = entry.getKey();
Object propNode = entry.getValue();
if (propName.equals("ignore_above")) {
if (propName.equals("depth_limit")) {
builder.depthLimit(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (propName.equals("ignore_above")) {
builder.ignoreAbove(XContentMapValues.nodeIntegerValue(propNode, -1));
iterator.remove();
} else if (propName.equals("null_value")) {
Expand Down Expand Up @@ -367,19 +380,22 @@ public Query wildcardQuery(String value,
}

private final JsonFieldParser fieldParser;
private int depthLimit;
private int ignoreAbove;

private JsonFieldMapper(String simpleName,
MappedFieldType fieldType,
MappedFieldType defaultFieldType,
int ignoreAbove,
int depthLimit,
Settings indexSettings) {
super(simpleName, fieldType, defaultFieldType, indexSettings, MultiFields.empty(), CopyTo.empty());
assert fieldType.indexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) <= 0;

this.depthLimit = depthLimit;
this.ignoreAbove = ignoreAbove;
this.fieldParser = new JsonFieldParser(fieldType.name(), keyedFieldName(),
ignoreAbove, fieldType.nullValueAsString());
depthLimit, ignoreAbove, fieldType.nullValueAsString());
}

@Override
Expand Down Expand Up @@ -453,14 +469,18 @@ protected void parseCreateField(ParseContext context, List<IndexableField> field
protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException {
super.doXContentBody(builder, includeDefaults, params);

if (includeDefaults || fieldType().nullValue() != null) {
builder.field("null_value", fieldType().nullValue());
if (includeDefaults || depthLimit != Defaults.DEPTH_LIMIT) {
builder.field("depth_limit", depthLimit);
}

if (includeDefaults || ignoreAbove != Defaults.IGNORE_ABOVE) {
builder.field("ignore_above", ignoreAbove);
}

if (includeDefaults || fieldType().nullValue() != null) {
builder.field("null_value", fieldType().nullValue());
}

if (includeDefaults || fieldType().splitQueriesOnWhitespace()) {
builder.field("split_queries_on_whitespace", fieldType().splitQueriesOnWhitespace());
}
Expand Down
11 changes: 11 additions & 0 deletions server/src/main/java/org/elasticsearch/index/mapper/JsonFieldParser.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,15 +40,18 @@ public class JsonFieldParser {
private final String rootFieldName;
private final String keyedFieldName;

private final int depthLimit;
private final int ignoreAbove;
private final String nullValueAsString;

JsonFieldParser(String rootFieldName,
String keyedFieldName,
int depthLimit,
int ignoreAbove,
String nullValueAsString) {
this.rootFieldName = rootFieldName;
this.keyedFieldName = keyedFieldName;
this.depthLimit = depthLimit;
this.ignoreAbove = ignoreAbove;
this.nullValueAsString = nullValueAsString;
}
Expand Down Expand Up @@ -104,6 +107,7 @@ private void parseFieldValue(XContentParser.Token token,
List<IndexableField> fields) throws IOException {
if (token == XContentParser.Token.START_OBJECT) {
path.add(currentName);
validateDepthLimit(path);
parseObject(parser, path, fields);
path.remove();
} else if (token == XContentParser.Token.START_ARRAY) {
Expand Down Expand Up @@ -141,6 +145,13 @@ private void addField(ContentPath path,
fields.add(new StringField(keyedFieldName, new BytesRef(keyedValue), Field.Store.NO));
}

private void validateDepthLimit(ContentPath path) {
if (path.length() + 1 > depthLimit) {
throw new IllegalArgumentException("The provided JSON field [" + rootFieldName + "] exceeds" +
" the maximum depth limit of [" + depthLimit + "].");
}
}

public static String createKeyedValue(String key, String value) {
return key + SEPARATOR + value;
}
Expand Down
29 changes: 29 additions & 0 deletions server/src/test/java/org/elasticsearch/index/mapper/JsonFieldMapperTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,6 +300,35 @@ public void testFieldMultiplicity() throws Exception {
assertEquals(new BytesRef("key3\0false"), keyedFields[2].binaryValue());
}

public void testDepthLimit() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
.startObject("properties")
.startObject("field")
.field("type", "json")
.field("depth_limit", 2)
.endObject()
.endObject()
.endObject()
.endObject());

DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
assertEquals(mapping, mapper.mappingSource().toString());

BytesReference doc = BytesReference.bytes(XContentFactory.jsonBuilder().startObject()
.startObject("field")
.startObject("key1")
.startObject("key2")
.field("key3", "value")
.endObject()
.endObject()
.endObject()
.endObject());

expectThrows(MapperParsingException.class, () ->
mapper.parse(new SourceToParse("test", "type", "1", doc, XContentType.JSON)));
}

public void testIgnoreAbove() throws IOException {
String mapping = Strings.toString(XContentFactory.jsonBuilder().startObject()
.startObject("type")
Expand Down
41 changes: 31 additions & 10 deletions server/src/test/java/org/elasticsearch/index/mapper/JsonFieldParserTests.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
import org.elasticsearch.common.xcontent.XContentParser;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.common.xcontent.json.JsonXContent;
import org.elasticsearch.index.mapper.JsonFieldMapper.RootJsonFieldType;
import org.elasticsearch.test.ESTestCase;
import org.elasticsearch.test.XContentTestUtils;
import org.junit.Before;
Expand All @@ -41,7 +40,8 @@ public class JsonFieldParserTests extends ESTestCase {
@Before
public void setUp() throws Exception {
super.setUp();
parser = new JsonFieldParser("field", "field._keyed", Integer.MAX_VALUE, null);
parser = new JsonFieldParser("field", "field._keyed",
Integer.MAX_VALUE, Integer.MAX_VALUE, null);
}

public void testTextValues() throws Exception {
Expand Down Expand Up @@ -213,15 +213,36 @@ public void testNestedObjects() throws Exception {
assertEquals(new BytesRef("parent2.key\0value"), keyedField2.binaryValue());
}

public void testDepthLimit() throws Exception {
String input = "{ \"parent1\": { \"key\" : \"value\" }," +
"\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}";
XContentParser xContentParser = createXContentParser(input);
JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed",
2, Integer.MAX_VALUE, null);

IllegalArgumentException e = expectThrows(IllegalArgumentException.class,
() -> configuredParser.parse(xContentParser));
assertEquals("The provided JSON field [field] exceeds the maximum depth limit of [2].", e.getMessage());
}

public void testDepthLimitBoundary() throws Exception {
String input = "{ \"parent1\": { \"key\" : \"value\" }," +
"\"parent2\": [{ \"key\" : { \"key\" : \"value\" }}]}";
XContentParser xContentParser = createXContentParser(input);
JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed",
3, Integer.MAX_VALUE, null);

List<IndexableField> fields = configuredParser.parse(xContentParser);
assertEquals(4, fields.size());
}

public void testIgnoreAbove() throws Exception {
String input = "{ \"key\": \"a longer field than usual\" }";
XContentParser xContentParser = createXContentParser(input);
JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed",
Integer.MAX_VALUE, 10, null);

RootJsonFieldType fieldType = new RootJsonFieldType();
fieldType.setName("field");
JsonFieldParser parserWithIgnoreAbove = new JsonFieldParser("field", "field._keyed", 10, null);

List<IndexableField> fields = parserWithIgnoreAbove.parse(xContentParser);
List<IndexableField> fields = configuredParser.parse(xContentParser);
assertEquals(0, fields.size());
}

Expand All @@ -233,10 +254,10 @@ public void testNullValues() throws Exception {
assertEquals(0, fields.size());

xContentParser = createXContentParser(input);
JsonFieldParser parserWithNullValue = new JsonFieldParser("field", "field._keyed",
Integer.MAX_VALUE, "placeholder");
JsonFieldParser configuredParser = new JsonFieldParser("field", "field._keyed",
Integer.MAX_VALUE, Integer.MAX_VALUE, "placeholder");

fields = parserWithNullValue.parse(xContentParser);
fields = configuredParser.parse(xContentParser);
assertEquals(2, fields.size());

IndexableField field = fields.get(0);
Expand Down

0 comments on commit b91b948

Please sign in to comment.