Skip to content

Commit

Permalink
ESQL: use field_caps native nested fields filtering (elastic#117201)
Browse files Browse the repository at this point in the history 
* Just filter the nested fields natively with field_caps support

---------

Co-authored-by: Elastic Machine <elasticmachine@users.noreply.github.com>
Co-authored-by: Craig Taverner <craig@amanzi.com>
  • Loading branch information
@astefan @elasticmachine @craigtaverner
3 people committed Nov 22, 2024
1 parent 203615f commit 27aa390
Show file tree
Hide file tree
Showing 4 changed files with 334 additions and 22 deletions.
6 changes: 6 additions & 0 deletions docs/changelog/117201.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 117201
summary: "Use `field_caps` native nested fields filtering"
area: ES|QL
type: bug
issues:
- 117054
318 changes: 318 additions & 0 deletions .../qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.hamcrest.Matcher;
import org.junit.Before;

Expand Down Expand Up @@ -1106,6 +1107,323 @@ public void testTypeConflictInObject() throws IOException {
);
}

/**
* Test for https://github.com/elastic/elasticsearch/issues/117054 fix
*/
public void testOneNestedSubField_AndSameNameSupportedField() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");

Map<String, Object> result = runEsql("FROM test");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", Collections.EMPTY_LIST)
);

index("test", """
{"Responses.process.pid": 123,"process.parent.command_line":"run.bat"}""");

result = runEsql("FROM test");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

result = runEsql("""
FROM test | where process.parent.command_line == "run.bat"
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test | SORT Responses.process.pid"));
String err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 1:18: Unknown column [Responses.process.pid]"));

e = expectThrows(ResponseException.class, () -> runEsql("""
FROM test
| SORT Responses.process.pid
| WHERE Responses.process IS NULL
"""));
err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
}

public void testOneNestedSubField_AndSameNameSupportedField_TwoIndices() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
}
}
""");
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
"properties": {
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
index("test1", """
{"Responses.process.pid": 123}""");
index("test2", """
{"process.parent.command_line":"run.bat"}""");

Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line ASC NULLS FIRST");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item(null).item(null), matchesList().item("run.bat").item("run.bat")))
);

result = runEsql("""
FROM test* | where process.parent.command_line == "run.bat"
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT Responses.process.pid"));
String err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 1:19: Unknown column [Responses.process.pid]"));

e = expectThrows(ResponseException.class, () -> runEsql("""
FROM test*
| SORT Responses.process.pid
| WHERE Responses.process IS NULL
"""));
err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
}

public void testOneNestedField_AndSameNameSupportedField_TwoIndices() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "integer",
"fields": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
index("test1", """
{"Responses.process.pid": 111,"process.parent.command_line":"run1.bat"}""");
index("test2", """
{"Responses.process": 222,"process.parent.command_line":"run2.bat"}""");

Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
)
.entry(
"values",
List.of(
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
)
)
);

result = runEsql("""
FROM test* | where Responses.process.pid == 111
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
).entry("values", List.of())
);

result = runEsql("FROM test* | SORT process.parent.command_line");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
)
.entry(
"values",
List.of(
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
)
)
);

result = runEsql("""
FROM test*
| SORT process.parent.command_line
| WHERE Responses.process IS NULL
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
).entry("values", List.of(matchesList().item(null).item(null).item("run1.bat").item("run1.bat")))
);
}

private void assumeIndexResolverNestedFieldsNameClashFixed() throws IOException {
// especially for BWC tests but also for regular tests
var capsName = EsqlCapabilities.Cap.FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER.name().toLowerCase(Locale.ROOT);
boolean requiredClusterCapability = clusterHasCapability("POST", "/_query", List.of(), List.of(capsName)).orElse(false);
assumeTrue(
"This test makes sense for versions that have the fix for https://github.com/elastic/elasticsearch/issues/117054",
requiredClusterCapability
);
}

private CheckedConsumer<XContentBuilder, IOException> empNoInObject(String empNoType) {
return index -> {
index.startObject("properties");
Expand Down
7 changes: 6 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,12 @@ public enum Cap {
/**
* LOOKUP JOIN
*/
JOIN_LOOKUP(Build.current().isSnapshot());
JOIN_LOOKUP(Build.current().isSnapshot()),

/**
* Fix for https://github.com/elastic/elasticsearch/issues/117054
*/
FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER;

private final boolean enabled;

Expand Down
Loading

0 comments on commit 27aa390

Please sign in to comment.