Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[8.x] ESQL: use field_caps native nested fields filtering (#117201) #117375

Merged
merged 3 commits into from
Nov 22, 2024
Merged

[8.x] ESQL: use field_caps native nested fields filtering (#117201) #117375

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
27aa390
ESQL: use field_caps native nested fields filtering (#117201)
astefan Nov 22, 2024
9a6320e
Add import
astefan Nov 22, 2024
faf0302
Merge branch '8.x' of https://github.com/elastic/elasticsearch into b…
astefan Nov 22, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/changelog/117201.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
pr: 117201
summary: "Use `field_caps` native nested fields filtering"
area: ES|QL
type: bug
issues:
- 117054
319 changes: 319 additions & 0 deletions .../qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.elasticsearch.client.ResponseException;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.network.NetworkAddress;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.core.CheckedConsumer;
import org.elasticsearch.geo.GeometryTestUtils;
import org.elasticsearch.index.mapper.BlockLoader;
Expand All @@ -27,6 +28,7 @@
import org.elasticsearch.xcontent.XContentBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.elasticsearch.xcontent.json.JsonXContent;
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
import org.hamcrest.Matcher;
import org.junit.Before;

Expand Down Expand Up @@ -1106,6 +1108,323 @@ public void testTypeConflictInObject() throws IOException {
);
}

/**
* Test for https://github.com/elastic/elasticsearch/issues/117054 fix
*/
public void testOneNestedSubField_AndSameNameSupportedField() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");

Map<String, Object> result = runEsql("FROM test");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", Collections.EMPTY_LIST)
);

index("test", """
{"Responses.process.pid": 123,"process.parent.command_line":"run.bat"}""");

result = runEsql("FROM test");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

result = runEsql("""
FROM test | where process.parent.command_line == "run.bat"
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test | SORT Responses.process.pid"));
String err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 1:18: Unknown column [Responses.process.pid]"));

e = expectThrows(ResponseException.class, () -> runEsql("""
FROM test
| SORT Responses.process.pid
| WHERE Responses.process IS NULL
"""));
err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
}

public void testOneNestedSubField_AndSameNameSupportedField_TwoIndices() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
}
}
""");
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
"properties": {
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
index("test1", """
{"Responses.process.pid": 123}""");
index("test2", """
{"process.parent.command_line":"run.bat"}""");

Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line ASC NULLS FIRST");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item(null).item(null), matchesList().item("run.bat").item("run.bat")))
);

result = runEsql("""
FROM test* | where process.parent.command_line == "run.bat"
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(columnInfo("process.parent.command_line", "keyword"), columnInfo("process.parent.command_line.text", "text"))
).entry("values", List.of(matchesList().item("run.bat").item("run.bat")))
);

ResponseException e = expectThrows(ResponseException.class, () -> runEsql("FROM test* | SORT Responses.process.pid"));
String err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 1:19: Unknown column [Responses.process.pid]"));

e = expectThrows(ResponseException.class, () -> runEsql("""
FROM test*
| SORT Responses.process.pid
| WHERE Responses.process IS NULL
"""));
err = EntityUtils.toString(e.getResponse().getEntity());
assertThat(err, containsString("line 2:8: Unknown column [Responses.process.pid]"));
}

public void testOneNestedField_AndSameNameSupportedField_TwoIndices() throws IOException {
assumeIndexResolverNestedFieldsNameClashFixed();
ESRestTestCase.createIndex("test1", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "nested",
"properties": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
ESRestTestCase.createIndex("test2", Settings.EMPTY, """
"properties": {
"Responses": {
"properties": {
"process": {
"type": "integer",
"fields": {
"pid": {
"type": "long"
}
}
}
}
},
"process": {
"properties": {
"parent": {
"properties": {
"command_line": {
"type": "wildcard",
"fields": {
"text": {
"type": "text"
}
}
}
}
}
}
}
}
""");
index("test1", """
{"Responses.process.pid": 111,"process.parent.command_line":"run1.bat"}""");
index("test2", """
{"Responses.process": 222,"process.parent.command_line":"run2.bat"}""");

Map<String, Object> result = runEsql("FROM test* | SORT process.parent.command_line");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
)
.entry(
"values",
List.of(
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
)
)
);

result = runEsql("""
FROM test* | where Responses.process.pid == 111
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
).entry("values", List.of())
);

result = runEsql("FROM test* | SORT process.parent.command_line");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
)
.entry(
"values",
List.of(
matchesList().item(null).item(null).item("run1.bat").item("run1.bat"),
matchesList().item(222).item(222).item("run2.bat").item("run2.bat")
)
)
);

result = runEsql("""
FROM test*
| SORT process.parent.command_line
| WHERE Responses.process IS NULL
""");
assertMap(
result,
matchesMapWithOptionalTook(result.get("took")).entry(
"columns",
List.of(
columnInfo("Responses.process", "integer"),
columnInfo("Responses.process.pid", "long"),
columnInfo("process.parent.command_line", "keyword"),
columnInfo("process.parent.command_line.text", "text")
)
).entry("values", List.of(matchesList().item(null).item(null).item("run1.bat").item("run1.bat")))
);
}

private void assumeIndexResolverNestedFieldsNameClashFixed() throws IOException {
// especially for BWC tests but also for regular tests
var capsName = EsqlCapabilities.Cap.FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER.name().toLowerCase(Locale.ROOT);
boolean requiredClusterCapability = clusterHasCapability("POST", "/_query", List.of(), List.of(capsName)).orElse(false);
assumeTrue(
"This test makes sense for versions that have the fix for https://github.com/elastic/elasticsearch/issues/117054",
requiredClusterCapability
);
}

private CheckedConsumer<XContentBuilder, IOException> empNoInObject(String empNoType) {
return index -> {
index.startObject("properties");
Expand Down
7 changes: 6 additions & 1 deletion x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,7 +504,12 @@ public enum Cap {
/**
* LOOKUP JOIN
*/
JOIN_LOOKUP(Build.current().isSnapshot());
JOIN_LOOKUP(Build.current().isSnapshot()),

/**
* Fix for https://github.com/elastic/elasticsearch/issues/117054
*/
FIX_NESTED_FIELDS_NAME_CLASH_IN_INDEXRESOLVER;

private final boolean enabled;

Expand Down
Loading