Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[test] Small enhancement to EcsDynamicTemplatesIT #110740

Merged
merged 8 commits into from
Jul 26, 2024
Merged

[test] Small enhancement to EcsDynamicTemplatesIT #110740

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
31d3e17
Adjust ecs@mappings to latest ECS and tests to consider normalizer
eyalkoren Jul 11, 2024
1705bf5
Update docs/changelog/110740.yaml
eyalkoren Jul 11, 2024
5613d63
Adjusting the stack registry version
eyalkoren Jul 11, 2024
a240f84
Merge remote-tracking branch 'eyalkoren/ecs-tests-fix' into ecs-tests…
eyalkoren Jul 11, 2024
8e4d18a
Improve PR summary
eyalkoren Jul 11, 2024
a552db7
Removing .caseless fields
eyalkoren Jul 25, 2024
810d91a
Merge remote-tracking branch 'upstream/main' into ecs-tests-fix
eyalkoren Jul 25, 2024
7d46885
Delete docs/changelog/110740.yaml
eyalkoren Jul 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
75 changes: 50 additions & 25 deletions ...ugin/stack/src/javaRestTest/java/org/elasticsearch/xpack/stack/EcsDynamicTemplatesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,7 @@ public class EcsDynamicTemplatesIT extends ESRestTestCase {

private static Map<String, Object> ecsDynamicTemplates;
private static Map<String, Map<String, Object>> ecsFlatFieldDefinitions;
private static Map<String, String> ecsFlatMultiFieldDefinitions;
private static Map<String, Map<String, Object>> ecsFlatMultiFieldDefinitions;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Instead of kipping only the subfield's type, keeping all mappings so we can validate any of them


@BeforeClass
public static void setupSuiteScopeCluster() throws Exception {
Expand Down Expand Up @@ -142,12 +142,11 @@ private static void prepareEcsDefinitions() throws IOException {
iterator.remove();
}

List<Map<String, String>> multiFields = (List<Map<String, String>>) definitions.get("multi_fields");
List<Map<String, Object>> multiFields = (List<Map<String, Object>>) definitions.get("multi_fields");
if (multiFields != null) {
multiFields.forEach(multiFieldsDefinitions -> {
String subfieldFlatName = Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
String subfieldType = Objects.requireNonNull(multiFieldsDefinitions.get("type"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, subfieldType);
String subfieldFlatName = (String) Objects.requireNonNull(multiFieldsDefinitions.get("flat_name"));
ecsFlatMultiFieldDefinitions.put(subfieldFlatName, multiFieldsDefinitions);
});
}
}
Expand Down Expand Up @@ -191,7 +190,7 @@ public void testNumericMessage() throws IOException {
verifyEcsMappings(indexName);
}

private void assertType(String expectedType, Map<String, Object> actualMappings) throws IOException {
private void assertType(String expectedType, Map<String, Object> actualMappings) {
assertNotNull("expected to get non-null mappings for field", actualMappings);
assertEquals(expectedType, actualMappings.get("type"));
}
Expand Down Expand Up @@ -411,32 +410,33 @@ private void verifyEcsMappings(String indexName) throws IOException {
if (expectedMappings == null) {
nonEcsFields.add(fieldName);
} else {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields
);
}
});

Map<String, String> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
Map<String, Map<String, Object>> shallowMultiFieldMapCopy = new HashMap<>(ecsFlatMultiFieldDefinitions);
logger.info("Testing mapping of {} ECS multi-fields", shallowMultiFieldMapCopy.size());
flatMultiFieldsMappings.forEach((fieldName, actualMappings) -> {
String expectedType = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedType != null) {
Map<String, Object> expectedMultiFieldMappings = shallowMultiFieldMapCopy.remove(fieldName);
if (expectedMultiFieldMappings != null) {
// not finding an entry in the expected multi-field mappings map is acceptable: our dynamic templates are required to
// ensure multi-field mapping for all fields with such ECS definitions. However, the patterns in these templates may lead
// to multi-field mapping for ECS fields for which such are not defined
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
compareExpectedToActualMappings(
fieldName,
actualMappings,
expectedMultiFieldMappings,
fieldToWrongMappingType,
wronglyIndexedFields,
wronglyDocValuedFields
);
}
});

Expand All @@ -460,7 +460,11 @@ private void verifyEcsMappings(String indexName) throws IOException {
);
});
fieldToWrongMappingType.forEach((fieldName, actualMappingType) -> {
String ecsExpectedType = (String) ecsFlatFieldDefinitions.get(fieldName).get("type");
Map<String, Object> fieldMappings = ecsFlatFieldDefinitions.get(fieldName);
if (fieldMappings == null) {
fieldMappings = ecsFlatMultiFieldDefinitions.get(fieldName);
}
Comment on lines +464 to +466
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The field mappings are either in the top-level mappings or subfields map. We didn't look in the latter before thus error message for subfields was not as descriptive

String ecsExpectedType = (String) fieldMappings.get("type");
logger.error(
"ECS field '{}' should be mapped to type '{}' but is mapped to type '{}'. Update {} accordingly.",
fieldName,
Expand Down Expand Up @@ -493,4 +497,25 @@ private void verifyEcsMappings(String indexName) throws IOException {
wronglyDocValuedFields.isEmpty()
);
}

private static void compareExpectedToActualMappings(
String fieldName,
Map<String, Object> actualMappings,
Map<String, Object> expectedMappings,
Map<String, String> fieldToWrongMappingType,
List<String> wronglyIndexedFields,
List<String> wronglyDocValuedFields
) {
String expectedType = (String) expectedMappings.get("type");
String actualMappingType = (String) actualMappings.get("type");
if (actualMappingType.equals(expectedType) == false) {
fieldToWrongMappingType.put(fieldName, actualMappingType);
}
if (expectedMappings.get("index") != actualMappings.get("index")) {
wronglyIndexedFields.add(fieldName);
}
if (expectedMappings.get("doc_values") != actualMappings.get("doc_values")) {
wronglyDocValuedFields.add(fieldName);
}
}
}