diff --git a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java index dc54e8149e9e0..2095517a1724c 100644 --- a/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java +++ b/server/src/main/java/org/elasticsearch/common/settings/IndexScopedSettings.java @@ -197,6 +197,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings { IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING, IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS, IndexSettings.LOGSDB_SORT_ON_HOST_NAME, + IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE, IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD, IndexSettings.PREFER_ILM_SETTING, DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING, diff --git a/server/src/main/java/org/elasticsearch/index/IndexSettings.java b/server/src/main/java/org/elasticsearch/index/IndexSettings.java index 50f3925c425e7..0f04c93f12a40 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSettings.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSettings.java @@ -668,6 +668,14 @@ public boolean isES87TSDBCodecEnabled() { Property.Final ); + public static final Setting LOGSDB_SORT_ON_MESSAGE_TEMPLATE = Setting.boolSetting( + "index.logsdb.sort_on_message_template", + false, + Property.IndexScope, + Property.PrivateIndex, + Property.Final + ); + public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled(); public static final Setting USE_DOC_VALUES_SKIPPER = Setting.boolSetting( "index.mapping.use_doc_values_skipper", diff --git a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java index 8235b94b4c3a9..fd445d470837c 100644 --- a/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java +++ b/server/src/main/java/org/elasticsearch/index/IndexSortConfig.java @@ -26,6 +26,7 @@ import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.search.sort.SortOrder; +import java.util.ArrayList; import java.util.Arrays; import java.util.EnumSet; import java.util.List; @@ -106,19 +107,21 @@ public final class IndexSortConfig { ); public static class IndexSortConfigDefaults { - public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + public static final FieldSortSpec[] TIME_SERIES_SORT, HOSTNAME_TIMESTAMP_BWC_SORT; + + private static final FieldSortSpec HOSTNAME_SPEC, MESSAGE_PATTERN_SPEC, TIMESTAMP_SPEC; static { - FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); - timeStampSpec.order = SortOrder.DESC; - TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec }; - TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec }; + TIMESTAMP_SPEC = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH); + TIMESTAMP_SPEC.order = SortOrder.DESC; + TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), TIMESTAMP_SPEC }; + + HOSTNAME_SPEC = new FieldSortSpec(IndexMode.HOST_NAME); + HOSTNAME_SPEC.order = SortOrder.ASC; + HOSTNAME_SPEC.missingValue = "_last"; + HOSTNAME_SPEC.mode = MultiValueMode.MIN; - FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME); - hostnameSpec.order = SortOrder.ASC; - hostnameSpec.missingValue = "_last"; - hostnameSpec.mode = MultiValueMode.MIN; - HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec }; + MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id"); // Older indexes use ascending ordering for host name and timestamp. HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] { @@ -148,7 +151,17 @@ public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) { IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT, IndexVersions.UPGRADE_TO_LUCENE_10_0_0 )) { - return (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT; + + List sortSpecs = new ArrayList<>(3); + if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) { + sortSpecs.add(HOSTNAME_SPEC); + } + if (IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings)) { + sortSpecs.add(MESSAGE_PATTERN_SPEC); + } + sortSpecs.add(TIMESTAMP_SPEC); + + return sortSpecs.toArray(FieldSortSpec[]::new); } else { return HOSTNAME_TIMESTAMP_BWC_SORT; } diff --git a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java index 6c3103e4a1f35..76e8412953006 100644 --- a/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java +++ b/server/src/test/java/org/elasticsearch/index/IndexSortSettingsTests.java @@ -332,6 +332,51 @@ public void testLogsdbIndexSortWithHostname() { assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); } + public void testLogsdbIndexSortWithMessage() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(2)); + + assertThat(config.sortSpecs[0].field, equalTo("message.template_id")); + assertThat(config.sortSpecs[1].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX)); + } + + public void testLogsdbIndexSortWithMessageAndHostname() { + Settings settings = Settings.builder() + .put(IndexSettings.MODE.getKey(), "logsdb") + .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) + .build(); + IndexSettings indexSettings = indexSettings(settings); + IndexSortConfig config = indexSettings.getIndexSortConfig(); + assertTrue(config.hasIndexSort()); + assertThat(config.sortSpecs.length, equalTo(3)); + + assertThat(config.sortSpecs[0].field, equalTo("host.name")); + assertThat(config.sortSpecs[1].field, equalTo("message.template_id")); + assertThat(config.sortSpecs[2].field, equalTo("@timestamp")); + assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[1].order, equalTo(SortOrder.ASC)); + assertThat(config.sortSpecs[2].order, equalTo(SortOrder.DESC)); + assertThat(config.sortSpecs[0].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[1].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[2].missingValue, equalTo("_last")); + assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MIN)); + assertThat(config.sortSpecs[2].mode, equalTo(MultiValueMode.MAX)); + } + public void testLogsdbIndexSortTimestampOnly() { Settings settings = Settings.builder() .put(IndexSettings.MODE.getKey(), "logsdb") diff --git a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java index 75f7a41db9d26..8f70622722682 100644 --- a/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java +++ b/x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java @@ -76,6 +76,89 @@ private DocWithId doc(String source) { return new DocWithId(Integer.toString(id++), source); } + public void testHostnameMessageTimestampSortConfig() throws IOException { + final String dataStreamName = "test-logsdb-sort-hostname-message-timestamp"; + + final String mapping = """ + { + "_doc": { + "properties": { + "@timestmap": { + "type": "date" + }, + "host.name": { + "type": "keyword" + }, + "message": { + "type": "pattern_text" + }, + "test_id": { + "type": "text", + "store": true + } + } + } + } + """; + + final DocWithId[] orderedDocs = { + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"bar 7\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"foo 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 2\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 7\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"bar 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"foo 9\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"test_id\": \"%id%\"}"), + doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"test_id\": \"%id%\"}") }; + + createDataStream(dataStreamName, mapping); + + List shuffledDocs = shuffledList(Arrays.asList(orderedDocs)); + indexDocuments(dataStreamName, shuffledDocs); + + Index backingIndex = getBackingIndex(dataStreamName); + + var featureService = getInstanceFromNode(FeatureService.class); + if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) { + assertSettings(backingIndex, settings -> { + assertThat( + IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings), + equalTo(List.of("host.name", "message.template_id", "@timestamp")) + ); + assertThat( + IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings), + equalTo(List.of(SortOrder.ASC, SortOrder.ASC, SortOrder.DESC)) + ); + assertThat( + IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings), + equalTo(List.of(MultiValueMode.MIN, MultiValueMode.MIN, MultiValueMode.MAX)) + ); + assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last", "_last", "_last"))); + }); + } + + assertOrder(backingIndex, orderedDocs); + } + public void testHostnameTimestampSortConfig() throws IOException { final String dataStreamName = "test-logsdb-sort-hostname-timestamp"; diff --git a/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java new file mode 100644 index 0000000000000..b1e745c20830b --- /dev/null +++ b/x-pack/plugin/logsdb/src/javaRestTest/java/org/elasticsearch/xpack/logsdb/patterntext/PatternTextBasicLicenseIT.java @@ -0,0 +1,144 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.logsdb.patterntext; + +import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.hasEntry; + +public class PatternTextBasicLicenseIT extends DataStreamLicenseChangeTestCase { + @Before + public void checkClusterFeature() { + assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text")); + } + + @SuppressWarnings("unchecked") + public void testStandardIndex() throws IOException { + final String dataStreamName = "test-foo"; + + final String mappingStr = """ + { + "index_patterns": ["%name%"], + "priority": 500, + "data_stream": {}, + "template": { + "mappings": { + "properties": { + "pattern_field": { + "type": "pattern_text" + } + } + } + } + }"""; + + final String doc = """ + {"index": {}} + {"@timestamp": "2025-01-01T00:00:00.000Z", "pattern_field": "foo"} + """; + + assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName))); + assertOK(bulkIndex(client(), dataStreamName, () -> doc)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex0); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "pattern_field" + ); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + + assertOK(rolloverDataStream(client(), dataStreamName)); + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); + Map mapping = getMapping(client(), backingIndex1); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "pattern_field" + ); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + } + + @SuppressWarnings("unchecked") + public void testLogsdbIndex() throws IOException { + final String dataStreamName = "test-bar"; + + final String mappingStr = """ + { + "index_patterns": ["%name%"], + "priority": 500, + "data_stream": {}, + "template": { + "settings": { + "index.mode": "logsdb" + }, + "mappings": { + "properties": { + "@timestamp": { + "type": "date" + }, + "message": { + "type": "pattern_text" + }, + "host.name": { + "type": "keyword" + } + } + } + } + } + """; + + final String doc = """ + {"index": {}} + {"@timestamp": "2025-01-01T00:00:00.000Z", "message": "foo 123", "host.name": "bar"} + """; + + assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName))); + assertOK(bulkIndex(client(), dataStreamName, () -> doc)); + + String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0); + { + assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating")); + assertEquals( + List.of("host.name", "message.template_id", "@timestamp"), + getSetting(client(), backingIndex0, "index.sort.field") + ); + Map mapping = getMapping(client(), backingIndex0); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "message" + ); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + + assertOK(rolloverDataStream(client(), dataStreamName)); + + String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1); + { + assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating")); + assertEquals( + List.of("host.name", "message.template_id", "@timestamp"), + getSetting(client(), backingIndex1, "index.sort.field") + ); + Map mapping = getMapping(client(), backingIndex1); + Map patternFieldMapping = (Map) ((Map) mapping.get("properties")).get( + "message" + ); + assertThat(patternFieldMapping, hasEntry("disable_templating", true)); + } + } +} diff --git a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java index ad8385a2838b2..c1b2ea52fc8c7 100644 --- a/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java +++ b/x-pack/plugin/logsdb/src/main/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProvider.java @@ -57,10 +57,13 @@ final class LogsdbIndexModeSettingsProvider implements IndexSettingProvider { private static final Logger LOGGER = LogManager.getLogger(LogsdbIndexModeSettingsProvider.class); static final String LOGS_PATTERN = "logs-*-*"; - private static final Set MAPPING_INCLUDES = Set.of("_source.*", "properties.host**", "properties.resource**", "subobjects") - .stream() - .flatMap(v -> Stream.of(v, "_doc." + v)) - .collect(Collectors.toSet()); + private static final Set MAPPING_INCLUDES = Set.of( + "_source.*", + "properties.host**", + "properties.resource**", + "subobjects", + "properties.message**" + ).stream().flatMap(v -> Stream.of(v, "_doc." + v)).collect(Collectors.toSet()); private final LogsdbLicenseService licenseService; private final SetOnce> mapperServiceFactory = new SetOnce<>(); @@ -154,6 +157,10 @@ && matchesLogsPattern(dataStreamName)) { additionalSettings.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true); } + if (mappingHints.sortOnMessageTemplate) { + additionalSettings.put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true); + } + // Inject routing path matching sort fields. if (settings.getAsBoolean(IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS.getKey(), false)) { if (supportFallbackLogsdbRouting.get() == false || licenseService.allowLogsdbRoutingOnSortField(isTemplateValidation)) { @@ -202,8 +209,14 @@ && matchesLogsPattern(dataStreamName)) { } } - record MappingHints(boolean hasSyntheticSourceUsage, boolean sortOnHostName, boolean addHostNameField, boolean maybeUsesPatternText) { - static MappingHints EMPTY = new MappingHints(false, false, false, false); + record MappingHints( + boolean hasSyntheticSourceUsage, + boolean sortOnHostName, + boolean addHostNameField, + boolean sortOnMessageTemplate, + boolean maybeUsesPatternText + ) { + static MappingHints EMPTY = new MappingHints(false, false, false, false, false); } private static boolean matchesLogsPattern(final String name) { @@ -242,12 +255,13 @@ MappingHints getMappingHints( hasSyntheticSourceUsage = sourceMode == SourceFieldMapper.Mode.SYNTHETIC; if (IndexSortConfig.INDEX_SORT_FIELD_SETTING.exists(indexTemplateAndCreateRequestSettings)) { // Custom sort config, no point for further checks on [host.name] field. - return new MappingHints(hasSyntheticSourceUsage, false, false, true); + return new MappingHints(hasSyntheticSourceUsage, false, false, false, true); } if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(indexTemplateAndCreateRequestSettings) - && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings)) { + && IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings) + && IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(indexTemplateAndCreateRequestSettings)) { // Settings for adding and sorting on [host.name] are already set, propagate them. - return new MappingHints(hasSyntheticSourceUsage, true, true, true); + return new MappingHints(hasSyntheticSourceUsage, true, true, true, true); } } @@ -281,6 +295,7 @@ MappingHints getMappingHints( } mapperService.merge(MapperService.SINGLE_MAPPING_NAME, combinedTemplateMappings, MapperService.MergeReason.INDEX_TEMPLATE); Mapper hostName = mapperService.mappingLookup().getMapper("host.name"); + Mapper messageField = mapperService.mappingLookup().getMapper("message"); hasSyntheticSourceUsage = hasSyntheticSourceUsage || mapperService.documentMapper().sourceMapper().isSynthetic(); boolean addHostNameField = IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(indexTemplateAndCreateRequestSettings) || (hostName == null @@ -291,7 +306,15 @@ MappingHints getMappingHints( || addHostNameField || (hostName instanceof NumberFieldMapper nfm && nfm.fieldType().hasDocValues()) || (hostName instanceof KeywordFieldMapper kfm && kfm.fieldType().hasDocValues()); - return new MappingHints(hasSyntheticSourceUsage, sortOnHostName, addHostNameField, maybeUsesPatternText); + boolean sortOnMessageTemplate = IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(indexTemplateAndCreateRequestSettings) + || (messageField instanceof PatternTextFieldMapper); + return new MappingHints( + hasSyntheticSourceUsage, + sortOnHostName, + addHostNameField, + sortOnMessageTemplate, + maybeUsesPatternText + ); } } catch (AssertionError | Exception e) { // In case invalid mappings or setting are provided, then mapper service creation can fail. diff --git a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java index d8a229877b686..2bad83d5a2474 100644 --- a/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java +++ b/x-pack/plugin/logsdb/src/test/java/org/elasticsearch/xpack/logsdb/LogsdbIndexModeSettingsProviderTests.java @@ -978,15 +978,9 @@ public void testPatternTextNotAllowedByLicense() throws IOException { "{\"properties\":{\"foo\":{\"type\":\"pattern_text\"}}}", "{\"properties\":{\"bar\":{\"properties\":{\"baz\":{\"type\":\"pattern_text\"}}}}}" }; - var expectedSettings = Settings.builder() - .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) - .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) - .put(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.getKey(), true) - .build(); - for (String mapping : patternTextLicenceCheckedFieldMappings) { var result = generateLogsdbSettings(Settings.EMPTY, getMapping(mapping), Version.CURRENT, basicLogsdbLicenseService); - assertEquals(expectedSettings, result); + assertTrue(PatternTextFieldMapper.DISABLE_TEMPLATING_SETTING.get(result)); } } @@ -1006,10 +1000,12 @@ public void testSortAndHostNamePropagateValue() throws Exception { .put(IndexSettings.MODE.getKey(), IndexMode.LOGSDB) .put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true) .put(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.getKey(), true) + .put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true) .build(); Settings result = generateLogsdbSettings(settings); assertTrue(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(result)); assertTrue(IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD.get(result)); + assertTrue(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(result)); assertEquals(0, newMapperServiceCounter.get()); } diff --git a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml index 86497444cd8ae..0604afb6ec45b 100644 --- a/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml +++ b/x-pack/plugin/logsdb/src/yamlRestTest/resources/rest-api-spec/test/80_index_sort_defaults.yml @@ -129,6 +129,116 @@ create logsdb data stream with host.name as keyword and timestamp as date: - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } +--- +create logsdb data stream with message as pattern text, and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + message: + type: pattern_text + host: + type: keyword + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: null } + - match: { .$backing_index.defaults.index.logsdb.sort_on_host_name: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_message_template: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "message.template_id", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last" ] } + + +--- +create logsdb data stream with host.name as keyword, message as pattern text, and timestamp as date: + - do: + cluster.put_component_template: + name: "logsdb-mappings" + body: + template: + settings: + mode: "logsdb" + mappings: + properties: + host.name: + type: "keyword" + message: + type: pattern_text + "@timestamp": + type: "date" + + - do: + indices.put_index_template: + name: "logsdb-index-template" + body: + index_patterns: ["logsdb"] + data_stream: {} + composed_of: ["logsdb-mappings"] + allowed_warnings: + - "index template [logsdb-index-template] has index patterns [logsdb] matching patterns from existing older templates [global] with patterns (global => [*]); this template [logsdb-index-template] will take precedence during new index creation" + + - do: + indices.create_data_stream: + name: "logsdb" + + - is_true: acknowledged + - do: + indices.get_data_stream: + name: "logsdb" + expand_wildcards: hidden + - length: { data_streams: 1 } + - set: { data_streams.0.indices.0.index_name: backing_index } + + - do: + indices.get_settings: + index: $backing_index + include_defaults: true + - match: { .$backing_index.settings.index.mode: logsdb } + - match: { .$backing_index.settings.index.logsdb.add_host_name_field: null } + - match: { .$backing_index.defaults.index.logsdb.add_host_name_field: "false" } + - match: { .$backing_index.settings.index.logsdb.sort_on_host_name: "true" } + - match: { .$backing_index.settings.index.logsdb.sort_on_message_template: "true" } + - match: { .$backing_index.defaults.index.sort.field: [ "host.name", "message.template_id", "@timestamp" ] } + - match: { .$backing_index.defaults.index.sort.order: [ "asc", "asc", "desc" ] } + - match: { .$backing_index.defaults.index.sort.mode: [ "min", "min", "max" ] } + - match: { .$backing_index.defaults.index.sort.missing: [ "_last", "_last", "_last" ] } + --- create logsdb data stream with host.name as integer and timestamp as date: - do: