Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,7 @@ public final class IndexScopedSettings extends AbstractScopedSettings {
IndexSettings.TIME_SERIES_ES87TSDB_CODEC_ENABLED_SETTING,
IndexSettings.LOGSDB_ROUTE_ON_SORT_FIELDS,
IndexSettings.LOGSDB_SORT_ON_HOST_NAME,
IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE,
IndexSettings.LOGSDB_ADD_HOST_NAME_FIELD,
IndexSettings.PREFER_ILM_SETTING,
DataStreamFailureStoreDefinition.FAILURE_STORE_DEFINITION_VERSION_SETTING,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -668,6 +668,14 @@ public boolean isES87TSDBCodecEnabled() {
Property.Final
);

public static final Setting<Boolean> LOGSDB_SORT_ON_MESSAGE_TEMPLATE = Setting.boolSetting(
"index.logsdb.sort_on_message_template",
false,
Property.IndexScope,
Property.PrivateIndex,
Property.Final
);

public static final boolean DOC_VALUES_SKIPPER = new FeatureFlag("doc_values_skipper").isEnabled();
public static final Setting<Boolean> USE_DOC_VALUES_SKIPPER = Setting.boolSetting(
"index.mapping.use_doc_values_skipper",
Expand Down
35 changes: 24 additions & 11 deletions server/src/main/java/org/elasticsearch/index/IndexSortConfig.java
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import org.elasticsearch.search.lookup.SearchLookup;
import org.elasticsearch.search.sort.SortOrder;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
import java.util.List;
Expand Down Expand Up @@ -106,19 +107,21 @@ public final class IndexSortConfig {
);

public static class IndexSortConfigDefaults {
public static final FieldSortSpec[] TIME_SERIES_SORT, TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_SORT, HOSTNAME_TIMESTAMP_BWC_SORT;
public static final FieldSortSpec[] TIME_SERIES_SORT, HOSTNAME_TIMESTAMP_BWC_SORT;

private static final FieldSortSpec HOSTNAME_SPEC, MESSAGE_PATTERN_SPEC, TIMESTAMP_SPEC;

static {
FieldSortSpec timeStampSpec = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH);
timeStampSpec.order = SortOrder.DESC;
TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), timeStampSpec };
TIMESTAMP_SORT = new FieldSortSpec[] { timeStampSpec };
TIMESTAMP_SPEC = new FieldSortSpec(DataStreamTimestampFieldMapper.DEFAULT_PATH);
TIMESTAMP_SPEC.order = SortOrder.DESC;
TIME_SERIES_SORT = new FieldSortSpec[] { new FieldSortSpec(TimeSeriesIdFieldMapper.NAME), TIMESTAMP_SPEC };

HOSTNAME_SPEC = new FieldSortSpec(IndexMode.HOST_NAME);
HOSTNAME_SPEC.order = SortOrder.ASC;
HOSTNAME_SPEC.missingValue = "_last";
HOSTNAME_SPEC.mode = MultiValueMode.MIN;

FieldSortSpec hostnameSpec = new FieldSortSpec(IndexMode.HOST_NAME);
hostnameSpec.order = SortOrder.ASC;
hostnameSpec.missingValue = "_last";
hostnameSpec.mode = MultiValueMode.MIN;
HOSTNAME_TIMESTAMP_SORT = new FieldSortSpec[] { hostnameSpec, timeStampSpec };
MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id");

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

medium

For consistency and readability, it would be better to explicitly define the properties for MESSAGE_PATTERN_SPEC, similar to how HOSTNAME_SPEC is defined. While the default values would be correctly applied, being explicit makes the configuration clearer. You can use a constructor overload and set the mode property to make it compact.

Suggested change
MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id");
MESSAGE_PATTERN_SPEC = new FieldSortSpec("message.template_id", SortOrder.ASC, "_last");
MESSAGE_PATTERN_SPEC.mode = MultiValueMode.MIN;

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Magic String Constant

Hardcoded string "message.template_id" creates coupling and maintenance risk. Field name changes require hunting through code. Extract to named constant for centralized management and refactoring safety.

Standards
  • Clean-Code-Constants
  • Maintainability-Quality-Coupling
  • Refactoring-Extract-Constant


// Older indexes use ascending ordering for host name and timestamp.
HOSTNAME_TIMESTAMP_BWC_SORT = new FieldSortSpec[] {
Expand Down Expand Up @@ -148,7 +151,17 @@ public static FieldSortSpec[] getDefaultSortSpecs(Settings settings) {
IndexVersions.LOGSB_OPTIONAL_SORTING_ON_HOST_NAME_BACKPORT,
IndexVersions.UPGRADE_TO_LUCENE_10_0_0
)) {
return (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) ? HOSTNAME_TIMESTAMP_SORT : TIMESTAMP_SORT;

List<FieldSortSpec> sortSpecs = new ArrayList<>(3);
if (IndexSettings.LOGSDB_SORT_ON_HOST_NAME.get(settings)) {
sortSpecs.add(HOSTNAME_SPEC);
}
if (IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.get(settings)) {
sortSpecs.add(MESSAGE_PATTERN_SPEC);
}
sortSpecs.add(TIMESTAMP_SPEC);

return sortSpecs.toArray(FieldSortSpec[]::new);
} else {
return HOSTNAME_TIMESTAMP_BWC_SORT;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,51 @@ public void testLogsdbIndexSortWithHostname() {
assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX));
}

public void testLogsdbIndexSortWithMessage() {
Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), "logsdb")
.put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true)
.build();
IndexSettings indexSettings = indexSettings(settings);
IndexSortConfig config = indexSettings.getIndexSortConfig();
assertTrue(config.hasIndexSort());
assertThat(config.sortSpecs.length, equalTo(2));

assertThat(config.sortSpecs[0].field, equalTo("message.template_id"));
assertThat(config.sortSpecs[1].field, equalTo("@timestamp"));
assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC));
assertThat(config.sortSpecs[1].order, equalTo(SortOrder.DESC));
assertThat(config.sortSpecs[0].missingValue, equalTo("_last"));
assertThat(config.sortSpecs[1].missingValue, equalTo("_last"));
assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN));
assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MAX));
}

public void testLogsdbIndexSortWithMessageAndHostname() {
Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), "logsdb")
.put(IndexSettings.LOGSDB_SORT_ON_HOST_NAME.getKey(), true)
.put(IndexSettings.LOGSDB_SORT_ON_MESSAGE_TEMPLATE.getKey(), true)
.build();
IndexSettings indexSettings = indexSettings(settings);
IndexSortConfig config = indexSettings.getIndexSortConfig();
assertTrue(config.hasIndexSort());
assertThat(config.sortSpecs.length, equalTo(3));

assertThat(config.sortSpecs[0].field, equalTo("host.name"));
assertThat(config.sortSpecs[1].field, equalTo("message.template_id"));
assertThat(config.sortSpecs[2].field, equalTo("@timestamp"));
assertThat(config.sortSpecs[0].order, equalTo(SortOrder.ASC));
assertThat(config.sortSpecs[1].order, equalTo(SortOrder.ASC));
assertThat(config.sortSpecs[2].order, equalTo(SortOrder.DESC));
assertThat(config.sortSpecs[0].missingValue, equalTo("_last"));
assertThat(config.sortSpecs[1].missingValue, equalTo("_last"));
assertThat(config.sortSpecs[2].missingValue, equalTo("_last"));
assertThat(config.sortSpecs[0].mode, equalTo(MultiValueMode.MIN));
assertThat(config.sortSpecs[1].mode, equalTo(MultiValueMode.MIN));
assertThat(config.sortSpecs[2].mode, equalTo(MultiValueMode.MAX));
}

public void testLogsdbIndexSortTimestampOnly() {
Settings settings = Settings.builder()
.put(IndexSettings.MODE.getKey(), "logsdb")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,89 @@ private DocWithId doc(String source) {
return new DocWithId(Integer.toString(id++), source);
}

public void testHostnameMessageTimestampSortConfig() throws IOException {
final String dataStreamName = "test-logsdb-sort-hostname-message-timestamp";

final String mapping = """
{
"_doc": {
"properties": {
"@timestmap": {

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

high

There seems to be a typo in the field name. It should be @timestamp instead of @timestmap.

Suggested change
"@timestmap": {
"@timestamp": {

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Typo Field Reference

Field name contains typo '@timestmap' instead of '@timestamp' in mapping configuration. Incorrect field reference prevents proper timestamp handling causing mapping failures. Index creation reliability affected by configuration validation errors.

Standards
  • ISO-IEC-25010-Functional-Correctness-Appropriateness
  • DbC-Precondition-Validation

"type": "date"
},
"host.name": {
"type": "keyword"
},
"message": {
"type": "pattern_text"
},
"test_id": {
"type": "text",
"store": true
}
}
}
}
""";

final DocWithId[] orderedDocs = {
Comment on lines +82 to +104
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Potential issue | 🟠 Major

Fix mapping typo: @timestmap → @timestamp

The mapping defines "@timestmap" (typo). Use "@timestamp" to align with data stream timestamp and test assertions.

-                  "@timestmap": {
+                  "@timestamp": {
                     "type": "date"
                   },
📝 Committable suggestion

‼️ IMPORTANT
Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

Suggested change
final String mapping = """
{
"_doc": {
"properties": {
"@timestmap": {
"type": "date"
},
"host.name": {
"type": "keyword"
},
"message": {
"type": "pattern_text"
},
"test_id": {
"type": "text",
"store": true
}
}
}
}
""";
final DocWithId[] orderedDocs = {
final String mapping = """
{
"_doc": {
"properties": {
"@timestamp": {
"type": "date"
},
"host.name": {
"type": "keyword"
},
"message": {
"type": "pattern_text"
},
"test_id": {
"type": "text",
"store": true
}
}
}
}
""";
final DocWithId[] orderedDocs = {
🤖 Prompt for AI Agents
In
x-pack/plugin/logsdb/src/internalClusterTest/java/org/elasticsearch/xpack/logsdb/LogsdbSortConfigIT.java
around lines 82 to 104 the mapping contains a typo: the field is defined as
"@timestmap" but should be "@timestamp". Update the mapping string to rename
"@timestmap" to "@timestamp" (keeping type "date") so it matches the data stream
timestamp field used by the test assertions.

doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"bar 7\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"aaa\",\"message\":\"bar 9\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"aaa\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"message\":\"foo 9\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":[\"aaa\",\"bbb\"],\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 5\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"bar 2\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 7\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"message\":\"foo 6\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"host.name\":\"bbb\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"bar 1\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"bar 4\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"message\":\"foo 1\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"message\":\"foo 9\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"message\":\"foo 3\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T13:00:00\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T12:00:00\",\"test_id\": \"%id%\"}"),
doc("{\"@timestamp\":\"2025-01-01T11:00:00\",\"test_id\": \"%id%\"}") };

createDataStream(dataStreamName, mapping);

List<DocWithId> shuffledDocs = shuffledList(Arrays.asList(orderedDocs));
indexDocuments(dataStreamName, shuffledDocs);

Index backingIndex = getBackingIndex(dataStreamName);

var featureService = getInstanceFromNode(FeatureService.class);
if (featureService.getNodeFeatures().containsKey("mapper.provide_index_sort_setting_defaults")) {
assertSettings(backingIndex, settings -> {
assertThat(
IndexSortConfig.INDEX_SORT_FIELD_SETTING.get(settings),
equalTo(List.of("host.name", "message.template_id", "@timestamp"))
);
assertThat(
IndexSortConfig.INDEX_SORT_ORDER_SETTING.get(settings),
equalTo(List.of(SortOrder.ASC, SortOrder.ASC, SortOrder.DESC))
);
assertThat(
IndexSortConfig.INDEX_SORT_MODE_SETTING.get(settings),
equalTo(List.of(MultiValueMode.MIN, MultiValueMode.MIN, MultiValueMode.MAX))
);
assertThat(IndexSortConfig.INDEX_SORT_MISSING_SETTING.get(settings), equalTo(List.of("_last", "_last", "_last")));
});
}

assertOrder(backingIndex, orderedDocs);
}

public void testHostnameTimestampSortConfig() throws IOException {
final String dataStreamName = "test-logsdb-sort-hostname-timestamp";

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
/*
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
* or more contributor license agreements. Licensed under the Elastic License
* 2.0; you may not use this file except in compliance with the Elastic License
* 2.0.
*/

package org.elasticsearch.xpack.logsdb.patterntext;

import org.elasticsearch.xpack.logsdb.DataStreamLicenseChangeTestCase;
import org.junit.Before;

import java.io.IOException;
import java.util.List;
import java.util.Map;

import static org.hamcrest.Matchers.hasEntry;

public class PatternTextBasicLicenseIT extends DataStreamLicenseChangeTestCase {
@Before
public void checkClusterFeature() {
assumeTrue("[patterned_text] must be available", clusterHasFeature("mapper.patterned_text"));
}

@SuppressWarnings("unchecked")
public void testStandardIndex() throws IOException {
final String dataStreamName = "test-foo";

final String mappingStr = """
{
"index_patterns": ["%name%"],
"priority": 500,
"data_stream": {},
"template": {
"mappings": {
"properties": {
"pattern_field": {
"type": "pattern_text"
}
}
}
}
}""";

final String doc = """
{"index": {}}
{"@timestamp": "2025-01-01T00:00:00.000Z", "pattern_field": "foo"}
""";

assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName)));
assertOK(bulkIndex(client(), dataStreamName, () -> doc));

String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0);
{
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}

assertOK(rolloverDataStream(client(), dataStreamName));

String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1);
{
assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating"));
Map<String, Object> mapping = getMapping(client(), backingIndex1);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"pattern_field"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}
}

@SuppressWarnings("unchecked")
public void testLogsdbIndex() throws IOException {
final String dataStreamName = "test-bar";

final String mappingStr = """
{
"index_patterns": ["%name%"],
"priority": 500,
"data_stream": {},
"template": {
"settings": {
"index.mode": "logsdb"
},
"mappings": {
"properties": {
"@timestamp": {
"type": "date"
},
"message": {
"type": "pattern_text"
},
"host.name": {
"type": "keyword"
}
}
}
}
}
""";

final String doc = """
{"index": {}}
{"@timestamp": "2025-01-01T00:00:00.000Z", "message": "foo 123", "host.name": "bar"}
""";

assertOK(putTemplate(client(), "logs@custom", mappingStr.replace("%name%", dataStreamName)));
assertOK(bulkIndex(client(), dataStreamName, () -> doc));

String backingIndex0 = getDataStreamBackingIndex(client(), dataStreamName, 0);
{
assertEquals("true", getSetting(client(), backingIndex0, "index.mapping.pattern_text.disable_templating"));
assertEquals(
List.of("host.name", "message.template_id", "@timestamp"),
getSetting(client(), backingIndex0, "index.sort.field")
);
Map<String, Object> mapping = getMapping(client(), backingIndex0);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"message"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}

assertOK(rolloverDataStream(client(), dataStreamName));

String backingIndex1 = getDataStreamBackingIndex(client(), dataStreamName, 1);
{
assertEquals("true", getSetting(client(), backingIndex1, "index.mapping.pattern_text.disable_templating"));
assertEquals(
List.of("host.name", "message.template_id", "@timestamp"),
getSetting(client(), backingIndex1, "index.sort.field")
);
Map<String, Object> mapping = getMapping(client(), backingIndex1);
Map<String, Object> patternFieldMapping = (Map<String, Object>) ((Map<String, Object>) mapping.get("properties")).get(
"message"
);
assertThat(patternFieldMapping, hasEntry("disable_templating", true));
}
}
}
Loading