Switch default logs template to search all fields by default (#102456)

elastic · Nov 29, 2023 · 038a852 · 038a852
1 parent 9054492
commit 038a852
Show file tree

Hide file tree

Showing 4 changed files with 104 additions and 18 deletions.
diff --git a/docs/changelog/102456.yaml b/docs/changelog/102456.yaml
@@ -0,0 +1,6 @@
+pr: 102456
+summary: Switch logs data streams to search all fields by default
+area: Data streams
+type: enhancement
+issues:
+ - 99872
diff --git a/...es/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java b/...es/data-streams/src/javaRestTest/java/org/elasticsearch/datastreams/LogsDataStreamIT.java
@@ -12,6 +12,7 @@
 import org.elasticsearch.client.ResponseException;
 import org.elasticsearch.client.RestClient;
 import org.junit.After;
+import org.junit.Before;
 
 import java.io.IOException;
 import java.util.List;
@@ -25,16 +26,21 @@
 
 public class LogsDataStreamIT extends DisabledSecurityDataStreamTestCase {
 
+    private RestClient client;
+
+    @Before
+    public void setup() throws Exception {
+        client = client();
+        waitForLogs(client);
+    }
+
     @After
     public void cleanUp() throws IOException {
         adminClient().performRequest(new Request("DELETE", "_data_stream/*"));
     }
 
     @SuppressWarnings("unchecked")
     public void testDefaultLogsSettingAndMapping() throws Exception {
-        RestClient client = client();
-        waitForLogs(client);
-
         String dataStreamName = "logs-generic-default";
         createDataStream(client, dataStreamName);
         String backingIndex = getWriteBackingIndex(client, dataStreamName);
@@ -104,9 +110,6 @@ public void testDefaultLogsSettingAndMapping() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testCustomMapping() throws Exception {
-        RestClient client = client();
-        waitForLogs(client);
-
         {
             Request request = new Request("POST", "/_component_template/logs@custom");
             request.setJsonEntity("""
@@ -182,9 +185,6 @@ public void testCustomMapping() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testLogsDefaultPipeline() throws Exception {
-        RestClient client = client();
-        waitForLogs(client);
-
         {
             Request request = new Request("POST", "/_component_template/logs@custom");
             request.setJsonEntity("""
@@ -284,9 +284,6 @@ public void testLogsDefaultPipeline() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testLogsMessagePipeline() throws Exception {
-        RestClient client = client();
-        waitForLogs(client);
-
         {
             Request request = new Request("PUT", "/_ingest/pipeline/logs@custom");
             request.setJsonEntity("""
@@ -412,8 +409,6 @@ public void testLogsMessagePipeline() throws Exception {
 
     @SuppressWarnings("unchecked")
     public void testNoSubobjects() throws Exception {
-        RestClient client = client();
-        waitForLogs(client);
         {
             Request request = new Request("POST", "/_component_template/logs-test-subobjects-mappings");
             request.setJsonEntity("""
@@ -633,6 +628,94 @@ public void testNoSubobjects() throws Exception {
 
     }
 
+    public void testAllFieldsAreSearchableByDefault() throws Exception {
+        final String dataStreamName = "logs-generic-default";
+        createDataStream(client, dataStreamName);
+
+        // index a doc with "message" field and an additional one that will be mapped to a "match_only_text" type
+        indexDoc(client, dataStreamName, """
+            {
+              "@timestamp": "2023-04-18",
+              "message": "Hello world",
+              "another.message": "Hi world"
+            }
+            """);
+
+        // verify that both fields are searchable when not querying specific fields
+        List<Object> results = searchDocs(client, dataStreamName, """
+            {
+              "query": {
+                "simple_query_string": {
+                  "query": "Hello"
+                }
+              }
+            }
+            """);
+        assertEquals(1, results.size());
+
+        results = searchDocs(client, dataStreamName, """
+            {
+              "query": {
+                "simple_query_string": {
+                  "query": "Hi"
+                }
+              }
+            }
+            """);
+        assertEquals(1, results.size());
+    }
+
+    public void testDefaultFieldCustomization() throws Exception {
+        Request request = new Request("POST", "/_component_template/logs@custom");
+        request.setJsonEntity("""
+            {
+              "template": {
+                "settings": {
+                  "index": {
+                    "query": {
+                      "default_field": ["message"]
+                    }
+                  }
+                }
+              }
+            }
+            """);
+        assertOK(client.performRequest(request));
+
+        final String dataStreamName = "logs-generic-default";
+        createDataStream(client, dataStreamName);
+
+        indexDoc(client, dataStreamName, """
+            {
+              "@timestamp": "2023-04-18",
+              "message": "Hello world",
+              "another.message": "Hi world"
+            }
+            """);
+
+        List<Object> results = searchDocs(client, dataStreamName, """
+            {
+              "query": {
+                "simple_query_string": {
+                  "query": "Hello"
+                }
+              }
+            }
+            """);
+        assertEquals(1, results.size());
+
+        results = searchDocs(client, dataStreamName, """
+            {
+              "query": {
+                "simple_query_string": {
+                  "query": "Hi"
+                }
+              }
+            }
+            """);
+        assertEquals(0, results.size());
+    }
+
     static void waitForLogs(RestClient client) throws Exception {
         assertBusy(() -> {
             try {

diff --git a/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/rest/ESRestTestCase.java
@@ -1926,7 +1926,7 @@ protected static boolean isXPackTemplate(String name) {
         if (name.startsWith("elastic-connectors")) {
             return true;
         }
-        if (name.contains("@")) {
+        if (name.contains("@") && name.endsWith("@custom") == false) {
             // We have a naming convention that internal component templates contain `@`. See also index-templates.asciidoc.
             return true;
         }

diff --git a/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json b/x-pack/plugin/core/template-resources/src/main/resources/logs@settings.json
@@ -6,9 +6,6 @@
           "name": "logs"
         },
         "codec": "best_compression",
-        "query": {
-          "default_field": ["message"]
-        },
         "mapping": {
           "ignore_malformed": true
         },