make true the default, and update 7.0 referernces

docs/reference/ingest/processors/user-agent.asciidoc

@@ -19,7 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab
 | `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
 | `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
-| `ecs`                  | no        | `false`                                                                                         | Whether to return the output in Elastic Common Schema format. NOTE: ECS format will be the default in Elasticsearch 7.0 and non-ECS format is deprecated.
+| `ecs`                  | no        | `true`                                                                                         | Whether to return the output in Elastic Common Schema format. NOTE: This setting is deprecated and will be removed in a future version.
 |======
 
 Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
@@ -32,8 +32,7 @@ PUT _ingest/pipeline/user_agent
   "processors" : [
     {
       "user_agent" : {
-        "field" : "agent",
-        "ecs" : true
+        "field" : "agent"
       }
     }
   ]

docs/reference/migration/migrate_7_0/settings.asciidoc

@@ -198,3 +198,9 @@ could have lead to dropping audit events while the operations on the system
 were allowed to continue as usual. The recommended replacement is the
 use of the `logfile` audit output type and using other components from the
 Elastic Stack to handle the indexing part.
+
+[float]
+[[ingest-user-agent-ecs-always]]
+==== Ingest User Agent processor defaults uses `ecs` output format
+https://github.com/elastic/ecs[ECS] format is now the default.
+The `ecs` setting for the user agent ingest processor now defaults to true.

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -150,7 +150,7 @@ public IngestDocument execute(IngestDocument ingestDocument) {
                 }
             }
         } else {
-            // Deprecated format, removed in 7.0
+            // Deprecated format, removed in 8.0
             for (Property property : this.properties) {
                 switch (property) {
                     case NAME:
@@ -292,7 +292,7 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", false);
+            boolean useECS = readBooleanProperty(TYPE, processorTag, config, "ecs", true);
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
@@ -316,7 +316,7 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
 
             if (useECS == false) {
                 deprecationLogger.deprecated("setting [ecs] to false for non-common schema " +
-                    "format is deprecated and will be removed in 7.0, set to true to use the non-deprecated format");
+                    "format is deprecated and will be removed in 8.0, set to true or remove to use the non-deprecated format");
             }
 
             return new UserAgentProcessor(processorTag, field, targetField, parser, properties, ignoreMissing, useECS);
@@ -326,12 +326,12 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
     enum Property {
 
         NAME,
-        // Deprecated in 6.7 (superceded by VERSION), to be removed in 7.0
+        // Deprecated in 6.7 (superceded by VERSION), to be removed in 8.0
         @Deprecated MAJOR,
         @Deprecated MINOR,
         @Deprecated PATCH,
         OS,
-        // Deprecated in 6.7 (superceded by just using OS), to be removed in 7.0
+        // Deprecated in 6.7 (superceded by just using OS), to be removed in 8.0
         @Deprecated OS_NAME,
         @Deprecated OS_MAJOR,
         @Deprecated OS_MINOR,

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -83,14 +83,12 @@ public void testBuildDefaults() throws Exception {
 
         Map<String, Object> config = new HashMap<>();
         config.put("field", "_field");
-        config.put("ecs", true);
 
         String processorTag = randomAlphaOfLength(10);
 
         UserAgentProcessor processor = factory.create(null, processorTag, config);
         assertThat(processor.getTag(), equalTo(processorTag));
         assertThat(processor.getField(), equalTo("_field"));
-        assertThat(processor.getTargetField(), equalTo("user_agent"));
         assertThat(processor.getUaParser().getUaPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getOsPatterns().size(), greaterThan(0));
         assertThat(processor.getUaParser().getDevicePatterns().size(), greaterThan(0));

x-pack/plugin/deprecation/src/main/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecks.java

@@ -7,7 +7,6 @@
 package org.elasticsearch.xpack.deprecation;
 
 import org.elasticsearch.cluster.ClusterState;
-import org.elasticsearch.ingest.ConfigurationUtils;
 import org.elasticsearch.ingest.IngestService;
 import org.elasticsearch.ingest.PipelineConfiguration;
 import org.elasticsearch.xpack.core.deprecation.DeprecationIssue;
@@ -34,18 +33,17 @@ static DeprecationIssue checkUserAgentPipelines(ClusterState state) {
                     .filter(Objects::nonNull)
                     .filter(processor -> processor.containsKey("user_agent"))
                     .map(processor -> processor.get("user_agent"))
-                    .anyMatch(processorConfig ->
-                        false == ConfigurationUtils.readBooleanProperty(null, null, processorConfig, "ecs", false));
+                    .anyMatch(processorConfig -> processorConfig.containsKey("ecs"));
             })
             .map(PipelineConfiguration::getId)
             .sorted() // Make the warning consistent for testing purposes
             .collect(Collectors.toList());
         if (pipelinesWithDeprecatedEcsConfig.isEmpty() == false) {
             return new DeprecationIssue(DeprecationIssue.Level.WARNING,
-                "User-Agent ingest plugin will use ECS-formatted output",
-                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
+                "User-Agent ingest plugin will always use ECS-formatted output",
+                "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
                     "#ingest-user-agent-ecs-always",
-                "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " will change to using ECS output format in 7.0");
+                "Ingest pipelines " + pipelinesWithDeprecatedEcsConfig + " uses the [ecs] option which needs to be removed to work in 8.0");
         }
         return null;
 

x-pack/plugin/deprecation/src/test/java/org/elasticsearch/xpack/deprecation/ClusterDeprecationChecksTests.java

@@ -67,10 +67,10 @@ public void testUserAgentEcsCheck() {
         List<DeprecationIssue> issues = DeprecationChecks.filterChecks(CLUSTER_SETTINGS_CHECKS, c -> c.apply(finalState));
 
         DeprecationIssue expected = new DeprecationIssue(DeprecationIssue.Level.WARNING,
-            "User-Agent ingest plugin will use ECS-formatted output",
-            "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-7.0.html" +
+            "User-Agent ingest plugin will always use ECS-formatted output",
+            "https://www.elastic.co/guide/en/elasticsearch/reference/master/breaking-changes-8.0.html" +
                 "#ingest-user-agent-ecs-always",
-            "Ingest pipelines [ecs_false, ecs_null] will change to using ECS output format in 7.0");
+            "Ingest pipelines [ecs_false, ecs_true] uses the [ecs] option which needs to be removed to work in 8.0");
         assertEquals(singletonList(expected), issues);
     }
 }