Add ECS schema for user-agent ingest processor (elastic#37727)

This switches the format of the user agent processor to use the schema from [ECS](https://github.com/elastic/ecs).
So rather than something like this:

```
{
  "patch" : "3538",
  "major" : "70",
  "minor" : "0",
  "os" : "Mac OS X 10.14.1",
  "os_minor" : "14",
  "os_major" : "10",
  "name" : "Chrome",
  "os_name" : "Mac OS X",
  "device" : "Other"
}
```

The structure is now like this:

```
{
  "name" : "Chrome",
  "original" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.102 Safari/537.36",
  "os" : {
    "name" : "Mac OS X",
    "version" : "10.14.1",
    "full" : "Mac OS X 10.14.1"
  },
  "device" : "Other",
  "version" : "70.0.3538.102"
}
```

This is now the default for 7.0. The deprecated `ecs` setting in 6.x is not
supported.

Resolves elastic#37329

docs/reference/ingest/processors/user-agent.asciidoc

@@ -19,6 +19,7 @@ The ingest-user-agent module ships by default with the regexes.yaml made availab
 | `regex_file`           | no        | -                                                                                               | The name of the file in the `config/ingest-user-agent` directory containing the regular expressions for parsing the user agent string. Both the directory and the file have to be created before starting Elasticsearch. If not specified, ingest-user-agent will use the regexes.yaml from uap-core it ships with (see below).
 | `properties`           | no        | [`name`, `major`, `minor`, `patch`, `build`, `os`, `os_name`, `os_major`, `os_minor`, `device`] | Controls what properties are added to `target_field`.
 | `ignore_missing`       | no        | `false`                                                                                         | If `true` and `field` does not exist, the processor quietly exits without modifying the document
+| `ecs`                  | no        | `false`                                                                                         | Whether to return the output in Elastic Common Schema format. NOTE: ECS format will be the default in Elasticsearch 7.0 and non-ECS format is deprecated.
 |======
 
 Here is an example that adds the user agent details to the `user_agent` field based on the `agent` field:
@@ -60,13 +61,13 @@ Which returns
     "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
     "user_agent": {
       "name": "Chrome",
-      "major": "51",
-      "minor": "0",
-      "patch": "2704",
-      "os_name": "Mac OS X",
-      "os": "Mac OS X 10.10.5",
-      "os_major": "10",
-      "os_minor": "10",
+      "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
+      "version": "51.0.2704",
+      "os": {
+        "name": "Mac OS X",
+        "version": "10.10.5",
+        "full": "Mac OS X 10.10.5"
+      },
       "device": "Other"
     }
   }

docs/reference/migration/migrate_7_0/settings.asciidoc

@@ -182,3 +182,9 @@ could have lead to dropping audit events while the operations on the system
 were allowed to continue as usual. The recommended replacement is the
 use of the `logfile` audit output type and using other components from the
 Elastic Stack to handle the indexing part.
+
+[float]
+[[ingest-user-agent-ecs-always]]
+==== Ingest User Agent processor always uses `ecs` output format
+The deprecated `ecs` setting for the user agent ingest processor has been
+removed. https://github.com/elastic/ecs[ECS] format is now the default.

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -19,6 +19,8 @@
 
 package org.elasticsearch.ingest.useragent;
 
+import org.apache.logging.log4j.LogManager;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
@@ -40,6 +42,8 @@
 
 public class UserAgentProcessor extends AbstractProcessor {
 
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class));
+
     public static final String TYPE = "user_agent";
 
     private final String field;
@@ -63,7 +67,7 @@ boolean isIgnoreMissing() {
     }
 
     @Override
-    public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
+    public IngestDocument execute(IngestDocument ingestDocument) {
         String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
 
         if (userAgent == null && ignoreMissing) {
@@ -75,68 +79,64 @@ public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
         Details uaClient = parser.parse(userAgent);
 
         Map<String, Object> uaDetails = new HashMap<>();
+
+        // Parse the user agent in the ECS (Elastic Common Schema) format
         for (Property property : this.properties) {
             switch (property) {
+                case ORIGINAL:
+                    uaDetails.put("original", userAgent);
+                    break;
                 case NAME:
                     if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
                         uaDetails.put("name", uaClient.userAgent.name);
-                    }
-                    else {
+                    } else {
                         uaDetails.put("name", "Other");
                     }
                     break;
-                case MAJOR:
+                case VERSION:
+                    StringBuilder version = new StringBuilder();
                     if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
-                        uaDetails.put("major", uaClient.userAgent.major);
-                    }
-                    break;
-                case MINOR:
-                    if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
-                        uaDetails.put("minor", uaClient.userAgent.minor);
-                    }
-                    break;
-                case PATCH:
-                    if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
-                        uaDetails.put("patch", uaClient.userAgent.patch);
-                    }
-                    break;
-                case BUILD:
-                    if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
-                        uaDetails.put("build", uaClient.userAgent.build);
+                        version.append(uaClient.userAgent.major);
+                        if (uaClient.userAgent.minor != null) {
+                            version.append(".").append(uaClient.userAgent.minor);
+                            if (uaClient.userAgent.patch != null) {
+                                version.append(".").append(uaClient.userAgent.patch);
+                                if (uaClient.userAgent.build != null) {
+                                    version.append(".").append(uaClient.userAgent.build);
+                                }
+                            }
+                        }
+                        uaDetails.put("version", version.toString());
                     }
                     break;
                 case OS:
                     if (uaClient.operatingSystem != null) {
-                        uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
-                    }
-                    else {
-                        uaDetails.put("os", "Other");
-                    }
-
-                    break;
-                case OS_NAME:
-                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
-                        uaDetails.put("os_name", uaClient.operatingSystem.name);
-                    }
-                    else {
-                        uaDetails.put("os_name", "Other");
-                    }
-                    break;
-                case OS_MAJOR:
-                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
-                        uaDetails.put("os_major", uaClient.operatingSystem.major);
-                    }
-                    break;
-                case OS_MINOR:
-                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
-                        uaDetails.put("os_minor", uaClient.operatingSystem.minor);
+                        Map<String, String> osDetails = new HashMap<>(3);
+                        if (uaClient.operatingSystem.name != null) {
+                            osDetails.put("name", uaClient.operatingSystem.name);
+                            StringBuilder sb = new StringBuilder();
+                            if (uaClient.operatingSystem.major != null) {
+                                sb.append(uaClient.operatingSystem.major);
+                                if (uaClient.operatingSystem.minor != null) {
+                                    sb.append(".").append(uaClient.operatingSystem.minor);
+                                    if (uaClient.operatingSystem.patch != null) {
+                                        sb.append(".").append(uaClient.operatingSystem.patch);
+                                        if (uaClient.operatingSystem.build != null) {
+                                            sb.append(".").append(uaClient.operatingSystem.build);
+                                        }
+                                    }
+                                }
+                                osDetails.put("version", sb.toString());
+                                osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
+                            }
+                            uaDetails.put("os", osDetails);
+                        }
                     }
                     break;
                 case DEVICE:
                     if (uaClient.device != null && uaClient.device.name != null) {
                         uaDetails.put("device", uaClient.device.name);
-                    }
-                    else {
+                    } else {
                         uaDetails.put("device", "Other");
                     }
                     break;
@@ -215,6 +215,10 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
+            Object ecsValue = config.remove("ecs");
+            if (ecsValue != null) {
+                deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
+            }
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
@@ -242,13 +246,16 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
 
     enum Property {
 
-        NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD;
+        NAME,
+        OS,
+        DEVICE,
+        ORIGINAL,
+        VERSION;
 
         public static Property parseProperty(String propertyName) {
             try {
                 return valueOf(propertyName.toUpperCase(Locale.ROOT));
-            }
-            catch (IllegalArgumentException e) {
+            } catch (IllegalArgumentException e) {
                 throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
                         Arrays.toString(EnumSet.allOf(Property.class).toArray()));
             }

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception {
         config.put("properties", Collections.singletonList("invalid"));
 
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
-        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
-                + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
+        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
+            "ORIGINAL, VERSION]"));
     }
 
     public void testInvalidPropertiesType() throws Exception {

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -103,16 +103,13 @@ public void testCommonBrowser() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("Chrome"));
-        assertThat(target.get("major"), is("33"));
-        assertThat(target.get("minor"), is("0"));
-        assertThat(target.get("patch"), is("1750"));
-        assertNull(target.get("build"));
-
-        assertThat(target.get("os"), is("Mac OS X 10.9.2"));
-        assertThat(target.get("os_name"), is("Mac OS X"));
-        assertThat(target.get("os_major"), is("10"));
-        assertThat(target.get("os_minor"), is("9"));
+        assertThat(target.get("version"), is("33.0.1750"));
 
+        Map<String, String> os = new HashMap<>();
+        os.put("name", "Mac OS X");
+        os.put("version", "10.9.2");
+        os.put("full", "Mac OS X 10.9.2");
+        assertThat(target.get("os"), is(os));
         assertThat(target.get("device"), is("Other"));
     }
 
@@ -131,15 +128,13 @@ public void testUncommonDevice() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("Android"));
-        assertThat(target.get("major"), is("3"));
-        assertThat(target.get("minor"), is("0"));
-        assertNull(target.get("patch"));
-        assertNull(target.get("build"));
+        assertThat(target.get("version"), is("3.0"));
 
-        assertThat(target.get("os"), is("Android 3.0"));
-        assertThat(target.get("os_name"), is("Android"));
-        assertThat(target.get("os_major"), is("3"));
-        assertThat(target.get("os_minor"), is("0"));
+        Map<String, String> os = new HashMap<>();
+        os.put("name", "Android");
+        os.put("version", "3.0");
+        os.put("full", "Android 3.0");
+        assertThat(target.get("os"), is(os));
 
         assertThat(target.get("device"), is("Motorola Xoom"));
     }
@@ -158,15 +153,9 @@ public void testSpider() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("EasouSpider"));
-        assertNull(target.get("major"));
-        assertNull(target.get("minor"));
-        assertNull(target.get("patch"));
-        assertNull(target.get("build"));
 
-        assertThat(target.get("os"), is("Other"));
-        assertThat(target.get("os_name"), is("Other"));
-        assertNull(target.get("os_major"));
-        assertNull(target.get("os_minor"));
+        assertNull(target.get("version"));
+        assertNull(target.get("os"));
 
         assertThat(target.get("device"), is("Spider"));
     }
@@ -190,10 +179,7 @@ public void testUnknown() throws Exception {
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
 
-        assertThat(target.get("os"), is("Other"));
-        assertThat(target.get("os_name"), is("Other"));
-        assertNull(target.get("os_major"));
-        assertNull(target.get("os_minor"));
+        assertNull(target.get("os"));
 
         assertThat(target.get("device"), is("Other"));
     }

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

@@ -31,13 +31,9 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Chrome" }
-  - match: { _source.user_agent.os: "Mac OS X 10.9.2" }
-  - match: { _source.user_agent.os_name: "Mac OS X" }
-  - match: { _source.user_agent.os_major: "10" }
-  - match: { _source.user_agent.os_minor: "9" }
-  - match: { _source.user_agent.major: "33" }
-  - match: { _source.user_agent.minor: "0" }
-  - match: { _source.user_agent.patch: "1750" }
+  - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
+  - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
+  - match: { _source.user_agent.version: "33.0.1750" }
   - match: { _source.user_agent.device: "Other" }
 
 ---
@@ -74,13 +70,8 @@
         type: test
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
-  - match: { _source.field2.os: "Mac OS X 10.9.2" }
+  - match: { _source.field2.os.full: "Mac OS X 10.9.2" }
   - is_false: _source.user_agent
   - is_false: _source.field2.name
-  - is_false: _source.field2.os_name
-  - is_false: _source.field2.os_major
-  - is_false: _source.field2.os_minor
-  - is_false: _source.field2.major
-  - is_false: _source.field2.minor
-  - is_false: _source.field2.patch
   - is_false: _source.field2.device
+  - is_false: _source.field2.original

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

@@ -32,11 +32,6 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Test" }
-  - match: { _source.user_agent.os: "Other" }
-  - match: { _source.user_agent.os_name: "Other" }
   - match: { _source.user_agent.device: "Other" }
-  - is_false: _source.user_agent.os_major
-  - is_false: _source.user_agent.os_minor
-  - is_false: _source.user_agent.major
-  - is_false: _source.user_agent.minor
-  - is_false: _source.user_agent.patch
+  - is_false: _source.user_agent.os
+  - is_false: _source.user_agent.version