Revert "Add ECS schema for user-agent ingest processor (elastic#37727) (

elastic#37984)"

This reverts commit cac6b8e.

Related: elastic/beats#10650

Will replace this commit with the 6.7 version

docs/reference/ingest/processors/user-agent.asciidoc

@@ -60,13 +60,13 @@ Which returns
     "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
     "user_agent": {
       "name": "Chrome",
-      "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36",
-      "version": "51.0.2704",
-      "os": {
-        "name": "Mac OS X",
-        "version": "10.10.5",
-        "full": "Mac OS X 10.10.5"
-      },
+      "major": "51",
+      "minor": "0",
+      "patch": "2704",
+      "os_name": "Mac OS X",
+      "os": "Mac OS X 10.10.5",
+      "os_major": "10",
+      "os_minor": "10",
       "device": "Other"
     }
   }

docs/reference/migration/migrate_7_0/settings.asciidoc

@@ -198,9 +198,3 @@ could have lead to dropping audit events while the operations on the system
 were allowed to continue as usual. The recommended replacement is the
 use of the `logfile` audit output type and using other components from the
 Elastic Stack to handle the indexing part.
-
-[float]
-[[ingest-user-agent-ecs-always]]
-==== Ingest User Agent processor always uses `ecs` output format
-The deprecated `ecs` setting for the user agent ingest processor has been
-removed. https://github.com/elastic/ecs[ECS] format is now the default.

modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java

@@ -19,8 +19,6 @@
 
 package org.elasticsearch.ingest.useragent;
 
-import org.apache.logging.log4j.LogManager;
-import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.ingest.AbstractProcessor;
 import org.elasticsearch.ingest.IngestDocument;
 import org.elasticsearch.ingest.Processor;
@@ -42,8 +40,6 @@
 
 public class UserAgentProcessor extends AbstractProcessor {
 
-    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class));
-
     public static final String TYPE = "user_agent";
 
     private final String field;
@@ -67,7 +63,7 @@ boolean isIgnoreMissing() {
     }
 
     @Override
-    public IngestDocument execute(IngestDocument ingestDocument) {
+    public IngestDocument execute(IngestDocument ingestDocument) throws Exception {
         String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing);
 
         if (userAgent == null && ignoreMissing) {
@@ -79,64 +75,68 @@ public IngestDocument execute(IngestDocument ingestDocument) {
         Details uaClient = parser.parse(userAgent);
 
         Map<String, Object> uaDetails = new HashMap<>();
-
-        // Parse the user agent in the ECS (Elastic Common Schema) format
         for (Property property : this.properties) {
             switch (property) {
-                case ORIGINAL:
-                    uaDetails.put("original", userAgent);
-                    break;
                 case NAME:
                     if (uaClient.userAgent != null && uaClient.userAgent.name != null) {
                         uaDetails.put("name", uaClient.userAgent.name);
-                    } else {
+                    }
+                    else {
                         uaDetails.put("name", "Other");
                     }
                     break;
-                case VERSION:
-                    StringBuilder version = new StringBuilder();
+                case MAJOR:
                     if (uaClient.userAgent != null && uaClient.userAgent.major != null) {
-                        version.append(uaClient.userAgent.major);
-                        if (uaClient.userAgent.minor != null) {
-                            version.append(".").append(uaClient.userAgent.minor);
-                            if (uaClient.userAgent.patch != null) {
-                                version.append(".").append(uaClient.userAgent.patch);
-                                if (uaClient.userAgent.build != null) {
-                                    version.append(".").append(uaClient.userAgent.build);
-                                }
-                            }
-                        }
-                        uaDetails.put("version", version.toString());
+                        uaDetails.put("major", uaClient.userAgent.major);
+                    }
+                    break;
+                case MINOR:
+                    if (uaClient.userAgent != null && uaClient.userAgent.minor != null) {
+                        uaDetails.put("minor", uaClient.userAgent.minor);
+                    }
+                    break;
+                case PATCH:
+                    if (uaClient.userAgent != null && uaClient.userAgent.patch != null) {
+                        uaDetails.put("patch", uaClient.userAgent.patch);
+                    }
+                    break;
+                case BUILD:
+                    if (uaClient.userAgent != null && uaClient.userAgent.build != null) {
+                        uaDetails.put("build", uaClient.userAgent.build);
                     }
                     break;
                 case OS:
                     if (uaClient.operatingSystem != null) {
-                        Map<String, String> osDetails = new HashMap<>(3);
-                        if (uaClient.operatingSystem.name != null) {
-                            osDetails.put("name", uaClient.operatingSystem.name);
-                            StringBuilder sb = new StringBuilder();
-                            if (uaClient.operatingSystem.major != null) {
-                                sb.append(uaClient.operatingSystem.major);
-                                if (uaClient.operatingSystem.minor != null) {
-                                    sb.append(".").append(uaClient.operatingSystem.minor);
-                                    if (uaClient.operatingSystem.patch != null) {
-                                        sb.append(".").append(uaClient.operatingSystem.patch);
-                                        if (uaClient.operatingSystem.build != null) {
-                                            sb.append(".").append(uaClient.operatingSystem.build);
-                                        }
-                                    }
-                                }
-                                osDetails.put("version", sb.toString());
-                                osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString());
-                            }
-                            uaDetails.put("os", osDetails);
-                        }
+                        uaDetails.put("os", buildFullOSName(uaClient.operatingSystem));
+                    }
+                    else {
+                        uaDetails.put("os", "Other");
+                    }
+
+                    break;
+                case OS_NAME:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) {
+                        uaDetails.put("os_name", uaClient.operatingSystem.name);
+                    }
+                    else {
+                        uaDetails.put("os_name", "Other");
+                    }
+                    break;
+                case OS_MAJOR:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) {
+                        uaDetails.put("os_major", uaClient.operatingSystem.major);
+                    }
+                    break;
+                case OS_MINOR:
+                    if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) {
+                        uaDetails.put("os_minor", uaClient.operatingSystem.minor);
                     }
                     break;
                 case DEVICE:
                     if (uaClient.device != null && uaClient.device.name != null) {
                         uaDetails.put("device", uaClient.device.name);
-                    } else {
+                    }
+                    else {
                         uaDetails.put("device", "Other");
                     }
                     break;
@@ -215,10 +215,6 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
             String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME);
             List<String> propertyNames = readOptionalList(TYPE, processorTag, config, "properties");
             boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false);
-            Object ecsValue = config.remove("ecs");
-            if (ecsValue != null) {
-                deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option");
-            }
 
             UserAgentParser parser = userAgentParsers.get(regexFilename);
             if (parser == null) {
@@ -246,16 +242,13 @@ public UserAgentProcessor create(Map<String, Processor.Factory> factories, Strin
 
     enum Property {
 
-        NAME,
-        OS,
-        DEVICE,
-        ORIGINAL,
-        VERSION;
+        NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD;
 
         public static Property parseProperty(String propertyName) {
             try {
                 return valueOf(propertyName.toUpperCase(Locale.ROOT));
-            } catch (IllegalArgumentException e) {
+            }
+            catch (IllegalArgumentException e) {
                 throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " +
                         Arrays.toString(EnumSet.allOf(Property.class).toArray()));
             }

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java

@@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception {
         config.put("properties", Collections.singletonList("invalid"));
 
         ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config));
-        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " +
-            "ORIGINAL, VERSION]"));
+        assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, "
+                + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]"));
     }
 
     public void testInvalidPropertiesType() throws Exception {

modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java

@@ -103,13 +103,16 @@ public void testCommonBrowser() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("Chrome"));
-        assertThat(target.get("version"), is("33.0.1750"));
+        assertThat(target.get("major"), is("33"));
+        assertThat(target.get("minor"), is("0"));
+        assertThat(target.get("patch"), is("1750"));
+        assertNull(target.get("build"));
+
+        assertThat(target.get("os"), is("Mac OS X 10.9.2"));
+        assertThat(target.get("os_name"), is("Mac OS X"));
+        assertThat(target.get("os_major"), is("10"));
+        assertThat(target.get("os_minor"), is("9"));
 
-        Map<String, String> os = new HashMap<>();
-        os.put("name", "Mac OS X");
-        os.put("version", "10.9.2");
-        os.put("full", "Mac OS X 10.9.2");
-        assertThat(target.get("os"), is(os));
         assertThat(target.get("device"), is("Other"));
     }
 
@@ -128,13 +131,15 @@ public void testUncommonDevice() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("Android"));
-        assertThat(target.get("version"), is("3.0"));
+        assertThat(target.get("major"), is("3"));
+        assertThat(target.get("minor"), is("0"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
 
-        Map<String, String> os = new HashMap<>();
-        os.put("name", "Android");
-        os.put("version", "3.0");
-        os.put("full", "Android 3.0");
-        assertThat(target.get("os"), is(os));
+        assertThat(target.get("os"), is("Android 3.0"));
+        assertThat(target.get("os_name"), is("Android"));
+        assertThat(target.get("os_major"), is("3"));
+        assertThat(target.get("os_minor"), is("0"));
 
         assertThat(target.get("device"), is("Motorola Xoom"));
     }
@@ -153,9 +158,15 @@ public void testSpider() throws Exception {
         Map<String, Object> target = (Map<String, Object>) data.get("target_field");
 
         assertThat(target.get("name"), is("EasouSpider"));
+        assertNull(target.get("major"));
+        assertNull(target.get("minor"));
+        assertNull(target.get("patch"));
+        assertNull(target.get("build"));
 
-        assertNull(target.get("version"));
-        assertNull(target.get("os"));
+        assertThat(target.get("os"), is("Other"));
+        assertThat(target.get("os_name"), is("Other"));
+        assertNull(target.get("os_major"));
+        assertNull(target.get("os_minor"));
 
         assertThat(target.get("device"), is("Spider"));
     }
@@ -179,7 +190,10 @@ public void testUnknown() throws Exception {
         assertNull(target.get("patch"));
         assertNull(target.get("build"));
 
-        assertNull(target.get("os"));
+        assertThat(target.get("os"), is("Other"));
+        assertThat(target.get("os_name"), is("Other"));
+        assertNull(target.get("os_major"));
+        assertNull(target.get("os_minor"));
 
         assertThat(target.get("device"), is("Other"));
     }

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml

@@ -29,9 +29,13 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Chrome" }
-  - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
-  - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} }
-  - match: { _source.user_agent.version: "33.0.1750" }
+  - match: { _source.user_agent.os: "Mac OS X 10.9.2" }
+  - match: { _source.user_agent.os_name: "Mac OS X" }
+  - match: { _source.user_agent.os_major: "10" }
+  - match: { _source.user_agent.os_minor: "9" }
+  - match: { _source.user_agent.major: "33" }
+  - match: { _source.user_agent.minor: "0" }
+  - match: { _source.user_agent.patch: "1750" }
   - match: { _source.user_agent.device: "Other" }
 
 ---
@@ -66,8 +70,13 @@
         index: test
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
-  - match: { _source.field2.os.full: "Mac OS X 10.9.2" }
+  - match: { _source.field2.os: "Mac OS X 10.9.2" }
   - is_false: _source.user_agent
   - is_false: _source.field2.name
+  - is_false: _source.field2.os_name
+  - is_false: _source.field2.os_major
+  - is_false: _source.field2.os_minor
+  - is_false: _source.field2.major
+  - is_false: _source.field2.minor
+  - is_false: _source.field2.patch
   - is_false: _source.field2.device
-  - is_false: _source.field2.original

modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml

@@ -30,6 +30,11 @@
         id: 1
   - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" }
   - match: { _source.user_agent.name: "Test" }
+  - match: { _source.user_agent.os: "Other" }
+  - match: { _source.user_agent.os_name: "Other" }
   - match: { _source.user_agent.device: "Other" }
-  - is_false: _source.user_agent.os
-  - is_false: _source.user_agent.version
+  - is_false: _source.user_agent.os_major
+  - is_false: _source.user_agent.os_minor
+  - is_false: _source.user_agent.major
+  - is_false: _source.user_agent.minor
+  - is_false: _source.user_agent.patch