From 3bd64f4d22f157e4df3531c0b2fedb6689fb9c2f Mon Sep 17 00:00:00 2001 From: Jake Landis Date: Mon, 11 Feb 2019 16:06:03 -0600 Subject: [PATCH] Revert "Add ECS schema for user-agent ingest processor (#37727) (#37984)" This reverts commit cac6b8e06f051d68919faf6081f1c87fa5b6757d. Related: https://github.com/elastic/beats/issues/10650 Will replace this commit with the 6.7 version --- .../ingest/processors/user-agent.asciidoc | 14 +-- .../migration/migrate_7_0/settings.asciidoc | 6 - .../ingest/useragent/UserAgentProcessor.java | 103 ++++++++---------- .../UserAgentProcessorFactoryTests.java | 4 +- .../useragent/UserAgentProcessorTests.java | 44 +++++--- .../20_useragent_processor.yml | 19 +++- .../test/ingest-useragent/30_custom_regex.yml | 9 +- 7 files changed, 107 insertions(+), 92 deletions(-) diff --git a/docs/reference/ingest/processors/user-agent.asciidoc b/docs/reference/ingest/processors/user-agent.asciidoc index f6b6d46fe7b9d..201e3beab8313 100644 --- a/docs/reference/ingest/processors/user-agent.asciidoc +++ b/docs/reference/ingest/processors/user-agent.asciidoc @@ -60,13 +60,13 @@ Which returns "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", "user_agent": { "name": "Chrome", - "original": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.103 Safari/537.36", - "version": "51.0.2704", - "os": { - "name": "Mac OS X", - "version": "10.10.5", - "full": "Mac OS X 10.10.5" - }, + "major": "51", + "minor": "0", + "patch": "2704", + "os_name": "Mac OS X", + "os": "Mac OS X 10.10.5", + "os_major": "10", + "os_minor": "10", "device": "Other" } } diff --git a/docs/reference/migration/migrate_7_0/settings.asciidoc b/docs/reference/migration/migrate_7_0/settings.asciidoc index 2e5631b378652..389aa07b97725 100644 --- a/docs/reference/migration/migrate_7_0/settings.asciidoc +++ b/docs/reference/migration/migrate_7_0/settings.asciidoc @@ -198,9 +198,3 @@ could have lead to dropping audit events while the operations on the system were allowed to continue as usual. The recommended replacement is the use of the `logfile` audit output type and using other components from the Elastic Stack to handle the indexing part. - -[float] -[[ingest-user-agent-ecs-always]] -==== Ingest User Agent processor always uses `ecs` output format -The deprecated `ecs` setting for the user agent ingest processor has been -removed. https://github.com/elastic/ecs[ECS] format is now the default. diff --git a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java index 6f2518eede673..6e7f588f0bd8a 100644 --- a/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java +++ b/modules/ingest-user-agent/src/main/java/org/elasticsearch/ingest/useragent/UserAgentProcessor.java @@ -19,8 +19,6 @@ package org.elasticsearch.ingest.useragent; -import org.apache.logging.log4j.LogManager; -import org.elasticsearch.common.logging.DeprecationLogger; import org.elasticsearch.ingest.AbstractProcessor; import org.elasticsearch.ingest.IngestDocument; import org.elasticsearch.ingest.Processor; @@ -42,8 +40,6 @@ public class UserAgentProcessor extends AbstractProcessor { - private static final DeprecationLogger deprecationLogger = new DeprecationLogger(LogManager.getLogger(UserAgentProcessor.class)); - public static final String TYPE = "user_agent"; private final String field; @@ -67,7 +63,7 @@ boolean isIgnoreMissing() { } @Override - public IngestDocument execute(IngestDocument ingestDocument) { + public IngestDocument execute(IngestDocument ingestDocument) throws Exception { String userAgent = ingestDocument.getFieldValue(field, String.class, ignoreMissing); if (userAgent == null && ignoreMissing) { @@ -79,64 +75,68 @@ public IngestDocument execute(IngestDocument ingestDocument) { Details uaClient = parser.parse(userAgent); Map uaDetails = new HashMap<>(); - - // Parse the user agent in the ECS (Elastic Common Schema) format for (Property property : this.properties) { switch (property) { - case ORIGINAL: - uaDetails.put("original", userAgent); - break; case NAME: if (uaClient.userAgent != null && uaClient.userAgent.name != null) { uaDetails.put("name", uaClient.userAgent.name); - } else { + } + else { uaDetails.put("name", "Other"); } break; - case VERSION: - StringBuilder version = new StringBuilder(); + case MAJOR: if (uaClient.userAgent != null && uaClient.userAgent.major != null) { - version.append(uaClient.userAgent.major); - if (uaClient.userAgent.minor != null) { - version.append(".").append(uaClient.userAgent.minor); - if (uaClient.userAgent.patch != null) { - version.append(".").append(uaClient.userAgent.patch); - if (uaClient.userAgent.build != null) { - version.append(".").append(uaClient.userAgent.build); - } - } - } - uaDetails.put("version", version.toString()); + uaDetails.put("major", uaClient.userAgent.major); + } + break; + case MINOR: + if (uaClient.userAgent != null && uaClient.userAgent.minor != null) { + uaDetails.put("minor", uaClient.userAgent.minor); + } + break; + case PATCH: + if (uaClient.userAgent != null && uaClient.userAgent.patch != null) { + uaDetails.put("patch", uaClient.userAgent.patch); + } + break; + case BUILD: + if (uaClient.userAgent != null && uaClient.userAgent.build != null) { + uaDetails.put("build", uaClient.userAgent.build); } break; case OS: if (uaClient.operatingSystem != null) { - Map osDetails = new HashMap<>(3); - if (uaClient.operatingSystem.name != null) { - osDetails.put("name", uaClient.operatingSystem.name); - StringBuilder sb = new StringBuilder(); - if (uaClient.operatingSystem.major != null) { - sb.append(uaClient.operatingSystem.major); - if (uaClient.operatingSystem.minor != null) { - sb.append(".").append(uaClient.operatingSystem.minor); - if (uaClient.operatingSystem.patch != null) { - sb.append(".").append(uaClient.operatingSystem.patch); - if (uaClient.operatingSystem.build != null) { - sb.append(".").append(uaClient.operatingSystem.build); - } - } - } - osDetails.put("version", sb.toString()); - osDetails.put("full", uaClient.operatingSystem.name + " " + sb.toString()); - } - uaDetails.put("os", osDetails); - } + uaDetails.put("os", buildFullOSName(uaClient.operatingSystem)); + } + else { + uaDetails.put("os", "Other"); + } + + break; + case OS_NAME: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.name != null) { + uaDetails.put("os_name", uaClient.operatingSystem.name); + } + else { + uaDetails.put("os_name", "Other"); + } + break; + case OS_MAJOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.major != null) { + uaDetails.put("os_major", uaClient.operatingSystem.major); + } + break; + case OS_MINOR: + if (uaClient.operatingSystem != null && uaClient.operatingSystem.minor != null) { + uaDetails.put("os_minor", uaClient.operatingSystem.minor); } break; case DEVICE: if (uaClient.device != null && uaClient.device.name != null) { uaDetails.put("device", uaClient.device.name); - } else { + } + else { uaDetails.put("device", "Other"); } break; @@ -215,10 +215,6 @@ public UserAgentProcessor create(Map factories, Strin String regexFilename = readStringProperty(TYPE, processorTag, config, "regex_file", IngestUserAgentPlugin.DEFAULT_PARSER_NAME); List propertyNames = readOptionalList(TYPE, processorTag, config, "properties"); boolean ignoreMissing = readBooleanProperty(TYPE, processorTag, config, "ignore_missing", false); - Object ecsValue = config.remove("ecs"); - if (ecsValue != null) { - deprecationLogger.deprecated("setting [ecs] is deprecated as ECS format is the default and only option"); - } UserAgentParser parser = userAgentParsers.get(regexFilename); if (parser == null) { @@ -246,16 +242,13 @@ public UserAgentProcessor create(Map factories, Strin enum Property { - NAME, - OS, - DEVICE, - ORIGINAL, - VERSION; + NAME, MAJOR, MINOR, PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD; public static Property parseProperty(String propertyName) { try { return valueOf(propertyName.toUpperCase(Locale.ROOT)); - } catch (IllegalArgumentException e) { + } + catch (IllegalArgumentException e) { throw new IllegalArgumentException("illegal property value [" + propertyName + "]. valid values are " + Arrays.toString(EnumSet.allOf(Property.class).toArray())); } diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java index f723c13f23022..d9c6fc17620da 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorFactoryTests.java @@ -178,8 +178,8 @@ public void testInvalidProperty() throws Exception { config.put("properties", Collections.singletonList("invalid")); ElasticsearchParseException e = expectThrows(ElasticsearchParseException.class, () -> factory.create(null, null, config)); - assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, OS, DEVICE, " + - "ORIGINAL, VERSION]")); + assertThat(e.getMessage(), equalTo("[properties] illegal property value [invalid]. valid values are [NAME, MAJOR, MINOR, " + + "PATCH, OS, OS_NAME, OS_MAJOR, OS_MINOR, DEVICE, BUILD]")); } public void testInvalidPropertiesType() throws Exception { diff --git a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java index 3938fccd832a3..0a8b453724c90 100644 --- a/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java +++ b/modules/ingest-user-agent/src/test/java/org/elasticsearch/ingest/useragent/UserAgentProcessorTests.java @@ -103,13 +103,16 @@ public void testCommonBrowser() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Chrome")); - assertThat(target.get("version"), is("33.0.1750")); + assertThat(target.get("major"), is("33")); + assertThat(target.get("minor"), is("0")); + assertThat(target.get("patch"), is("1750")); + assertNull(target.get("build")); + + assertThat(target.get("os"), is("Mac OS X 10.9.2")); + assertThat(target.get("os_name"), is("Mac OS X")); + assertThat(target.get("os_major"), is("10")); + assertThat(target.get("os_minor"), is("9")); - Map os = new HashMap<>(); - os.put("name", "Mac OS X"); - os.put("version", "10.9.2"); - os.put("full", "Mac OS X 10.9.2"); - assertThat(target.get("os"), is(os)); assertThat(target.get("device"), is("Other")); } @@ -128,13 +131,15 @@ public void testUncommonDevice() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("Android")); - assertThat(target.get("version"), is("3.0")); + assertThat(target.get("major"), is("3")); + assertThat(target.get("minor"), is("0")); + assertNull(target.get("patch")); + assertNull(target.get("build")); - Map os = new HashMap<>(); - os.put("name", "Android"); - os.put("version", "3.0"); - os.put("full", "Android 3.0"); - assertThat(target.get("os"), is(os)); + assertThat(target.get("os"), is("Android 3.0")); + assertThat(target.get("os_name"), is("Android")); + assertThat(target.get("os_major"), is("3")); + assertThat(target.get("os_minor"), is("0")); assertThat(target.get("device"), is("Motorola Xoom")); } @@ -153,9 +158,15 @@ public void testSpider() throws Exception { Map target = (Map) data.get("target_field"); assertThat(target.get("name"), is("EasouSpider")); + assertNull(target.get("major")); + assertNull(target.get("minor")); + assertNull(target.get("patch")); + assertNull(target.get("build")); - assertNull(target.get("version")); - assertNull(target.get("os")); + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); assertThat(target.get("device"), is("Spider")); } @@ -179,7 +190,10 @@ public void testUnknown() throws Exception { assertNull(target.get("patch")); assertNull(target.get("build")); - assertNull(target.get("os")); + assertThat(target.get("os"), is("Other")); + assertThat(target.get("os_name"), is("Other")); + assertNull(target.get("os_major")); + assertNull(target.get("os_minor")); assertThat(target.get("device"), is("Other")); } diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml index fc44d7261e80f..28c218edd6935 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/20_useragent_processor.yml @@ -29,9 +29,13 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Chrome" } - - match: { _source.user_agent.original: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - - match: { _source.user_agent.os: {"name":"Mac OS X", "version":"10.9.2", "full":"Mac OS X 10.9.2"} } - - match: { _source.user_agent.version: "33.0.1750" } + - match: { _source.user_agent.os: "Mac OS X 10.9.2" } + - match: { _source.user_agent.os_name: "Mac OS X" } + - match: { _source.user_agent.os_major: "10" } + - match: { _source.user_agent.os_minor: "9" } + - match: { _source.user_agent.major: "33" } + - match: { _source.user_agent.minor: "0" } + - match: { _source.user_agent.patch: "1750" } - match: { _source.user_agent.device: "Other" } --- @@ -66,8 +70,13 @@ index: test id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - - match: { _source.field2.os.full: "Mac OS X 10.9.2" } + - match: { _source.field2.os: "Mac OS X 10.9.2" } - is_false: _source.user_agent - is_false: _source.field2.name + - is_false: _source.field2.os_name + - is_false: _source.field2.os_major + - is_false: _source.field2.os_minor + - is_false: _source.field2.major + - is_false: _source.field2.minor + - is_false: _source.field2.patch - is_false: _source.field2.device - - is_false: _source.field2.original diff --git a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml index ac90a3457fa65..22df584e13166 100644 --- a/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml +++ b/modules/ingest-user-agent/src/test/resources/rest-api-spec/test/ingest-useragent/30_custom_regex.yml @@ -30,6 +30,11 @@ id: 1 - match: { _source.field1: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.149 Safari/537.36" } - match: { _source.user_agent.name: "Test" } + - match: { _source.user_agent.os: "Other" } + - match: { _source.user_agent.os_name: "Other" } - match: { _source.user_agent.device: "Other" } - - is_false: _source.user_agent.os - - is_false: _source.user_agent.version + - is_false: _source.user_agent.os_major + - is_false: _source.user_agent.os_minor + - is_false: _source.user_agent.major + - is_false: _source.user_agent.minor + - is_false: _source.user_agent.patch