diff --git a/_beats/dev-tools/ecs-migration.yml b/_beats/dev-tools/ecs-migration.yml index ea3268a00e..3ec8dfde0a 100644 --- a/_beats/dev-tools/ecs-migration.yml +++ b/_beats/dev-tools/ecs-migration.yml @@ -15,11 +15,6 @@ # Beat fields -- from: beat.hostname - to: host.hostname - alias6: true - alias: true - - from: beat.timezone to: event.timezone alias6: true diff --git a/_beats/libbeat/_meta/fields.common.yml b/_beats/libbeat/_meta/fields.common.yml index 4866b04628..d9401b8c19 100644 --- a/_beats/libbeat/_meta/fields.common.yml +++ b/_beats/libbeat/_meta/fields.common.yml @@ -66,9 +66,6 @@ Alias fields for compatibility with 7.x. fields: # Common Beats fields - - name: host.hostname - type: alias - path: beat.hostname - name: event.timezone type: alias path: beat.timezone diff --git a/_beats/libbeat/processors/add_host_metadata/_meta/fields.yml b/_beats/libbeat/processors/add_host_metadata/_meta/fields.yml index b579df3b27..770ca050f7 100644 --- a/_beats/libbeat/processors/add_host_metadata/_meta/fields.yml +++ b/_beats/libbeat/processors/add_host_metadata/_meta/fields.yml @@ -15,14 +15,6 @@ type: keyword description: > Unique host id. - - name: architecture - type: keyword - description: > - Host architecture (e.g. x86_64, arm, ppc, mips). - - name: os.platform - type: keyword - description: > - OS platform (e.g. centos, ubuntu, windows). - name: os.version type: keyword description: > @@ -31,12 +23,7 @@ type: keyword description: > OS family (e.g. redhat, debian, freebsd, windows). - - name: ip - type: ip - description: > - List of IP-addresses. - name: mac type: keyword description: > List of hardware-addresses, usually MAC-addresses. - diff --git a/_beats/libbeat/scripts/generate_fields_docs.py b/_beats/libbeat/scripts/generate_fields_docs.py index aadc1e0d6b..bdcff718fd 100644 --- a/_beats/libbeat/scripts/generate_fields_docs.py +++ b/_beats/libbeat/scripts/generate_fields_docs.py @@ -1,6 +1,8 @@ -import yaml -import os import argparse +from collections import OrderedDict +import os + +import yaml def document_fields(output, section, sections, path): @@ -66,8 +68,8 @@ def document_field(output, field, field_path): if not field["index"]: output.write("{}\n\n".format("Field is not indexed.")) - if "enable" in field: - if not field["enable"]: + if "enabled" in field: + if not field["enabled"]: output.write("{}\n\n".format("Object is not enabled.")) if "multi_fields" in field: @@ -103,6 +105,21 @@ def fields_to_asciidoc(input, output, beat): print("fields.yml file is empty. fields.asciidoc cannot be generated.") return + # deduplicate fields, last one wins + for section in docs: + if not section.get("fields"): + continue + fields = OrderedDict() + for field in section["fields"]: + name = field["name"] + if name in fields: + assert field["type"] == fields[name]["type"], 'field "{}" redefined with different type "{}"'.format( + name, field["type"]) + fields[name].update(field) + else: + fields[name] = field + section["fields"] = list(fields.values()) + # Create sections from available fields sections = {} for v in docs: diff --git a/_meta/ecs-migration.yml b/_meta/ecs-migration.yml new file mode 100644 index 0000000000..dfb0705d78 --- /dev/null +++ b/_meta/ecs-migration.yml @@ -0,0 +1,140 @@ +# The ECS migration file contains the information about all the fields which are migrated to ECS in 7.0. +# The goal of the file is to potentially have scripts on top of this information to convert visualisations and templates +# based on this information in an automated way and to keep track of all changes which were applied. +# +# The format of the file is as following: +# +# - from: source-field-in-6.x +# to: target-filed-in-ECS +# # Alias field is useful for fields where there is a 1-1 mapping from old to new +# alias: true-if-alias-is-required-in-6x (default is true) +# # Copy to is useful for fields where multiple fields map to the same ECS field +# copy_to: true-if-field-should-be-copied-to-target-in-6x (default is false) + +- from: beat.hostname + to: observer.hostname + +- from: beat.name + to: observer.type + +- from: beat.version + to: observer.version + +- from: context.service.agent.name + to: agent.name + +- from: context.service.agent.version + to: agent.version + +- from: context.system.architecture + to: host.architecture + +- from: context.system.hostname + to: host.hostname + +- from: context.system.ip + to: host.ip + +- from: context.system.platform + to: host.os.platform + +- from: context.request.method + to: http.request.method + +- from: context.request.http_version + to: http.version + +- from: context.process.pid + to: process.pid + +- from: context.process.ppid + to: process.ppid + +- from: context.process.title + to: process.title + + # not in ECS +- from: context.service.environment + to: service.environment + + # not in ECS +- from: context.service.framework.name + to: service.framework.name + + # not in ECS +- from: context.service.framework.version + to: service.framework.version + + # not in ECS +- from: context.service.language.name + to: service.language.name + + # not in ECS +- from: context.service.language.version + to: service.language.version + +- from: context.service.name + to: service.name + + # not in ECS +- from: context.service.runtime.name + to: service.runtime.name + + # not in ECS +- from: context.service.runtime.version + to: service.runtime.version + +- from: context.service.version + to: service.version + +- from: context.request.url.full + to: url.full + +- from: context.request.url.hash + to: url.fragment + +- from: context.request.url.hostname + to: url.domain + +- from: context.request.url.pathname + to: url.path + +- from: context.request.url.port + to: url.port + alias: false + copy_to: true + +- from: context.request.url.raw + to: url.original + +- from: context.request.url.search + to: url.query + +- from: context.request.url.protocol + to: url.scheme + alias: false + copy_to: true + +- from: context.response.finished + to: http.response.finished + +- from: context.response.status_code + to: http.response.status_code + +- from: context.user.email + to: user.email + +- from: context.user.id + to: user.id + +- from: context.user.username + to: user.name + +- from: context.user.ip + to: client.ip + +- from: context.user.user-agent + to: user_agent.original.text + +- from: listening + to: observer.listening diff --git a/_meta/fields.common.yml b/_meta/fields.common.yml index 0aaf5dbc6c..9f7711dbb1 100644 --- a/_meta/fields.common.yml +++ b/_meta/fields.common.yml @@ -30,6 +30,67 @@ Any arbitrary contextual information regarding the event, captured by the agent, optionally provided by the user. dynamic: false fields: + - name: custom + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + + - name: db + type: group + fields: + - name: instance + type: keyword + index: false + analyzed: false + searchable: false + aggregatable: false + + - name: statement + type: keyword + index: false + analyzed: false + searchable: false + aggregatable: false + + - name: type + type: keyword + index: false + analyzed: false + searchable: false + aggregatable: false + + - name: user + type: keyword + index: false + analyzed: false + searchable: false + aggregatable: false + + - name: http + type: group + dynamic: false + fields: + - name: method + type: keyword + index: false + aggregatable: false + searchable: false + doc_values: false + + - name: status_code + type: long + description: > + The status code of the http response. + + - name: url + type: keyword + index: false + aggregatable: false + searchable: false + doc_values: false - name: tags type: object @@ -38,15 +99,6 @@ description: > A flat mapping of user-defined tags with string values. - - name: http - type: group - fields: - - - name: status_code - type: long - description: > - The status code of the http response. - - name: user type: group fields: @@ -82,6 +134,60 @@ type: group fields: + - name: body + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + + - name: cookies + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + + - name: headers + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + # intake enforces: + # fields: + # - name: content-type + # type: keyword + # - name: cookie + # type: keyword + # - name: user-agent + # type: keyword + + - name: env + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + + - name: socket + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + # intake enforces: + # fields: + # - name: socket.encrypted + # type: boolean + # - name: socket.remote_address + # type: keyword + - name: url type: group description: > @@ -142,6 +248,25 @@ type: group fields: + - name: headers + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + # intake enforces: + # fields: + # - name: content-type + # type: keyword + # - name: user-agent + # type: keyword + + - name: headers_sent + type: boolean + index: false + aggregatable: false + - name: status_code type: long description: > @@ -183,6 +308,14 @@ description: > Information pertaining to the running process where the data was collected fields: + - name: argv + type: object + enabled: false + indexed: false + analyzed: false + searchable: false + aggregatable: false + - name: pid type: long description: > @@ -316,3 +449,216 @@ type: keyword description: > The ID of the parent event. + + # ECS + - name: agent + type: group + dynamic: false + fields: + - name: name + type: alias + path: context.service.agent.name + + - name: version + type: alias + path: context.service.agent.version + + - name: client + type: group + dynamic: false + fields: + - name: ip + type: alias + path: context.user.ip + + - name: observer + type: group + dynamic: false + fields: + - name: hostname + type: alias + path: beat.hostname + + - name: listening + type: alias + path: listening + + - name: type + type: alias + path: beat.name + + - name: version + type: alias + path: beat.version + + - name: host + type: group + dynamic: false + fields: + - name: architecture + type: alias + path: context.system.architecture + + - name: ip + type: alias + path: context.system.ip + + - name: hostname + type: alias + path: context.system.hostname + + - name: os + type: group + fields: + - name: platform + type: alias + path: context.system.platform + + - name: http + type: group + dynamic: false + fields: + - name: request.method + type: alias + path: context.request.method + + - name: response.finished + type: alias + path: context.response.finished + + - name: response.status_code + type: alias + path: context.response.status_code + + - name: version + type: alias + path: context.request.http_version + + - name: process + type: group + dynamic: false + fields: + - name: pid + type: alias + path: context.process.pid + + - name: ppid + type: alias + path: context.process.ppid + + - name: title + type: alias + path: context.process.title + + - name: service + type: group + dynamic: false + fields: + # not in ECS + - name: environment + type: alias + path: context.service.environment + + # not in ECS + - name: framework + type: group + fields: + - name: name + type: alias + path: context.service.framework.name + + - name: version + type: alias + path: context.service.framework.version + + # not in ECS + - name: language + type: group + fields: + - name: name + type: alias + path: context.service.language.name + + - name: version + type: alias + path: context.service.language.version + + - name: name + type: alias + path: context.service.name + + # not in ECS + - name: runtime + type: group + fields: + - name: name + type: alias + path: context.service.runtime.name + + - name: version + type: alias + path: context.service.runtime.version + + - name: version + type: alias + path: context.service.version + + - name: url + type: group + dynamic: false + fields: + - name: domain + type: alias + path: context.request.url.hostname + + - name: fragment + type: alias + path: context.request.url.hash + + - name: full + type: alias + path: context.request.url.full + + - name: original + type: alias + path: context.request.url.raw + + - name: path + type: alias + path: context.request.url.pathname + + # context.request.url.port keyword -> long + - name: port + type: long + description: > + The port of the request, e.g. 443. + + - name: query + type: alias + path: context.request.url.search + + # context.request.url.protocol minus the ":" + - name: scheme + type: keyword + description: > + The scheme of the request, e.g. "https". + + - name: user + type: group + dynamic: false + fields: + - name: email + type: alias + path: context.user.email + + - name: id + type: alias + path: context.user.id + + - name: name + type: alias + path: context.user.username + + - name: user_agent.original.text + type: alias + path: context.user.user-agent diff --git a/docs/data/elasticsearch/error.json b/docs/data/elasticsearch/error.json index 056d5dc63c..7310a030d8 100644 --- a/docs/data/elasticsearch/error.json +++ b/docs/data/elasticsearch/error.json @@ -249,5 +249,9 @@ }, "transaction": { "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79" + }, + "url": { + "port": 8080, + "scheme": "https" } } diff --git a/docs/data/elasticsearch/generated/errors.json b/docs/data/elasticsearch/generated/errors.json index 0b98bf4d3b..eac01fb3b9 100644 --- a/docs/data/elasticsearch/generated/errors.json +++ b/docs/data/elasticsearch/generated/errors.json @@ -268,6 +268,10 @@ }, "timestamp": { "us": 1494342245999999 + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/docs/data/elasticsearch/generated/metricsets.json b/docs/data/elasticsearch/generated/metricsets.json index da6753b359..310f1c6d0f 100644 --- a/docs/data/elasticsearch/generated/metricsets.json +++ b/docs/data/elasticsearch/generated/metricsets.json @@ -30,6 +30,10 @@ "double_gauge": 3.141592653589793, "float_gauge": 9.16, "integer_gauge": 42767, + "labels": { + "code": "200", + "some.other.code": "abc" + }, "long_gauge": 3147483648, "negative": { "d": { diff --git a/docs/data/elasticsearch/generated/transactions.json b/docs/data/elasticsearch/generated/transactions.json index 09416a199f..e4ae91e348 100644 --- a/docs/data/elasticsearch/generated/transactions.json +++ b/docs/data/elasticsearch/generated/transactions.json @@ -228,6 +228,10 @@ "started": 17 }, "type": "request" + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/docs/data/elasticsearch/metric.json b/docs/data/elasticsearch/metric.json index 52a499f24d..dacf0d7ef0 100644 --- a/docs/data/elasticsearch/metric.json +++ b/docs/data/elasticsearch/metric.json @@ -43,6 +43,10 @@ "double_gauge": 3.141592653589793, "float_gauge": 9.16, "integer_gauge": 42767, + "labels": { + "code": "200", + "some.other.code": "abc" + }, "long_gauge": 3147483648, "negative": { "d": { diff --git a/docs/data/elasticsearch/transaction.json b/docs/data/elasticsearch/transaction.json index b9ac247a1e..9b8a98d742 100644 --- a/docs/data/elasticsearch/transaction.json +++ b/docs/data/elasticsearch/transaction.json @@ -135,5 +135,9 @@ } }, "type": "request" + }, + "url": { + "port": 8080, + "scheme": "https" } } diff --git a/docs/fields.asciidoc b/docs/fields.asciidoc index d548e9f760..227fee26e3 100644 --- a/docs/fields.asciidoc +++ b/docs/fields.asciidoc @@ -33,15 +33,6 @@ Alias fields for compatibility with 7.x. -*`host.hostname`*:: -+ --- -type: alias - -alias to: beat.hostname - --- - *`event.timezone`*:: + -- @@ -194,16 +185,61 @@ Any arbitrary contextual information regarding the event, captured by the agent, -*`context.tags`*:: +*`context.custom`*:: + -- type: object -A flat mapping of user-defined tags with string values. +Object is not enabled. + +-- + +*`context.db.instance`*:: ++ +-- +type: keyword + +Field is not indexed. + +-- + +*`context.db.statement`*:: ++ +-- +type: keyword + +Field is not indexed. -- +*`context.db.type`*:: ++ +-- +type: keyword + +Field is not indexed. + +-- + +*`context.db.user`*:: ++ +-- +type: keyword + +Field is not indexed. + +-- + + +*`context.http.method`*:: ++ +-- +type: keyword + +Field is not indexed. + +-- *`context.http.status_code`*:: + @@ -213,6 +249,25 @@ type: long The status code of the http response. +-- + +*`context.http.url`*:: ++ +-- +type: keyword + +Field is not indexed. + +-- + +*`context.tags`*:: ++ +-- +type: object + +A flat mapping of user-defined tags with string values. + + -- @@ -267,6 +322,51 @@ Software agent acting in behalf of a user, eg. a web browser / OS combination. -- +*`context.request.body`*:: ++ +-- +type: object + +Object is not enabled. + +-- + +*`context.request.cookies`*:: ++ +-- +type: object + +Object is not enabled. + +-- + +*`context.request.headers`*:: ++ +-- +type: object + +Object is not enabled. + +-- + +*`context.request.env`*:: ++ +-- +type: object + +Object is not enabled. + +-- + +*`context.request.socket`*:: ++ +-- +type: object + +Object is not enabled. + +-- + [float] == url fields @@ -375,6 +475,24 @@ The http method of the request leading to this event. -- +*`context.response.headers`*:: ++ +-- +type: object + +Object is not enabled. + +-- + +*`context.response.headers_sent`*:: ++ +-- +type: boolean + +Field is not indexed. + +-- + *`context.response.status_code`*:: + -- @@ -449,6 +567,15 @@ Information pertaining to the running process where the data was collected +*`context.process.argv`*:: ++ +-- +type: object + +Object is not enabled. + +-- + *`context.process.pid`*:: + -- @@ -635,6 +762,372 @@ type: keyword The ID of the parent event. +-- + + +*`agent.name`*:: ++ +-- +type: alias + +alias to: context.service.agent.name + +-- + +*`agent.version`*:: ++ +-- +type: alias + +alias to: context.service.agent.version + +-- + + +*`client.ip`*:: ++ +-- +type: alias + +alias to: context.user.ip + +-- + + +*`observer.hostname`*:: ++ +-- +type: alias + +alias to: beat.hostname + +-- + +*`observer.listening`*:: ++ +-- +type: alias + +alias to: listening + +-- + +*`observer.type`*:: ++ +-- +type: alias + +alias to: beat.name + +-- + +*`observer.version`*:: ++ +-- +type: alias + +alias to: beat.version + +-- + + +*`host.architecture`*:: ++ +-- +type: alias + +alias to: context.system.architecture + +-- + +*`host.ip`*:: ++ +-- +type: alias + +alias to: context.system.ip + +-- + +*`host.hostname`*:: ++ +-- +type: alias + +alias to: context.system.hostname + +-- + + +*`host.os.platform`*:: ++ +-- +type: alias + +alias to: context.system.platform + +-- + + +*`http.request.method`*:: ++ +-- +type: alias + +alias to: context.request.method + +-- + +*`http.response.finished`*:: ++ +-- +type: alias + +alias to: context.response.finished + +-- + +*`http.response.status_code`*:: ++ +-- +type: alias + +alias to: context.response.status_code + +-- + +*`http.version`*:: ++ +-- +type: alias + +alias to: context.request.http_version + +-- + + +*`process.pid`*:: ++ +-- +type: alias + +alias to: context.process.pid + +-- + +*`process.ppid`*:: ++ +-- +type: alias + +alias to: context.process.ppid + +-- + +*`process.title`*:: ++ +-- +type: alias + +alias to: context.process.title + +-- + + +*`service.environment`*:: ++ +-- +type: alias + +alias to: context.service.environment + +-- + + +*`service.framework.name`*:: ++ +-- +type: alias + +alias to: context.service.framework.name + +-- + +*`service.framework.version`*:: ++ +-- +type: alias + +alias to: context.service.framework.version + +-- + + +*`service.language.name`*:: ++ +-- +type: alias + +alias to: context.service.language.name + +-- + +*`service.language.version`*:: ++ +-- +type: alias + +alias to: context.service.language.version + +-- + +*`service.name`*:: ++ +-- +type: alias + +alias to: context.service.name + +-- + + +*`service.runtime.name`*:: ++ +-- +type: alias + +alias to: context.service.runtime.name + +-- + +*`service.runtime.version`*:: ++ +-- +type: alias + +alias to: context.service.runtime.version + +-- + +*`service.version`*:: ++ +-- +type: alias + +alias to: context.service.version + +-- + + +*`url.domain`*:: ++ +-- +type: alias + +alias to: context.request.url.hostname + +-- + +*`url.fragment`*:: ++ +-- +type: alias + +alias to: context.request.url.hash + +-- + +*`url.full`*:: ++ +-- +type: alias + +alias to: context.request.url.full + +-- + +*`url.original`*:: ++ +-- +type: alias + +alias to: context.request.url.raw + +-- + +*`url.path`*:: ++ +-- +type: alias + +alias to: context.request.url.pathname + +-- + +*`url.port`*:: ++ +-- +type: long + +The port of the request, e.g. 443. + + +-- + +*`url.query`*:: ++ +-- +type: alias + +alias to: context.request.url.search + +-- + +*`url.scheme`*:: ++ +-- +type: keyword + +The scheme of the request, e.g. "https". + + +-- + + +*`user.email`*:: ++ +-- +type: alias + +alias to: context.user.email + +-- + +*`user.id`*:: ++ +-- +type: alias + +alias to: context.user.id + +-- + +*`user.name`*:: ++ +-- +type: alias + +alias to: context.user.username + +-- + +*`user_agent.original.text`*:: ++ +-- +type: alias + +alias to: context.user.user-agent + -- [[exported-fields-apm-error]] @@ -1310,26 +1803,6 @@ type: keyword Unique host id. --- - -*`host.architecture`*:: -+ --- -type: keyword - -Host architecture (e.g. x86_64, arm, ppc, mips). - - --- - -*`host.os.platform`*:: -+ --- -type: keyword - -OS platform (e.g. centos, ubuntu, windows). - - -- *`host.os.version`*:: @@ -1350,16 +1823,6 @@ type: keyword OS family (e.g. redhat, debian, freebsd, windows). --- - -*`host.ip`*:: -+ --- -type: ip - -List of IP-addresses. - - -- *`host.mac`*:: diff --git a/docs/spec/v2_system.json b/docs/spec/v2_system.json index 32795da96f..a162cec77e 100644 --- a/docs/spec/v2_system.json +++ b/docs/spec/v2_system.json @@ -4,7 +4,7 @@ "type": ["object", "null"], "allOf": [ { "$ref": "common_system.json" }, - { + { "properties": { "kubernetes": { "properties": { diff --git a/include/fields.go b/include/fields.go index 27ade5f37c..3bdf1d8a87 100644 --- a/include/fields.go +++ b/include/fields.go @@ -31,5 +31,5 @@ func init() { // Asset returns asset data func Asset() string { - return "eJzcPWtz27i13/MrznVnx8mMxNhOnKSa2fb6Jvvw7KbJbJLezrQdBQSPJDQkwACgFW2n//0OHiRBEtTDlrfp1YfsiiLOCwcHB+cBT+ETbmZAyuIBgGY6xxn8gBwlyeHq7esHABkqKlmpmeAz+MMDAIDvGeaZAiqKQnDQAm6IZKJSZgTgDXKtkgcAC/vazA6ZAicFziBnSiNnfGmfAuhNiTNDw1rIzD+LYDSfqyyTqBToFYJCeYMSmGoBguBJB1UpBUWlhEzM933xva1HWSjJgxGQlsvDYdphPaCaFag0KcoOuKUUVf0kFGQ4slLNo3pYLhrRmg8VFdczOA8ejUjXfN7XlIBYWDFbcoFxKBiVQiEVPFOgGKcIHzj7AlgKuurxQwXX+EWPcjM2vXwDRKZMSyI3NZCK5MD4QsiCmPdB4pLIzMx2Q94EKCl1JTGDdGMfk6V9LCwKkucbM283LGvfqBTKpCZnw0nB6AwWJFfYk/hA5Josh0IX6T+Q6uCxezCPKUYHpZYV7jc1V7DIiYaClKXV9YXlYZrhgnHMLFmwZnoFSkvzwg3JK1TJkIGV1uWAgXB6ouy345UmulJzKjLsEBhVv60sAbw3K9nCAwOv1jpDIkhUpeAKIywYzu/CghkfmIQQynCu9mChhlfTn4vlEjOzbJyeRUhg2TGQX2fINVswlAegxoKw/BjYvzOADuG5jGDtPdzO7tsam8EB6xVKDM2UAolUyAyziYHOqF37BNaYQirFOlj07fQxZQaKVBO7kBZSFBbmX6bfC7kmBpr5P1ghyVBODAXrFaMr+9KCSaUBuZYbA8U8aokUki0ZJznQnFmrH0FtlymsxBrNdlaw5UoDFxpSBI5mzyCS5RtjJ5Q2bBEFTAMl3LyxEHLpbBqBguSM2j14VP7WYFjbGJmHwGLvMRPvxEKvifSmFgjVxuYwQ9WK5AsjAWLxTQCXSXcK4DG8eWd8h5Rxa9QjK1zi5wqVvtMilzEl7wPYweeVobPMUSN8kPnEW1i6wgInsBJKA+EZlESvurMboavDHln3ftm2CHdSCd4OSbKeQMVLIhVm8OGXn2tN9OKcACZLa13V7PFj/EIMawkVxezp0yePFRJJV3/8/C3mRGlG3fffaVEmY3yUUmhBRX4fzNSwYzwkcOK4OBklbVHl90KWgTuBUijFUmNcjP5PiVJYpPlvI3SjdoPd60jc1bBHhB5QPy75Ukh9LwohpI7T9fTpk3FqiF7dl7QM7BFJ+Ykdl5L7/T6ocpD9uym6XelzhXJT+4Zxkns6OE76iqh7IdzA7dFm1lNNnxblSXRfM2trfoNSMcGP5dJZF9TD7NOUI3EnEAHa7OHhma5LWYF6JY7i6TU0OZCHktTuq86p/poOAJatyCmgptW5EKcXZ2enUSEvGGdqhTExp0LkSPgh+70fAoxnjBLr1qxXqFcoO0TBmqgGMwhpnLaIvNVGaSx2SnsLTW/8IdaD8hOTbJ+qnZvFttW6Y63GtglPW+uKG+nUvjh0PLyQOGNmmEZqDu/HJTCEfEciy5zohZDFcQmsod6RuN7paeRQtZOi9sxifVq9ItojVy1JEfX24bC76Pd1ENwpUZpTWGPIEGTF7VePKDjyZUQTKyQq8hyp7qz/frysQzPrT9eI4doptT9VBUpGG+KuXzWzifKGUax/GdOs45MyIOFUQUmkUajttNjI7/FU/F1XAA58zD669+6iQDWqccMY4/e4NvG6KCpN0hyh4uxzhdAxjp5ALJjWLnLZ7tBdOG4pDM+uADlJMZ9rLIzhwBmc/POfNnTwr3+d9N4UJfJ5zvinOeNzWkkz+XNN0kGw0XwqGQAdsDWFgvHas5rByWVylpz18ZmPJWUGJ0nymJTl408sJZz87nFG1CoVRGaPn56nl9nvL86mz19cnE/Pz/H59AV9+nz67DJ98fQyvaSL9Mkf5+Tbh9ZPnbn/zJ27OntIOMk3v+J8zfKMEpnN/ktP3IunPkyceCHbSP/sm4uLRjzfXFycPnr0aEh1n7lnhrkpycsVOf9NeMwJX1ZkibO8oshxK0d/a+f7byenhp2oVsec4Dsp9p+7HvBWVY5ShPyGScGLftDpKNYlAD6CvpZxFPcwFrQzcHMfJ8k/BbailGIpSVEY2da0Q6UwGz2Mxaf8zkT1Jn5fuppzRsU1G7GwX6HYPblfg6S3kNIceCQpcC3kp/8U8TYEfw0C3kpMczIZRMm/YvG6GPxXINoBIY13KQlXhOqWhEhWeFsats9Px3EezbJuSXSvMCQKrl91HMeoGzZwwf7McA3vSsJV6C/s7X+N+14918S6XT3p7/RGnuCTDJ+fnU2fZ3jmvJH0/Pxymi1+T39/lpGLbHF+K48rEFvCst2+Vo+Z0M26b55GPKweBw35rV8V6C1F+DpU1upse75zpGkRJCNd4CBFc3xUoEW/dsWdA79KbjxttSfpi5JSJLqtSvof9y0C9KXgmjDeFCSZcZ5yIDeE5fZoxjiQPPdSMiR3apQ6ojIAwqKhEUYMD+FJz1AICnlTn5KLJRSoFFmiSuA6eMsOY23kRKEtszG/U8EXbFlJFxNZsBwn5jl3YRmXMGbKmVkDk9nMNxc6BDZpojkek3//vbCoOnRMzG/20Ufz9WMDxxXQjNOVDIXWizduEVxDmw106UrytjxHlChd7NXHxQQP9NwSHsjOB4ki1BhP6lfB96CmfvM+qeluwluI6eUdrDrbyV/awjyN2fDgVWM6+e+mnOyks9Qzoms5SPxcMYlZZ0uqN73gPZ/um8FVtayUhotnegUXZ+fPJnB+MXtyObt8kjx5crGfdF1d2dopcm2szAJxgUYby2v46/sPbdHVWPFYUzhmK6GstHyVhNH3EqWbKJuuRxnxSJyceoiddejIsVPrtaXOa4TQxlbZEpZmTRkDVUewOhSglEKO2+w4ku/MoNoCUofR6C/JMubTCYwvhFnZlChrvyyeJnw2Zva9MRvY/l71yBbD70jzcJIBgl5WKRoP3QndABmCNrDutmk56F5N6sLZnBHVblJX/msEiv2pnpSFJbMoiWYpy5neuNKS58mXfuHs7+Cl29WM/qpQI8McT9/wOu5q4synJHoVM9KNollj0rOYW8E079Z05qLK2hLZGKnUvJL4aky5A0mBmiSREV1gjCtNOMWkcUn2gFcPmvtBIyD3EGgM6EC27veC0BXjmASKuAdUP2rejOoC9f6N1aH5HjMXQI4PHUyVsXAHCdePictW4rJ/CtwKzL/vVeyVoJ9Q7tAxZ+9Q7iY6s+CSwYghqD00YQBsqAYtnsJYv9sAtSNr42NF1BqfZgHatWKlmBFN4ubotf/VVTzSzlBldoqgnjrL5vaFeQ0yKIQf86HHVm/gViDd4TuEUY4uhQm8dSVY6CudgUg0ACewpDgBISFjS6ZJLigSnozS1rcEo7Rc+xeDU4tNltarejeG3X5xgyM8VeyHZWAmAjnri6TAjFXFduyvHQiri4chHzNCDQWVmiJRenpOd7hxASCw/jhrfW2mHDlMtU72FpULbVBAiv9l+mV/1fNDDC0/CLHM0a20cewdIzeC4Bf7zi7+/EJ3ZqBd6a/q7xHg3kYqbdyFJjvulrn7zaxZtRJSz53/2R75CacrIWt802aVjzTzNGTBQW0rEQsNt/PIPrh0awMQWBbzKbvG804YQ72w4OqzsSfAHGPSiuW6aUeKk9ILCt+CkpcNTteoNI7LRi6P0LWypYrDSsLhaZTWKHOrsj+6bxEg1+YoEiiqkBHT0+qmeb5TMz3uw/Ty7nPyo/esh7NxJE13BiKi5CO1VLfkoVtA9dAWYH558Wz+7OkEiCwmUJZ0AgUr1aMhKUIlkaKpW1Dy5l1bJ+VooMi1UBOo0orragJrxjOxHiFimPS4HQ0eThTHghQs39wZhQPjmZSYrYieQIYpI3wCC4mYqmwbt2zY0sX2rJ/5mSlbVn39dkpcgyWqIYKC0LsxWaNZEZmticQW2QQqVdlOnddXL0MaajvyqUpRctQYnLN/Cp9F0La/N25w16dtgUJoS7Zvi+2gnQaoQzQcZIZKkR1hewgkUIrM2bYoququpinA9FZk8OH61RCR+VeVJFLqdVtULcQhMpHhcSVoII6IcN/NdT9EDhoUpBxiIpwLbaPvR0MXgIzjPKbDEuClHd9lG9ojuGxRvJ1zNCmLaR1orQN5b1+7eF/fvtiHU1UiZQtGXR2qcVlcs3zMFNwwXPv46oNxJhyKGZxkgv41WgV2+vfEnnXruH6T3oSSMJ63hbjD/HU8d21ZaZLXeyWu40nrHQnr3Ynd54tnC0LPps/pM+ISu4RcXk6fpGfZ5QV9fk6fnf1G5YH7pauPztERigE7GQNgGTDaC7RtUzg7KrHbE+PL+SfcHFPdpjEaRzfGkbX8yiy1sNefcF9yIrGUqJDbJB3hPrkkqNFi35hKoBCcaWGG1tKs0R10AcBeu6W/duHpfgaqmwt3MxGxiVVeSjbsie3lXjzqizHU31fc1bxQkuc+9mAOrS7QwQoiN1CiLFFLooVvLN/SUhSqzN3s9A8e0k+46XWVO5U2VrZSNiRUI71t1Y7L5LxCTVj+NVbvXC7OXpAXz49vEIfL/Det4DmUrxGzGOEiUsUTail+oVjq2Gnwlr0qJBWV7rT55+ZgIcWa1ys4UM1tLVqDRroDaqPf1/bCdc4p1EGG3T5fkbJEjpnvojTuSkpUOGqkBHKY6oXxywKiZidKbXMpAnYTwXEaRFbdvi3FoHMQ2sPC4G6GvrWF8YzxLuSN0Y8CWxGe5Rhv+In1Ke4n0mvXpihkp0vRydZ2R5FqudKgRIGud6op5smwbVscHmrE8i4L5apbZ9CsmfoAbsx6c4fPQYslxxvs98YcpBAKb1AyvWm7TKmQY2XAdvuR80gB70E4ef9uGAl1UmhbEfI9rcC2CCTfdDfY7YuxJJIU821E3aqZ48oBRi3Zr5g1NMB3xlydvhRVntkbUajgHKkGLeAbddrvnGrujChR6k0NBZgCpVmeNyVPE1sBpFYWbIqAnyuS192GHQ4nkFbaVWWUOaG4ErnNjEq0X7MhBdfcrjNQTFf+TD2AWlfiGZR2QfmDIWixtOu3iZ43rcP+QHpiTqTvXAHaayMtah2XiHT9S76lunDvTiBnnxBevv1gJVBgIeQGKsepLZciEvvpolj12zB55D2j2NG30/+8t6P/0Q37WB/XFeSCti3QnqN+rdKDrrrSsnrQVc+uARtVy4+0rAaojdxsRq2d9rHwnRaa5AkXskhKOnTXFSU5ZvNFLoiOeK8lStrtPdhxcPADjG6JhaXT9o+o0hYDb+oEpm3DdBe3WTNElI71IAaXEtmiUlmQ3K5LD4lXRepumzKYqJA2iJQB0yAJX6LqQLNKdGZ0/fzs7JukP0VOCW85S27wYKK8Yh8yV4Mp6tWb1VOTbnSHv20TY+B6WiK5GqqrCNpDNlgLoV3FmNlZWEjE7R2wbQ8R7nV7wxjvu+5RcPQZLDWRjDsgCVzbFDclOa1yW8xqHNIMhPNM3rxL4A2Hnxmvvti7rgRXTGnV9u80MHtIy7wyYOnK62RaLRYolQX35t1ffKE0AVXZtvuQOPO6zehyQjW7qZ/bof/rsi4TP97uGP3tT9Q2K/EDDfCPA4Xv98sfpvF+dKDy9bquUxwTuyobix8Y+p7N3JKbD8zmLRRzX+O5TTdHDegOE7rNiO5xD8lxDemxTenQmPbFNlgTe0zeazumjWabWWL2DiBbJd3cGdAOLyUu2JcZnPzViv/vJ3tNqWK/3qe5sTXm1uTeMBlaxnDOVqTDSNOqqlQyRHd8+n5BxTKjSu9Qwzv2KybOGy+M2260IEKyoLQqmbtWsSDmH/fOw1+uXj9KwuyFEpWkWJCym8F4FzzuUNj8YBs5FCCXjK4wcw5vUDnoYjhuTh2S+YLlGmUzz1NokCchGVF/MPgdxkKv+2dK/59cH9E/I9Zt9jbu2ois09kVI+ro3f+1bJoCiMEEpBXPcqsRWBK9GkzEYSUCgro4QS2GhnOJObHbsj+oGXz13Vc2vq+7i6G0sZRwHbgn3SVQEh7P443dXW0TeQb4oYk23yTaXQoluWUz7GAWIikJoymwn9hfYSmRWkfs4R++hWfJ5aMErtzWnW/qG02Cm2VKElOFFX6Zj+dG9qSlmwkxmCbAuJYiq2hIX7zAYA/1O+w+bHsRulEOVGzJG+UklrI6lKaocPWyJOwqipC4ZwPKYST+5OUrFtDossQcb2x0qSaxuQwoE3YfeYjLGZxmaVIKpZcS1ec8sbH10wmc1tqboEzNd+tSn04ANY1JXlXpvXB2BYtK2pCmqtJpxm5Y2BRni5RdvVTLwwQ6Vxg+itRQ0GgS4K602shiLf5PjNv58Bd5Mb2qp6FKp5buJl/ZeoNWoxw/diImdcArwoTSRO5/N25IZw2h6juXoxdPReSwQxbm82axUKgHdjtYH6cq6Apk7h7rTR0Ttgz2rUDnDvqIHma+NXVvwfxbJPOqbqDdzl13UNObOWTRfBgvKz2vXwoB9V4UlQ7fJOo1y3O29V2zNzCX+DuL3W/IKQwEPkxnbE2q2QwGqk4Gwy6HNVGAX5BWZp0YVCspuKhUvgEhgXSexLIYrqs8ujfeYWt8b6P4I5ujb2TvbJWBZ9JtOw0clPedH7ox+/ang8qOjn7pRnSL7SUiiirXbL7N9MQ90duUC4xtzuHFHsM9WjG+zFsf+6G9XPSH797D40qhVI9nLDuNbXN3bx+99VadwKl3ds12nBL6ycwhz/4hUr8t38EahgQf30juOvnYQ3vWsYlMhTP4n2sbJaoqH1qfg++pcXCCez8aFxN+fP/+befqXGMRzMOp3XkxC1+P7ZkFkZ/+TX/OpPPnS4I/a2L100Z7LXFu+tuZGGMi4eSGLa0GvGcF48P89XauRmKK5jHjy/mCUC3kDM7P7GeU+eEOadvOhueig/bIYAtQbbLu1MM+hTXLc2Cc5lWG9pKT8NaTJv+ejMDhQtdkOkjmwYrcuP1Lga0TcXWI8AoXpMq1sh6drKJ3epaEz61zdCcXNZOiLEfqJbZdwgW9z1iqpwsxYroGc9JGGdugnafSS8q1LnUNWPLg/wIAAP//9QNksQ==" + return "eJzsff9z27ix+O/5K/Bx5sbJjMTYTpxcNdP245dcr542Teac9HWm7SgguBJRkwADgFZ0nf7vb/CFJEiCFGXJd8mb5x/uIorYXSwWu4vdxWqObmG7QLjIHyGkqMpggX4EBgJn6Or920cIJSCJoIWinC3Q7x4hhNAfKGSJRITnOWdIcXSHBeWl1CMQ3AFTMnqE0Mq8tjBD5ojhHBYoo1IBo2xtniKktgUsNA0bLhL3LIBR/10liQApkUoBSRB3IBCVDUDEWdRCVQhOQEouIv15Kr731SgDJXo0ANLMcn+YZlgHqKI5SIXzogVuLXhZPfEZ6Y8sZf2oGpbxmrX6j/CSqQU69x4NcFf/fagoQXxl2GzIRZShnBLBJRDOEokkZQTQR0a/ICg4STvzIZwp+KIGZzO0vGyLsIipElhsKyAlzhBlKy5yrN9HAtZYJHq1a/JmiOBClQISFG/NY7w2j7lBgbNsq9ftjibNG6UEEVXkbBnOKVmgFc4k7OA4KaXieY/rPP4XEOU9BobjDJI2UP1HWQJfQl9ghrPtz6FvJGBBUg0vMGq91ixR/rc9opO4R7C/HqH5+sMpkwozAq0vw1LfmmWf2vF57prrzvn6REuFFeTNHv02qNbUfVME64301RNcEZsqVezcCUF1gHZskRxUyruTvS8jAlPZjxkJJ8s7nJUgd26RUi4JT8Iy1zElaNx26L8P2jIbqEhDrayI5jsSIAvOZGVRu8SUIvsG2FfvU7zum96eEbAPlkMTqSVNiRKmGegrtMqwQjkuCuPxrMwGnCewogwSQxbaUJUiqYR+wU4j6k+gs2/HjcKjwHJJEJ5XtWvFRuXmgzPIGl4lMhlfryHRnoc11QESaHIM5NcJMEVXFMQeqCHHNDsG9h80oH3mXASwdh6OT/d9hU3jQJsUBPienkQCCBcJJDMNnRLjPmG0gRjFgm88v6lZPir1QB4rbKRwJXhuYP5t/gcuNlhD0/9CKeAExExTsEkpSc1LKyqkQsCU2Goo+lFDJBd0TRnOEMmocZwDqI2Mo5RvQJ8IcrpOFWJcoRgQA+12Y0Gzrd5kUulpYYmoQgQz/caKi7V1CzHKcUaJOcYM8t/sNuNeBtbBc3onrMQNX6kNFs5bRZgovWGppirF2UpzABt8MwTrqL0E6Bl6d6OPXzFlxi8O7HABn0uQ6pBNHvNkG5hnT8+NuLsjDu+4yR9X0jv8k+Ykwm8pyG96DnbTfENz8L9/jChT+BYQ6DMcgY779DjoVD1unySZmvfc4sejKvdxRwDuNXRgq/eHh+wDu/t21itAv+TkFkIq7iudgv/9gSJnpx4BI2JbKOhKR8WImPMMMBsHIiDnCpbYxq32F6OuTxxW4DvszJW2E0UGCtBHkc2ce0hSyGGGUi4VwixBBVZp27oG7IJPm8Cbnvs/5rHvPDtYP1DgzQyVrMBCQoI+/vTnyhNw5myGIFqbA4VcPHsGX7CeWkR4vnjx4vkzKzm///xbyLBUlNjPjxUvoqF5FIIrTnj38HGUyVSwQ3OI0ImdxckgaasyexCyNNwZKriUNNbOndZxcywl5Hoj/xJM12LXOz0caXYV7AGme9QPc77gohs3Oo5AcKHCdL148XyYGqzSh+KWhj3AKbeww1yy3z8EVRayezcGeyr4XILYVgfbMMkdGRwmPcXyQQjXcDu06f1U0ad4cRI8V+i9tbwDISlnxzpSm6iLg9mlKQNsg+gcKX2G8tMSbcoCka1DabIg9yWpOdfYONIhB5v/86kP8KkPcYwd45cyfIIOeVWDob6pvuxAlHMgxjlNhANBzkou7XH99OLs7DS4oVaUUZlCaEuFpr/Dt3NDNJcowSaEsElBpSBaRKENljVmxAViPLS35FYq6Ge3eiHyYZreuZybA+VkMRrfljsdgzHNPCEk3XUJHG1N2Etzp4p7oVY0xSdOb1qqgKhSHJlAH/KBRBYZVisu8uMSWEE9kLhOpHIggLmToiY+aM4vKsXKIZcNSQHxdtn7Q+T72stFFyAUpqw2WoBEycxHh8gLryZYYcMkwrMMSPtgOZZZwmJ9F2RawFKN2qpRa/WL5QsLGs6S7Z9r+kuZg6Ck5vX1m1o4QdxRAtU3Qxvl+KT0SDiVqMBC749xWkzdzfF27E2bARZ8SN3b9w7ZDxWqYT0fmu9xVfx1npdG8lDJ6OcSUEvXOwIhp0rZupHGuWzDsTs7lIrMcAzZUkGu9SAs0Mm//22yDv/5z0nnTV4AW2aU3S4pW5JS6MVfKhz3knz6rxQe0N605iinrDoULNDJZXQWnXXx6T9DygKdRNEzXBTPbmmMGX78LMEyjTkWybMX5/Fl8puLs/mr7y/O5+fn8Gr+PXnxav7yMv7+xWV8SVbx898v8W+fmCPWwv5vaU9aiydOMyw3NEsIFsni/6mZffHUFelEjsmmzmrx3cVFzZ7vLi5Onz592qe6O7mXenJznBUpPv9F5phhti7xGhZZSYDB6Iz+0az3P05O9XSCUh06vx0k2H9tH95GRTlIEbA7KjjbswxmonbxgA+gr3gcxN0PY+6MOT5EEOQvnq4oBF8LnOeatxXt+riTDMYRwkt+MFGdhZ9KV31ELpmiAxr2K2S7I/dr4PQIKfX5TeAcNlzcfivsrQn+Ghg8Skzt8/aCC18xe236/itgbY+Q2rsUmElMVENCoCZ3nyLYluM8WN00Umacgk8Uun7TchyDbljPBfsrhQ26KTCTvr8w2f8a9r06rolxuzrc3+mNPIfnCbw6O5u/SuDMeiPx+fnlPFn9hvzmLMEXyer8Xh6Xx7aIJrt9rc5kfDfroec04GF1ZlCT3/hVntwSQF+HyBqZbc53ljTFvTomGweJQR8fJVK8e3PAngO/ytk42nxP8jH64fVNawK+Uj6Q/I5KttBwRrEfmimwSuu7BLU/bqiw9zl6YPtqdn/IFYzW1G3x2ZGWrl+BPE6fqUWjRZsiHtsLMMehKRB1HaMsBqyiekwPWveGzy5wzfs9UJ0cxU6ijiEXBlBQDPSUj8PwgUjyJHE1kdeoBeFwIXNQKzE7QDg6EIfFhO+Ow46FkEbj3F3SBsmrobQXurkncOBCu+Rm1MuiTmFhZ3AAuKuoD6SUpsHvjh9GEc6i7YXFB3Ek3V1xqJU+b5vdVsj/wNUsAnZ3nMAq7KtH9sEdAC8IsBtG3geiHdviXTs8fE/ePTa12JTV/gTaGZ/ax2r7EKbhDJ3f91c/gxHsCarHkV4T0jFbaGJI8T6YWntkJ6sC8btfhVMVHQ/OqBpRj09HcF3bxI8yvh/A+1X47sh4cLZXeAa5fphnHzQMTXThQKOQ8BzT+xquUmQjLtJK4PU9dGMLOJZpAHC7snRfoGZ436Fzd3UOACzwJmAmsUoPgFlXT3pbL/geF6o6RaP579pp4IHC0GDCeEfga2oNaIXShE0OmL+r0tw1+6pUOaestFUUJ4uTHjG2brxHzd7RPwtnrC76pBM/8S4sHrhhu5f3Jh/C7cD+YWtfJ86e6AP+2z2Mi4FVX8bscWxpoxrV1ox6fRl86AOQXa3fI9cpRJ+Pm1Yh/2U/BZb7NWcKU1Z3CdHj3JogfIdpZjL2lCGcZS54pklqNQ5pzac54Y+KmBYvvwBAU4gksLppRMbXKAcp8RpkhK69t8ww2jBbgrHQ+nvC2YquS2Erf1Y0g5l+zmzxkb2CSKWNvmuY1NylZFz5wGZ1zZLD5N7/wA2qFh0z/Z159El//FTDsV0thumK+kzrHOFHGFfTZsq5VClY0zODFyBshaGr/uLMC38awj3euVKoADXa1P/M2QRqqjcfkpq2azFCTKeS2oizWfy16ZajIOnn4ytMJ/+/7vFy0tp+CVYVH7QSpAKSVqaiyoV477kLDAt0Va5LqdDFS5Wii7PzlzN0frF4frm4fB49f34xjbu22cvGCnIVw9YbxJbTmYq1en7dtFJzB36oo0vdzcVcTDfccvdutbwXIOxCmQtIIAKJKsunDmKrHVp8bBXDjVy7HyC01lXmUnS9p7SCqgqbWhSAEHzEGoWR/KAHVRqQWIxafnGSUFc0S9mK651NsDT6y+Cpq6qGDJpTZj3L0bmPPGKSLWkOTtRDEIz6TPd6LHQNpA86GOHdy52w0J2YVN2snFVzRurKfQxAMV9Vi7IyZOYFVjSmGVVbe1nuVfSl283qMXptrZqWX+lLZC0iRg10dN2QzW3rxQpDxsuk6TgVQkL0K5FrbiR2IMlB4Sgwog2savYT1W7NBHjVoKUbNACSddtwTQNaD2uDzTFJKYPIE6EJUN2oZT2qDdR5Jmb1lxNWzoMcHtpbKq2b9mKuGxPmrYB1N60/Csy970TsDSe3IHbImNVU1mUdx5MYcFFvRB/UBEnoAeuLQYMnr6ND+1KoR1Zqw7CoURv1BjR7xXAxwQqHFclb963tfkFaQ6XW8V57siRZmheWFUivr9yQ9zu0ez2HAMgOq++XrbQpjNB7ex0UXMsYhAVogDO0JjBDXKCErqnCGSeAWTRIW1cTDNJy7V700tCmmL/a1bsx7PZoaxz+eWAalp6a8PisLqIcElrm49jfWhBGFvdDPqSEagpKOQcs1fyc7HDAPEDIeNK08ZKptORQ2bjHIyLn6yCPFPfN/Mt00XNDNC0/cr7OwO60YewtJTeA4Cfzzq75uY1u1UCz099UnwPAnY6UShv6+vaG3eb2O71nZcqFWlrPsQlDYEZSLip883qXD/TGrMlCQb9ysCdhX0Oj+/lSH239fA0Q0STkDbaV50EYfbkw4KpTrSNAH0Dikmaq7u4ZJiUYOtmLktc1Tmb6fg7jMqVoR2j/NXLLyHDC4qmF1pUfOJH9o/0UAHKtDxGeoHIRUD2NbOrnOyVztPRhrzKfvfjwRxeO6K/GkSTdKoiAkHMZDSUd9sLz7qYKHARxrHBOs350d18UFgx6YuKnApIUqxlKIKaYzdBKAMQymaENZQnfyKd9SnJMDqPhz1SaoHaKRbLBAuau5wnIGSplaZqIvb163Tyupfq2jEEwUOCd1/7kPwtgbb6vnbK2h9UARb5kjyvpZtDO7dAiGu21KQqeHEFZeRwoeILC6SONqjx0o3iY3vMEfbx+E45bywIHbpLdF1UDsY+MJ3BcDmqIAyycquqnIbLQUI4DZVaYMa5MFPdo6DyQYZzHNJ8eXtKypGNoj+BABPG2TnW4yOdVwK4KCL1/a+NGXf1iHs5lAYSuKLG3drUBtZ3QQ6rgjsLGxekeDU/Coligk4STvwcvmZ3+MzInryo+XFdPowJTljU1TP3y+HBpvJlKXRs/qS4+XBO/ox5+d934q9XLFSZn81fkJbZ14xhfXs6fx2fJ5QV5dU5env1Ctw+nVcMffUZHuGvYijwjmiBKOmGfMYEzoyJjnihbL29he0xxm4doHDSMA3v5jd5qfiN3zNyNFgGFAAnMJHswc0kKTrQUu5aZGOWcUcX10IqbFbqx1PD93ErXU//FNAXVLrW3KxHQiWVWCNqvLejE8B3qiyHUfyiZvVJDcJa5k7A+QtljN82x2KICRAFKYMVdy9uRZju+yBymp390kP4E206/WyvSWsuW0gQoKqT3vRRkMwJvQGGafY2Xgy5XZ9/j718dXyH2t/kvekFo33kNqMXALAKXhHwphS8EChU6q92zsweOealaDYgzfbAQfMOqHeyJ5lhDm8Hm6hOuXn+o9IXtMyRBeZla8zzFRQEMElc5o92VGEt/1MANy37KEA2pHBRWO0Fq63bN0E4ohmngSXn/rhcanYXQHBZ6XaO72hYNZx53Ia+VfhBYilmSQbifSKir0zSWXtumTly0ejpZ3ppeMrhcpwpJnoPtNFMXhSTQNHnqH2p4/2rOHhvlqp2vrvdMdQDXar3+gZa9NksGd7DPrwD0BELCHQiqtk1hGeFi6JaxMT9iOVgnOxEn63atF/UPpozdcX6gHdgUE2TbtoEd34wFFjhfjhF1r14RVxYwKEF/hqSmAf2g1dXpa15miamDJpwxIAopjr6Tp93GLHU31QKE2lZQEJVIKppldenMzFSSyNSAjQHB5xJnVW+m1gxnKC6Vze4XGSaQ8szk6QSYj0mfgmtm9hmSVJXuTN2DWlV0aZRmQ7mDIVJ8bfZvHfWqG625A+mJPpHe2EKmt5pbxDguAe66l1wDuty+O0MZvQX0+v1Hw4Ecci62qLQzNWU3WEA3eRGqouqnMpxnFDr6trrFTXb0P9lhn6rjukQZJ03DODejbs3Lo7a4kqJ81BbPtgIbFMtPpCh7qDXfTH6nWfah8J3iCmcR4yKPCtJ31yXBGSTLVcaxCnivBQjSrufeVSxsB2jZ4itDp2lPIQtz13hbpdNMlyf7q1xGDWGpQi2OvJ9LMMWJIseZ2ZcOEivz2P4OhsZEuDBBpARRhQRm604jfSNEZ1rWz8/Ovou6S2SF8J6rZAf3FsoJ9j5r1VuiTt1StTTxVrXmN7YwGq6jpR9Lx0SVAbT7GFgDodnFkJhVWAmA8QZbzb0BmNTrcmjuu7pOWvo0lopIyiyQCF2bhCvBGSkzUxSpHdIEceuZvLuJ0DuG/kxZ+cX8Cgdnkkolm/YgNcwO0iIrNViSOpmMy9UKhDTg3t38zRXcYiRL06TQJ06/bvKLDBNF76rnZuh/25zIzI03FqNr/nilsyI3UAP/1BP4bnfB/STejfZEvtrXVYpjZnZlrfE9Rd/RmSOZYk9t3kMwpyrPMdkcVKA7VOiYEp3QtfW4ivTYqrSvTLts6+2JCYv31oxpotl6lajpjm2qbeuWhM3wQsCKflmgk78b9v/zZNKSSvrzQ6obU6tsVO4dFb5m9Ncsxa2J1BfppIz66I5P308gaaJF6QYUuqE/Q2S98Vy77VoKAiRzQsqC2h98yrH+j33nyU9Xb59GfvZC8lIQyHHRzmDceI9bFNZfmAsBEgETlKSQWIfXq2OzMRy7phbJckUzBaJe5zmqkUc+GUF/0PseDYVep2dK/5d0p+yeEasufibuWrOs1TgmRNTRmwtWvKnLE3oLEJcsyYxEQPAO3n4VApzYOEHFhnrmAjJszLI7qGl81ZUwE99X7c1QmFiKvw/sk/YWKDAL5/GGfpjYJPI08H0Tba4HVXsrFPievbZ6qxBISWhJQdPY/gYKAcQ4Yk9+91v0Mrp8GqEra7qzbdUw1WtcW+CQKKTwZTmcG5lISzsTojHNEGVK8KQkPn0TL8YNJmYm/tix+ZVrLRwg6ZrVwokNZVUoTRJuqzexfzslQOLEiwz7kfgnx1++QrUsC8jgzkSXKhLrXsP2KjJ6AusFOk3iqOBSrQXIz1lkYuunM3RaSW8EItafjUt9OkOgSIjzsowfZGZXaFUKE9KUZTxP6B31L1eZkllbzdTMYYZaP+7xNFBDQYJJgENpNZHFiv23lJn1cG3PqUqrZShj85sKTb6y8QaNRNn5mIWYVQGvwCSkwoE7xvvc/C+7zuWAvxPkww5e6L93q5UE1dPb3v44ld7tMmp/YXNbxYTNBLtaoPUD4wE5TNwVx8mM+VU486a6iDk+u/ag+o5ff4rIdG0vSrWsXvIBdV7kpfLfxPItzTI6+q62DdQm/s5CvwbBCOoxvJ/OGE2qmQwGyFYGw2yHDZYIvgAp9T7RqFLBGS9ltkVcINx6Espi2KZ1Qdt4gGn8YKL4A8bR9clrmUrPM2lfX/QclA+tL9ox++arvcqOjt7TM2hiO4mIvMwUXY6pnrAnep9ygSHj7PcN7dtoSdk6a3zsJ+anWH784QN6VkoQ8tmCJqchM3f4NcR7m+oInTpnV5vjGJNbvYYs+RePnVk+QBv6BB9fSe46+ZhDe9LSiVT6K/jt6kYBssz62mfvRhgWjtdWtHYx0R8/fHjf+qEhrRH0w7mxvJD4r4dsZo7F7a/0K+WtXyX3fq3cyKeJ9hri7PI3KzE0iYjhO7o2EvCB5qHWkuOzGogp6seUrZcrTBQXC3R+Zv4GJ9+3kOYSVP9ctJeN9EyAbJJ1pw72KdrQLEOUkaxMwDTL8Ltn1Pn3aAAO46oi00LSD1J8Z+2XRKZOxNYhojewwmWmpPHoRBn8yZACs6Vxjg5yURPBi2KgXmKsxzfq/A2letoQA6qrtyZNlLEJ2jkqHafsRZq2Aose/U8AAAD//ws4QMg=" } diff --git a/model/ecs.go b/model/ecs.go new file mode 100644 index 0000000000..161a57eabb --- /dev/null +++ b/model/ecs.go @@ -0,0 +1,44 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package model + +import ( + "strconv" + "strings" + + "github.com/elastic/apm-server/utility" + "github.com/elastic/beats/libbeat/common" +) + +func CopyECS(fields common.MapStr) { + // context.request.url.protocol -> url.scheme (minus trailing colon) + if scheme, err := fields.GetValue("context.request.url.protocol"); err == nil { + if schemeStr, ok := scheme.(string); ok { + utility.MergeAdd(fields, "url", common.MapStr{"scheme": strings.TrimSuffix(schemeStr, ":")}) + } + } + + // context.request.url.port -> url.port (as int) + if port, err := fields.GetValue("context.request.url.port"); err == nil { + if portStr, ok := port.(string); ok { + if portNum, err := strconv.Atoi(portStr); err == nil { + utility.MergeAdd(fields, "url", common.MapStr{"port": portNum}) + } + } + } +} diff --git a/model/error/event.go b/model/error/event.go index 2dd351b3f5..80d12e34ac 100644 --- a/model/error/event.go +++ b/model/error/event.go @@ -200,6 +200,8 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event { utility.AddId(fields, "parent", e.ParentId) utility.AddId(fields, "trace", e.TraceId) + m.CopyECS(fields) + if e.v2Event { if e.Timestamp.IsZero() { e.Timestamp = tctx.RequestTime diff --git a/model/metadata/generated/schema/metadata.go b/model/metadata/generated/schema/metadata.go index 25a3509302..00e0540b60 100644 --- a/model/metadata/generated/schema/metadata.go +++ b/model/metadata/generated/schema/metadata.go @@ -161,7 +161,7 @@ const ModelSchema = `{ "maxLength": 1024 } } }, - { + { "properties": { "kubernetes": { "properties": { diff --git a/model/metricset/event.go b/model/metricset/event.go index eb14a9a32d..6d0d6b2034 100644 --- a/model/metricset/event.go +++ b/model/metricset/event.go @@ -165,6 +165,7 @@ func (me *Metricset) Transform(tctx *transform.Context) []beat.Event { context := common.MapStr{} if me.Tags != nil { context["tags"] = me.Tags + fields["labels"] = context["tags"] } fields["context"] = context diff --git a/model/metricset/event_test.go b/model/metricset/event_test.go index 149a209963..1775168620 100644 --- a/model/metricset/event_test.go +++ b/model/metricset/event_test.go @@ -209,6 +209,9 @@ func TestTransform(t *testing.T) { "a": common.MapStr{"counter": float64(612)}, "some": common.MapStr{"gauge": float64(9.16)}, "processor": common.MapStr{"event": "metric", "name": "metric"}, + "labels": common.MapStr{ + "a.tag": "a.tag.value", + }, }, }, Msg: "Payload with valid metric.", diff --git a/model/span/event.go b/model/span/event.go index 7f51801d18..5bbf72df35 100644 --- a/model/span/event.go +++ b/model/span/event.go @@ -187,6 +187,8 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event { utility.AddId(fields, "parent", &e.ParentId) utility.AddId(fields, "trace", &e.TraceId) + m.CopyECS(fields) + timestamp := e.Timestamp if timestamp.IsZero() { timestamp = tctx.RequestTime diff --git a/model/transaction/event.go b/model/transaction/event.go index 966f001b60..4bd4c1f1a8 100644 --- a/model/transaction/event.go +++ b/model/transaction/event.go @@ -23,6 +23,7 @@ import ( "github.com/santhosh-tekuri/jsonschema" + m "github.com/elastic/apm-server/model" "github.com/elastic/apm-server/model/span" "github.com/elastic/apm-server/model/transaction/generated/schema" "github.com/elastic/apm-server/transform" @@ -201,6 +202,8 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event { utility.AddId(fields, "parent", e.ParentId) utility.AddId(fields, "trace", &e.TraceId) + m.CopyECS(fields) + if e.v2Event { utility.Add(fields, "timestamp", utility.TimeAsMicros(e.Timestamp)) } diff --git a/model/transaction/event_test.go b/model/transaction/event_test.go index a1914ce239..4dc1e255e7 100644 --- a/model/transaction/event_test.go +++ b/model/transaction/event_test.go @@ -195,12 +195,12 @@ func TestEventTransform(t *testing.T) { }{ { Event: Event{}, - Output: common.MapStr{ + Output: common.MapStr{"transaction": common.MapStr{ "id": "", "type": "", "duration": common.MapStr{"us": 0}, "sampled": true, - }, + }}, Msg: "Empty Event", }, { @@ -209,12 +209,12 @@ func TestEventTransform(t *testing.T) { Type: "tx", Duration: 65.98, }, - Output: common.MapStr{ + Output: common.MapStr{"transaction": common.MapStr{ "id": id, "type": "tx", "duration": common.MapStr{"us": 65980}, "sampled": true, - }, + }}, Msg: "SpanCount empty", }, { @@ -224,13 +224,13 @@ func TestEventTransform(t *testing.T) { Duration: 65.98, SpanCount: SpanCount{Started: &startedSpans}, }, - Output: common.MapStr{ + Output: common.MapStr{"transaction": common.MapStr{ "id": id, "type": "tx", "duration": common.MapStr{"us": 65980}, "span_count": common.MapStr{"started": 14}, "sampled": true, - }, + }}, Msg: "SpanCount only contains `started`", }, { @@ -240,15 +240,83 @@ func TestEventTransform(t *testing.T) { Duration: 65.98, SpanCount: SpanCount{Dropped: &dropped}, }, - Output: common.MapStr{ + Output: common.MapStr{"transaction": common.MapStr{ "id": id, "type": "tx", "duration": common.MapStr{"us": 65980}, "span_count": common.MapStr{"dropped": common.MapStr{"total": 5}}, "sampled": true, - }, + }}, Msg: "SpanCount only contains `dropped`", }, + // duplicates some integration tests + { + Event: Event{ + Context: common.MapStr{ + "request": common.MapStr{ + "url": common.MapStr{ + "port": "80", + "protocol": "http:", + }, + }, + }, + }, + Output: common.MapStr{ + "context": common.MapStr{ + "request": common.MapStr{ + "url": common.MapStr{ + "port": "80", + "protocol": "http:", + }, + }, + }, + "transaction": common.MapStr{ + "id": "", + "type": "", + "duration": common.MapStr{"us": 0}, + "sampled": true, + }, + "url": common.MapStr{ + "port": 80, + "scheme": "http", + }, + }, + Msg: "ECS transform numeric request port and protocol with trailing colon", + }, + { + Event: Event{ + Context: common.MapStr{ + "request": common.MapStr{ + "url": common.MapStr{ + "port": "notanumber", + // protocol present so that output url.* is populated + // as tests do not check missing entries + "protocol": "http:", + }, + }, + }, + }, + Output: common.MapStr{ + "context": common.MapStr{ + "request": common.MapStr{ + "url": common.MapStr{ + "port": "notanumber", + "protocol": "http:", + }, + }, + }, + "transaction": common.MapStr{ + "id": "", + "type": "", + "duration": common.MapStr{"us": 0}, + "sampled": true, + }, + "url": common.MapStr{ + "scheme": "http", + }, + }, + Msg: "ECS transform non-numeric request port", + }, { Event: Event{ Id: id, @@ -262,7 +330,7 @@ func TestEventTransform(t *testing.T) { Sampled: &sampled, SpanCount: SpanCount{Started: &startedSpans, Dropped: &dropped}, }, - Output: common.MapStr{ + Output: common.MapStr{"transaction": common.MapStr{ "id": id, "name": "mytransaction", "type": "tx", @@ -270,7 +338,7 @@ func TestEventTransform(t *testing.T) { "duration": common.MapStr{"us": 65980}, "span_count": common.MapStr{"started": 14, "dropped": common.MapStr{"total": 5}}, "sampled": false, - }, + }}, Msg: "Full Event", }, } @@ -279,7 +347,9 @@ func TestEventTransform(t *testing.T) { for idx, test := range tests { output := test.Event.Transform(tctx) - assert.Equal(t, test.Output, output[0].Fields["transaction"], fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) + for key, expected := range test.Output { + assert.Equal(t, expected, output[0].Fields[key], fmt.Sprintf("Failed at idx %v; %s", idx, test.Msg)) + } } } diff --git a/processor/error/package_tests/TestProcessErrorFull.approved.json b/processor/error/package_tests/TestProcessErrorFull.approved.json index 7bc92db3e9..f2dd1f703f 100644 --- a/processor/error/package_tests/TestProcessErrorFull.approved.json +++ b/processor/error/package_tests/TestProcessErrorFull.approved.json @@ -252,6 +252,10 @@ }, "transaction": { "id": "945254c5-67a5-417e-8a4e-aa29efcbfb79" + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/processor/error/package_tests/attrs_common.go b/processor/error/package_tests/attrs_common.go index 9716299c07..908a1eceea 100644 --- a/processor/error/package_tests/attrs_common.go +++ b/processor/error/package_tests/attrs_common.go @@ -47,9 +47,11 @@ func payloadAttrsNotInFields(s *tests.Set) *tests.Set { func fieldsNotInPayloadAttrs(s *tests.Set) *tests.Set { return tests.Union(s, tests.NewSet( - "listening", "view errors", "error id icon", + "listening", "view errors", "error id icon", "context.response.headers.user-agent", "context.user.user-agent", "context.user.ip", "context.system.ip", - "context.http", "context.http.status_code", + "context.http", "context.http.method", "context.http.status_code", "context.http.url", + tests.Group("container"), + tests.Group("context.db"), tests.Group("timestamp"), )) } @@ -101,7 +103,7 @@ func condRequiredKeys(c map[string]tests.Condition) map[string]tests.Condition { func keywordExceptionKeys(s *tests.Set) *tests.Set { return tests.Union(s, tests.NewSet( "processor.event", "processor.name", "listening", "error.grouping_key", - "error.id", "transaction.id", "context.tags", "parent.id", "trace.id", + "error.id", "transaction.id", "context.tags", "labels", "parent.id", "trace.id", "url.scheme", "view errors", "error id icon")) } diff --git a/processor/metricset/package_tests/TestProcessMetricset.approved.json b/processor/metricset/package_tests/TestProcessMetricset.approved.json index c543ec0ee0..cad3d00ba3 100644 --- a/processor/metricset/package_tests/TestProcessMetricset.approved.json +++ b/processor/metricset/package_tests/TestProcessMetricset.approved.json @@ -46,6 +46,10 @@ "double_gauge": 3.141592653589793, "float_gauge": 9.16, "integer_gauge": 42767, + "labels": { + "code": "200", + "some.other.code": "abc" + }, "long_gauge": 3147483648, "negative": { "d": { diff --git a/processor/metricset/package_tests/TestProcessMetricsetMultipleSamples.approved.json b/processor/metricset/package_tests/TestProcessMetricsetMultipleSamples.approved.json index 91d8dc8b60..e8cd8ea398 100644 --- a/processor/metricset/package_tests/TestProcessMetricsetMultipleSamples.approved.json +++ b/processor/metricset/package_tests/TestProcessMetricsetMultipleSamples.approved.json @@ -39,6 +39,9 @@ "http": { "requests": 1 }, + "labels": { + "url": "/foo" + }, "processor": { "event": "metric", "name": "metric" @@ -83,6 +86,9 @@ "http": { "requests": 6 }, + "labels": { + "url": "/bar" + }, "processor": { "event": "metric", "name": "metric" @@ -127,6 +133,9 @@ "http": { "requests": 2 }, + "labels": { + "url": "/foo" + }, "processor": { "event": "metric", "name": "metric" diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationErrors.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationErrors.approved.json index 0b98bf4d3b..eac01fb3b9 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationErrors.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationErrors.approved.json @@ -268,6 +268,10 @@ }, "timestamp": { "us": 1494342245999999 + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationMetricsets.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationMetricsets.approved.json index da6753b359..310f1c6d0f 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationMetricsets.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationMetricsets.approved.json @@ -30,6 +30,10 @@ "double_gauge": 3.141592653589793, "float_gauge": 9.16, "integer_gauge": 42767, + "labels": { + "code": "200", + "some.other.code": "abc" + }, "long_gauge": 3147483648, "negative": { "d": { diff --git a/processor/stream/approved-es-documents/testV2IntakeIntegrationTransactions.approved.json b/processor/stream/approved-es-documents/testV2IntakeIntegrationTransactions.approved.json index 09416a199f..e4ae91e348 100644 --- a/processor/stream/approved-es-documents/testV2IntakeIntegrationTransactions.approved.json +++ b/processor/stream/approved-es-documents/testV2IntakeIntegrationTransactions.approved.json @@ -228,6 +228,10 @@ "started": 17 }, "type": "request" + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/processor/stream/package_tests/error_attrs_test.go b/processor/stream/package_tests/error_attrs_test.go index 8c275489c1..7768cd9e5f 100644 --- a/processor/stream/package_tests/error_attrs_test.go +++ b/processor/stream/package_tests/error_attrs_test.go @@ -59,9 +59,11 @@ func errorPayloadAttrsNotInFields() *tests.Set { func errorFieldsNotInPayloadAttrs() *tests.Set { return tests.NewSet( - "listening", "view errors", "error id icon", + "listening", "view errors", "error id icon", "context.response.headers.user-agent", "context.user.user-agent", "context.user.ip", "context.system.ip", - "context.http", "context.http.status_code", + "context.http", "context.http.method", "context.http.status_code", "context.http.url", + tests.Group("container"), + tests.Group("context.db"), // we don't support these yet "kubernetes.labels", @@ -129,8 +131,8 @@ func errorCondRequiredKeys() map[string]tests.Condition { func errorKeywordExceptionKeys() *tests.Set { return tests.NewSet( - "processor.event", "processor.name", "listening", "error.grouping_key", - "context.tags", + "process.args", "processor.event", "processor.name", "listening", "error.grouping_key", "url.scheme", + "context.tags", "labels", "view errors", "error id icon", // metadata fields - tested in metadata tests diff --git a/processor/stream/package_tests/metadata_attrs_test.go b/processor/stream/package_tests/metadata_attrs_test.go index 27cd197655..0a8961058c 100644 --- a/processor/stream/package_tests/metadata_attrs_test.go +++ b/processor/stream/package_tests/metadata_attrs_test.go @@ -134,7 +134,7 @@ func TestMetadataPayloadMatchJsonSchema(t *testing.T) { func TestKeywordLimitationOnMetadataAttrs(t *testing.T) { metadataProcSetup().KeywordLimitation( t, - tests.NewSet("processor.event", "processor.name", "listening", + tests.NewSet("processor.event", "processor.name", "listening", "labels", "url.scheme", tests.Group("context.request"), tests.Group("context.tags"), tests.Group("transaction"), @@ -142,13 +142,13 @@ func TestKeywordLimitationOnMetadataAttrs(t *testing.T) { tests.Group("trace"), // we don't support these yet - "kubernetes.labels", "kubernetes.annotations", - "kubernetes.container.name", "kubernetes.container.image", + "kubernetes.container.name", + "kubernetes.labels", + "docker.container.image", "docker.container.labels", "docker.container.name", - "docker.container.image", ), fieldMapping, ) diff --git a/processor/stream/package_tests/span_attrs_test.go b/processor/stream/package_tests/span_attrs_test.go index 0664b32c37..65f68acf0c 100644 --- a/processor/stream/package_tests/span_attrs_test.go +++ b/processor/stream/package_tests/span_attrs_test.go @@ -47,6 +47,7 @@ func spanPayloadAttrsNotInFields() *tests.Set { "context.http.url", "context.http.method", "context.tags.tag1", + "labels.tag1", ) } @@ -56,11 +57,16 @@ func spanFieldsNotInPayloadAttrs() *tests.Set { "listening", "view spans", "span.parent", // from v1 + // ECS field copies + tests.Group("container"), + + "url", + "url.port", + "url.scheme", ), // not valid for the span context transactionContext(), ) - } func spanPayloadAttrsNotInJsonSchema() *tests.Set { @@ -109,13 +115,14 @@ func transactionContext() *tests.Set { tests.Group("context.response"), tests.Group("context.request"), tests.Group("context.system"), + tests.Group("process"), ) } func spanKeywordExceptionKeys() *tests.Set { return tests.Union(tests.NewSet( - "processor.event", "processor.name", "listening", - "context.tags", + "processor.event", "processor.name", "listening", "url.scheme", + "context.tags", "labels", ), transactionContext(), ) diff --git a/processor/stream/package_tests/transaction_attrs_test.go b/processor/stream/package_tests/transaction_attrs_test.go index c9049d3895..aa3efeb415 100644 --- a/processor/stream/package_tests/transaction_attrs_test.go +++ b/processor/stream/package_tests/transaction_attrs_test.go @@ -62,8 +62,13 @@ func transactionFieldsNotInPayloadAttrs() *tests.Set { "context.user.user-agent", "context.user.ip", "context.system.ip", + tests.Group("context.db"), "context.http", + "context.http.url", + "context.http.method", "context.http.status_code", + "context.response.headers.user-agent", + tests.Group("container"), // we don't support these yet "kubernetes.labels", @@ -106,9 +111,11 @@ func transactionRequiredKeys() *tests.Set { func transactionKeywordExceptionKeys() *tests.Set { return tests.NewSet( - "processor.event", "processor.name", "listening", + "host.name", "processor.event", "processor.name", "listening", "transaction.marks", "context.tags", + "labels", + "url.scheme", // length always <= context.request.url.protocol // metadata fields - tested in metadata tests tests.Group("context.process"), diff --git a/processor/transaction/package_tests/TestProcessTransactionFull.approved.json b/processor/transaction/package_tests/TestProcessTransactionFull.approved.json index 02090cea98..a586d108ab 100644 --- a/processor/transaction/package_tests/TestProcessTransactionFull.approved.json +++ b/processor/transaction/package_tests/TestProcessTransactionFull.approved.json @@ -138,6 +138,10 @@ } }, "type": "request" + }, + "url": { + "port": 8080, + "scheme": "https" } }, { diff --git a/processor/transaction/package_tests/attrs_common.go b/processor/transaction/package_tests/attrs_common.go index 60f6ec251a..0e6c800fe8 100644 --- a/processor/transaction/package_tests/attrs_common.go +++ b/processor/transaction/package_tests/attrs_common.go @@ -45,8 +45,10 @@ func payloadAttrsNotInFields(s *tests.Set) *tests.Set { func fieldsNotInPayloadAttrs(s *tests.Set) *tests.Set { return tests.Union(s, tests.NewSet( - "listening", "view spans", "context.user.user-agent", + "listening", "view spans", "context.response.headers.user-agent", "context.user.user-agent", "context.user.ip", "context.system.ip", + tests.Group("container"), + tests.Group("timestamp"))) } @@ -100,9 +102,9 @@ func condRequiredKeys(c map[string]tests.Condition) map[string]tests.Condition { func keywordExceptionKeys(s *tests.Set) *tests.Set { return tests.Union(s, tests.NewSet( - "processor.event", "processor.name", "listening", "parent.id", "trace.id", - "transaction.id", "transaction.marks", "context.tags", "span.hex_id", - "span.subtype", "span.action")) + "process.args", "processor.event", "processor.name", "listening", "parent.id", "trace.id", + "transaction.id", "transaction.marks", "context.tags", "labels", "url.scheme", + "span.hex_id", "span.subtype", "span.action")) } func templateToSchemaMapping(mapping map[string]string) map[string]string { diff --git a/tests/fields.go b/tests/fields.go index 24f0988cf7..ca11aa809c 100644 --- a/tests/fields.go +++ b/tests/fields.go @@ -46,8 +46,11 @@ func (ps *ProcessorSetup) PayloadAttrsMatchFields(t *testing.T, payloadAttrsNotI //dynamically indexed: "context.tags.organization_uuid", "context.tags.span_tag", + "labels.organization_uuid", + "labels.span_tag", //known not-indexed fields: Group("context.custom"), + Group("context.db"), Group("context.request.headers"), Group("context.request.cookies"), Group("context.request.socket"), @@ -65,7 +68,8 @@ func (ps *ProcessorSetup) PayloadAttrsMatchFields(t *testing.T, payloadAttrsNotI } func (ps *ProcessorSetup) EventFieldsInTemplateFields(t *testing.T, eventFields, allowedNotInFields *Set, fieldMapping map[string]string) { - allFieldNames, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, isEnabled) + allFieldNames, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, isEnabled, isNotAlias) + require.NoError(t, err) t.Log("Old Field names: ", allFieldNames.Array()) @@ -85,7 +89,7 @@ func (ps *ProcessorSetup) EventFieldsInTemplateFields(t *testing.T, eventFields, } func (ps *ProcessorSetup) TemplateFieldsInEventFields(t *testing.T, eventFields, allowedNotInEvent *Set) { - allFieldNames, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, isEnabled) + allFieldNames, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, isEnabled, isNotAlias) require.NoError(t, err) missing := Difference(allFieldNames, eventFields) @@ -220,3 +224,23 @@ func isEnabled(f common.Field) bool { func isDisabled(f common.Field) bool { return f.Enabled != nil && !*f.Enabled } + +func isIndexed(f common.Field) bool { + return f.Index == nil || *f.Index +} + +func isNotAlias(f common.Field) bool { + if f.Type == "alias" { + return false + } + + if f.Type == "group" { + onlyAliases := true + for _, child := range f.Fields { + onlyAliases = onlyAliases && !isNotAlias(child) + } + return !onlyAliases + } + + return true +} diff --git a/tests/fields_test.go b/tests/fields_test.go index 4132bea676..1fb32a53bf 100644 --- a/tests/fields_test.go +++ b/tests/fields_test.go @@ -101,3 +101,34 @@ func TestFlattenFieldNames(t *testing.T) { flattenFieldNames(fields, "", disabledFields, hasName, isDisabled) assert.Equal(t, expectDisabled, disabledFields) } + +func TestNotAlias(t *testing.T) { + fields := common.Fields{ + {Name: "mixed", Type: "group", Fields: common.Fields{ + {Name: "a_keyword", Type: "keyword"}, + {Name: "an_alias", Type: "alias"}, + }}, + {Name: "just_aliases", Type: "group", Fields: common.Fields{ + {Name: "an_alias", Type: "alias"}, + {Name: "another_alias", Type: "alias"}, + {Name: "more_just_aliases", Type: "group", Fields: common.Fields{ + {Name: "one_more_alias", Type: "alias"}, + }}, + }}, + {Name: "also", Type: "group", Fields: common.Fields{ + {Name: "an_alias", Type: "alias"}, + {Name: "another_alias", Type: "alias"}, + {Name: "mixed", Type: "group", Fields: common.Fields{ + {Name: "one_more_alias", Type: "alias"}, + {Name: "not_an_alias", Type: "Keyword"}, + }}, + }}, + {Name: "top_level_alias", Type: "alias"}, + {Name: "top_level_keyword", Type: "keyword"}, + } + expected := NewSet( + "mixed", "mixed.a_keyword", "also", "also.mixed", "also.mixed.not_an_alias", "top_level_keyword") + flat := NewSet() + flattenFieldNames(fields, "", flat, isNotAlias) + assert.Equal(t, expected, flat) +} diff --git a/tests/json_schema.go b/tests/json_schema.go index 1b39b56e2a..0f6da26e4c 100644 --- a/tests/json_schema.go +++ b/tests/json_schema.go @@ -185,7 +185,7 @@ func (ps *ProcessorSetup) AttrsPresence(t *testing.T, requiredKeys *Set, condReq func (ps *ProcessorSetup) KeywordLimitation(t *testing.T, keywordExceptionKeys *Set, fieldMapping map[string]string) { // fetch keyword restricted field names from ES template - keywordFields, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, + keywordFields, err := fetchFlattenedFieldNames(ps.TemplatePaths, hasName, isIndexed, func(f common.Field) bool { return f.Type == "keyword" }) require.NoError(t, err) diff --git a/tests/system/apmserver.py b/tests/system/apmserver.py index 1033215193..4901ecb6f0 100644 --- a/tests/system/apmserver.py +++ b/tests/system/apmserver.py @@ -423,3 +423,19 @@ def config(self): def get_debug_vars(self): return requests.get(self.expvar_url) + + +class SubCommandTest(ServerSetUpBaseTest): + def wait_until_started(self): + self.apmserver_proc.check_wait() + + # command and go test output is combined in log, pull out the command output + log = self.get_log() + pos = -1 + for _ in range(2): + # export always uses \n, not os.linesep + pos = log[:pos].rfind("\n") + self.command_output = log[:pos] + for trimmed in log[pos:].strip().splitlines(): + # ensure only skipping expected lines + assert trimmed.split(None, 1)[0] in ("PASS", "coverage:"), trimmed diff --git a/tests/system/test_ecs_mappings.py b/tests/system/test_ecs_mappings.py new file mode 100644 index 0000000000..30fa92fde9 --- /dev/null +++ b/tests/system/test_ecs_mappings.py @@ -0,0 +1,134 @@ +import json + +import yaml + +from apmserver import SubCommandTest + + +def flatmap(fields, pfx=None): + if pfx is None: + pfx = [] + for field, attribs in sorted(fields.items()): + if 'properties' in attribs: + for f in flatmap(attribs['properties'], pfx + [field]): + yield f + else: + yield ".".join(pfx + [field]), attribs + + +class ECSTest(SubCommandTest): + """ + Test export template subcommand. + """ + + def start_args(self): + return { + "extra_args": ["export", "template"], + "logging_args": None, + } + + def test_ecs_migration(self): + """ + Test that all fields are aliased or otherwise accounted for in ECS migration. + """ + all_fields = set() + alias_source_fields = set() + alias_target_fields = set() + exception_fields = set() + for f, a in flatmap(yaml.load(self.command_output)["mappings"]["doc"]["properties"]): + if a.get("type") == "object" and not a.get("enabled", True): + exception_fields.add(f) + if not a.get("index", True): + exception_fields.add(f) + all_fields.add(f) + if a.get("type") == "alias": + alias_source_fields.add(f) + alias_target_fields.add(a["path"]) + + # fields with special exception, due to mapping type changes, etc + # no comment means unchanged + exception_fields.update({ + "@timestamp", + "container.labels", # target for docker.container.labels copy + "context.http.status_code", # staying put, like other context.http.* (and context.db.*) + "context.request.url.port", # field copy to url.port, keyword -> int + "context.request.url.protocol", # field copy to url.scheme, drop trailing ":" + "context.tags", # field copy, can't alias objects + "docker.container.labels", # field copy, can't alias objects + "error.code", "error.culprit", "error.exception.code", "error.exception.handled", "error.exception.message", + "error.exception.module", "error.exception.type", "error.grouping_key", "error.id", "error.log.level", + "error.log.logger_name", "error.log.message", "error.log.param_message", "error.message", "error.type", + "fields", + "labels", # target for context.tags copy + "kubernetes.annotations", "kubernetes.container.image", "kubernetes.container.name", "kubernetes.labels", + "kubernetes.namespace", "kubernetes.node.name", "kubernetes.pod.name", "kubernetes.pod.uid", + "parent.id", + "processor.event", "processor.name", + "sourcemap.bundle_filepath", "sourcemap.service.name", "sourcemap.service.version", + "span.duration.us", "span.hex_id", "span.id", "span.name", "span.parent", "span.start.us", "span.sync", + "span.action", "span.subtype", "span.type", + "system.cpu.total.norm.pct", "system.memory.actual.free", "system.memory.total", + "system.process.cpu.total.norm.pct", "system.process.memory.rss.bytes", "system.process.memory.size", + "tags", + "timestamp.us", + "trace.id", + "transaction.duration.us", "transaction.id", "transaction.marks.navigationTiming", "transaction.name", + "transaction.result", "transaction.sampled", "transaction.span_count.dropped.total", "transaction.type", + "url.port", # field copy from context.request.url.port + "url.scheme", # field copy from context.request.url.protocol + + # scripted fields + "error id icon", + "view errors", + "view spans", + + # host processor fields, already ECS compliant + "host.id", "host.mac", "host.name", "host.os.family", "host.os.version" + }) + + should_not_be_aliased = alias_target_fields - all_fields + self.assertFalse(should_not_be_aliased, json.dumps(sorted(should_not_be_aliased))) + + # check that all fields are accounted for + not_aliased = all_fields - alias_target_fields - alias_source_fields - exception_fields + fmt = "\nall fields ({:d}):\n{}\n\naliased ({:d}):\n{}\n\naliases ({:d}):\n{}\n\nunaccounted for ({:d}):\n{}" + self.assertFalse(not_aliased, + fmt.format( + len(all_fields), json.dumps(sorted(all_fields)), + len(alias_target_fields), json.dumps(sorted(alias_target_fields)), + len(alias_source_fields), json.dumps(sorted(alias_source_fields)), + len(not_aliased), json.dumps(sorted(not_aliased)), + )) + + def test_ecs_migration_log(self): + aliases = {} + all_fields = set() + not_indexed = set() + for f, a in flatmap(yaml.load(self.command_output)["mappings"]["doc"]["properties"]): + self.assertNotIn(f, all_fields) + self.assertNotIn(f, aliases) + all_fields.add(f) + if a.get("type") == "alias": + aliases[f] = a["path"] + if not a.get("index", True): + not_indexed.add(f) + + aliases_logged = {} + for migration_log in self._beat_path_join("_meta", "ecs-migration.yml"), \ + self._beat_path_join("_beats", "dev-tools", "ecs-migration.yml"): + with open(migration_log) as f: + for m in yaml.load(f): + if m.get("index", "apm-server") != "apm-server": + continue + if m.get("alias", True): + self.assertNotIn(m["to"], aliases_logged, "duplicate log entry for: {}".format(m["to"])) + aliases_logged[m["to"]] = m["from"] + elif m.get("copy_to", False): + if m["from"] not in not_indexed and m["to"] not in not_indexed: + self.assertIn(m["from"], all_fields) + else: + self.fail(m) + self.assertIn(m["to"], all_fields) + + # false if ecs-migration.yml log does not match fields.yml + self.assertEqual(aliases, aliases_logged) diff --git a/tests/system/test_export.py b/tests/system/test_export.py index 3ab30b719b..f1b51f960a 100644 --- a/tests/system/test_export.py +++ b/tests/system/test_export.py @@ -1,23 +1,7 @@ import json import yaml -from apmserver import ServerSetUpBaseTest - - -class SubCommandTest(ServerSetUpBaseTest): - def wait_until_started(self): - self.apmserver_proc.check_wait() - - # command and go test output is combined in log, pull out the command output - log = self.get_log() - pos = -1 - for _ in range(2): - # export always uses \n, not os.linesep - pos = log[:pos].rfind("\n") - self.command_output = log[:pos] - for trimmed in log[pos:].strip().splitlines(): - # ensure only skipping expected lines - assert trimmed.split(None, 1)[0] in ("PASS", "coverage:"), trimmed +from apmserver import ServerSetUpBaseTest, SubCommandTest class ExportConfigDefaultTest(SubCommandTest): diff --git a/utility/map_str_enhancer.go b/utility/map_str_enhancer.go index e1045d4154..bc8c19e9b7 100644 --- a/utility/map_str_enhancer.go +++ b/utility/map_str_enhancer.go @@ -136,7 +136,7 @@ func Add(m common.MapStr, key string, val interface{}) { } } -// MergeAdd modifies `m` *in place*, inserting `valu` at the given `key`. +// MergeAdd modifies `m` *in place*, inserting `val` at the given `key`. // If `key` doesn't exist in m(at the top level), it gets created. // If the value under `key` is not a map, MergeAdd does nothing. func MergeAdd(m common.MapStr, key string, val common.MapStr) { diff --git a/vendor/github.com/elastic/beats/libbeat/kibana/fields_transformer.go b/vendor/github.com/elastic/beats/libbeat/kibana/fields_transformer.go index 8f4f7005a3..e5320fb3e0 100644 --- a/vendor/github.com/elastic/beats/libbeat/kibana/fields_transformer.go +++ b/vendor/github.com/elastic/beats/libbeat/kibana/fields_transformer.go @@ -41,7 +41,7 @@ func newFieldsTransformer(version *common.Version, fields common.Fields) (*field version: version, transformedFields: []common.MapStr{}, transformedFieldFormatMap: common.MapStr{}, - keys: common.MapStr{}, + keys: common.MapStr{}, }, nil } @@ -137,6 +137,10 @@ func transformField(version *common.Version, f common.Field) (common.MapStr, com field["searchable"] = false } + if f.Type == "object" && f.Enabled != nil { + field["enabled"] = getVal(f.Enabled, true) + } + if f.Type == "text" { field["aggregatable"] = false }