From 4fa0510fba81faae713bd8a3e5d15484ae7fe8ba Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 25 Feb 2020 13:44:22 -0700 Subject: [PATCH 1/6] Add cloudwatch fileset in aws module --- filebeat/docs/fields.asciidoc | 27 +++++++ .../module/aws/cloudwatch/_meta/fields.yml | 15 ++++ .../aws/cloudwatch/config/cloudwatch.yml | 22 ++++++ .../module/aws/cloudwatch/config/file.yml | 6 ++ .../module/aws/cloudwatch/ingest/pipeline.yml | 26 +++++++ .../module/aws/cloudwatch/manifest.yml | 13 ++++ .../aws/cloudwatch/test/cloudwatch_ec2.log | 6 ++ .../test/cloudwatch_ec2.log-expected.json | 74 +++++++++++++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- 9 files changed, 190 insertions(+), 1 deletion(-) create mode 100644 x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch/config/file.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch/manifest.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log create mode 100644 x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index e37b70dc0dbf..4e34fa84cb65 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1309,6 +1309,33 @@ type: keyword -- Identifies the VPC endpoint in which requests were made from a VPC to another AWS service, such as Amazon S3. +type: keyword + +-- + +[float] +=== cloudwatch + +Fields for AWS CloudWatch logs. + + + +*`aws.cloudwatch.ip`*:: ++ +-- +The internet address of the requester. + + +type: ip + +-- + +*`aws.cloudwatch.program_name`*:: ++ +-- +The internet address of the requester. + + type: keyword -- diff --git a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml new file mode 100644 index 000000000000..bb1370e29e84 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml @@ -0,0 +1,15 @@ +- name: cloudwatch + type: group + release: beta + default_field: false + description: > + Fields for AWS CloudWatch logs. + fields: + - name: ip + type: ip + description: > + The internet address of the requester. + - name: program_name + type: keyword + description: > + The program name of the log entry. diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml b/x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml new file mode 100644 index 000000000000..2af7bebff30c --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/config/cloudwatch.yml @@ -0,0 +1,22 @@ +type: s3 +queue_url: {{ .queue_url }} + +{{ if .credential_profile_name }} +credential_profile_name: {{ .credential_profile_name }} +{{ end }} + +{{ if .shared_credential_file }} +shared_credential_file: {{ .shared_credential_file }} +{{ end }} + +{{ if .visibility_timeout }} +visibility_timeout: {{ .visibility_timeout }} +{{ end }} + +{{ if .api_timeout }} +api_timeout: {{ .api_timeout }} +{{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/cloudwatch/config/file.yml b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml new file mode 100644 index 000000000000..8bfbcc9f802b --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/config/file.yml @@ -0,0 +1,6 @@ +type: log +paths: + {{ range $i, $path := .paths }} + - {{$path}} + {{ end }} +exclude_files: [".gz$"] diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml new file mode 100644 index 000000000000..4ee5b892f338 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -0,0 +1,26 @@ +description: "Pipeline for s3 server access logs" + +processors: + - grok: + field: message + patterns: + - >- + %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.cloudwatch.ip} %{NOTSPACE:aws.cloudwatch.program_name}: %{GREEDYDATA:message} + - >- + %{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message} + + - date: + field: "_tmp_.timestamp" + target_field: "@timestamp" + ignore_failure: true + formats: + - "dd/MMM/yyyy:H:m:s Z" + - remove: + field: + - _tmp + ignore_missing: true + +on_failure: + - set: + field: "error.message" + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml new file mode 100644 index 000000000000..ce14df8f1711 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -0,0 +1,13 @@ +module_version: 1.0 + +var: + - name: input + default: s3 + - name: shared_credential_file + - name: credential_profile_name + - name: visibility_timeout + - name: api_timeout + - name: endpoint + +ingest_pipeline: ingest/pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log new file mode 100644 index 000000000000..4487fdf08d2e --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log @@ -0,0 +1,6 @@ +2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root. +2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s diff --git a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json new file mode 100644 index 000000000000..fdf6779614e4 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-02-25T20:41:45.297Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "systemd", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 0, + "message": "Stopping User Slice of root.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-25T20:41:45.297Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "dhclient[3000]", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 96, + "message": "XMT: Solicit on eth0, interval 125240ms.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-25T20:41:45.304Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 211, + "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-25T20:41:45.304Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 345, + "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-25T20:41:45.312Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 461, + "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-25T20:41:45.312Z", + "aws.cloudwatch.ip": "ip-172-31-81-156", + "aws.cloudwatch.program_name": "ec2net", + "event.dataset": "aws.cloudwatch", + "event.module": "aws", + "fileset.name": "cloudwatch", + "input.type": "log", + "log.offset": 586, + "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "service.type": "aws" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index e976c49dffa6..e77180088a58 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMW1uT2zayfvev6PJLxlWSTsVJnTo1p7JV8ni80WZiz47kZPeJhoiWiBUEMAAoWf71Ww2AF4mk5iIpu3qwNSLZ/XWj7wCHsMLdNbCtfQXghJN4DePfp68ADEpkFq9hjo69AuBoUyNyJ7S6hr+8AgD4VfNCIiy0gYwpLoVagtRLCwuj10Rm9ApgIVBye+0fGIJiayzZ0cftcryGpdFFHn/p4EOfD55MRdnzGcWrTRZNNqnUBXeGCVld6uJIn0Npyw/HBSukSzyLa1gwaXHvcifYJmBtPN4bwjIjLHvQu+A3RcANKpds0Fih1d4dpSQr3G214QfXjgCjzyzDJqJIH/QCXIYEMDAm9GvmRp3QCosmERyVE27XCe1QyW1gw05kRHkSCQNKXBOUVCvHhLLA0TEhLbC5LpzHS9xAL1q0JuNfoQQILmMO1oyjf8TgHwVaNwCmOGwzkWaQGvT3MmlhiwZb5AqLfASTBThc59ows2s94+8ZeA4lbpvprYVMb+nXFs0WAT0nKZGPDm7tMpLmapAOWheP20h7OTpuCCsSNewF61nyhnebQ0t9PpK2YZRQxmv2TSt4QKsLkyJ8ZGuEq/HDxzclwNwIlYqcyYM1T5mUh2ptoE5TtDZZ4S4RXfjOhT/wIUIweR8Qbpn1hgNOgxVL1bTQfsAWLTltQo6BX10v5C4vfCrgyaKJxQP16twKlzXcwGJamC6TgH0TJ3erHMOLnhu9ERwtCBViDYWh2rOjjJ10K9WlBplD7kOty7TFJsuOR/tcqanc9YIlrHAZUUmJeufdj1vFUxUN0To2TBYIwoIz9H9Uv9bOB0XQxgc1/31LovYS64xMUUX1gjJptdfhnqxheVm32unz64cxcNyIFP8ftMvQbIXFQciObYNt6tWvFVktZ64PfNDpkRueo1Ai44O8E2uEbYbBu9q229aYsLZoB+JaFqE2eoU8mXfZ/bnCBbEqAxvVERYN6b0vndkizYB1WT2UkfP25i2MC6dhmjJfssUK5VYy60QK75Ap65hcdad9NEabJNX8cHWeXo50Z/2mdJ5JZf7RWA26wijrYwhdP4Zvjday5TkhTo6DCUm/QaRctCNQI60kZ4at0aE5XLdTVVoTHpAymdoNYtgk+7YUhUMk78s3NVKba2UxicH73EBL+lVyoEDFUnrGlqa+QkgzppZo4SqE/EG7RMvJ3wdk0hwlkusHIm+65WKcC7rOZOILXs72qv9TRRtX5CkOsUZiC+V1lcKUdrRYLhpNi1BpddpUqjpuUq0S5tQVConJAw4F4EKg3TMcf1vpwHMUatmudpmUyGGJCg1z/nlhA+keZ/btT0dte5In7+Mva9vGkpQAeWOtDKba8B4zysXJPdqjOMf3k6pRY9bqVNS52l/f2nEubpiULUpeghnJeUTXa6bY0ntf8IVz+gG801oiUz1mtM2QqoiGtoWFQ0eEBsJwV58LMJ5oJbt70pOXosYqLOic7IRWhAB71kNiXV/ojae+e+mOoy/pm8cghfXxo6IdWw3kIFSt2uc2lpfr5qoWbvzwsV2xNPoyXaiOiHYuGOPAgLoxXeb4CExvFZrLttyPqMYHp8pTTLmSoVfyo7rra7a1wxh3hx7ZNSWbIT3q/+6xwFTkgpy9V8GnOMwD5gapwAixi9U69r5vMEWx8fFV2GPOHOUKESmJo5/Lxdiq/iR2AxAqlQWnGnlLqJ0RyyWakBa6g2xoNYINFbJPqIwZ5FGms6r9r58n7xvZa75rTvmchkKJPwqUu9Kkmte7BYojV68cakWoSg81VYziNtSRTgMXiwUa+iNMkPc/0QRst0o2eZqg4rkW51bJwQr/dn8DJSPypjD7izVMbPx8W+XFbucget5pYMr3vc2mpWq+ykZr+kMta1XQyPmrQ7l6Z9HL/Un0U0fNt3fvnjVjbsWIk0fLzaZVasZhziRTKfb0QSfVd50AmvPKPQBeSZu3cEc/vos/9tikY2aJLvHLM2qnwRMhjh8+lggDo2AH9U7Ko00ZJXxsJ6gTcXnriZRbwRoh1Uph2l/Z5EY7nerDaHIiqJJq95peZc7l1Bu5NO9p9KpW22iqiYRaJk6scWQx7US6kJodVsBPsTvtmAyjJqHAYqoVt2CFSvFAe6GVC12dsLWOC+WEBLEXWKmwXNKSUOqZs3SFqqcRiRf/i8RsiEFXIkBwQsq9H6xjxtnY3VLKfWQQ8R+WsBpYNNeuypR7Uu6vpVePFL0FT626INe6u9qRutVeP0+q6EjNlVKwFlKKKOwgShvg6xwV7gmUSm0Pp6NV5JQ2oThmM7bCy8pRTrRnd1OoWJKiU73O/QToQC7QHVaaMQtzRAVoHZtLYbM+0Ur3E4cN2okRbnIPjHOD1pZLUxpRbemhWn4sApcIc21e3sN3R2FtXNmBnIqOQvaommVZx1xhT5sqd0IOhIEIt93zqnm17aDhAcqH5L9Om9BJf+mF/qW72rcySUWenTtRT6d3EOiGvR2hyAn+x/9cLUJPaUOYLpOpiX2VrZ+NK820RZWkaNxFK67AB4iPWPhNL4j9atiFbRjBS+FbNIKdWbmBJqhiPUdzYVmESvXap1dpEybx3MFEKIdLNHEiqBc+gns+jYw633WV8KUfB6n81kaUuD8d0TeedE3WTpSDYHvioX7vQWBYiqf0tZ2s/c4/tejREr78Yzhef1PDGXEbTvgXyJDxvq5rzVyaIU9MIamcErpjB//ksj1QrRfZ106FrPYvPYT9LUyxoIKX7omXe2buoUjCr5gW7f35E4HHCUdJPOwb76W6eifwaqHNlhk+gIX4inxYZobB3qb6aDR6M4KJg5Sp8iwTWNygYTKop8cPDXJhMHVJYc4cTT4/3MUI7TUe+fjthfDfltlKBUc2W0cGmX3xzkcntLD/G+iW28DVYkR0CyZkE1eV234Ic++Lz1mmP/jRD5ryTM9zhi7zIl2hS7omzKdGBaa0EimT4ZhIPdb2vMo/4nw5wOgp1fy1y02GAv0QCQ5PGUVfo9VeMqH696XX2mHSU4y3fn6K5+c5MyFXOjQK3WFZHlH2RdXq8p+0qBU/v+nNYOjNs1D7B3rK+ebFNo1boMdgnaFQuTeNriaj3cNoZPUsthtqta92XvXW+3h+9MUhCweAgGMqGRX+zML00/h+VN05gIfb6Wz082x2n6zRZZqPyn0jv2E9gN9v300ns9tjt2gD78azm59H72/vbme3o0/v/nZ7M+sWfYVnzs6vV7h73Tx2UOdgSg2oqKfhHuTr4esyDNeq4hrDyQVHLTfzRxqrEyfHLa0w4ryyPATCw88Pkz2JSPdVYImHgrqhUVeXhGbujJMKVazRiDTgaLab9VbjkdMcZzhr1d0JVW546xPtjebYXGelYwbWaVoY0zsL2Tm0ie07K/BijcXmppqxeT6+QR8Afi235LxK68HwBg1Vu00xvqHRPWFk/i8qqaz41q3ZU8ZSRLRKuZ4P9V3eAnunIv5JPyi7iCb35l+HqVYoWEixzFxjp8+XNd9ZyNHYnIrCTY+FusKohBldKP6nwWeuYcA2p2TdKM13ujCPHWZboDHnztB7LY63zYfIJ3Zhj/ep/o0Ktjzl7M0ToX22aIZj4nS0RYzHjc7euJbHmCbvy3FilXmemmwiiQl/LOVk+syVDQnwdcjW34aCD9/607OVNeJXh4rXBRdM3vcM4MRSMVcYvMy7PRX5Uk8DmIrlbx4tfflx0H7zoFkx7gWJF9eVYTqZ2EK0jnKfOmXElISbauogLNyxHRq4mk7v3pQj0fpoJS61E9ULAmT+0y7R6ELPpKEW2W/LXGy3ujqVv8cwvlM0Llz2s/fVcDh2/57gxXYAfy/Q7Kah9Kb7/qC/y1r8Kjc4JNtATiXem5cvrfeqwPS8uqjOZpRmGQeV9LXjXAUcjPku4k0zw5T1mx/B0KblKwNXs7vpmyqaNSwtzi0PN/oap10WUm+fPqFovZX41BnFb/c3QKyeNZu4iBIJyQdCcqeXtmThX6fb6YJWOx6g9+9bxgNu4VB9qV9h4W31AJUl1DIySAvr9LrviR5bOcOht+7K2h+Pqg67lbuT5RL0jdodmsUlJsT1nECh22qzqnl5bOH0kz+DYthiIdK4n60NPz53vci4tTxQ1nW6OeIbwPjm5vZ+RpHr4ba/WZZ6eayZezFSqZdLiqSxlYvKLZd3AJ9+GcDHT+/Hs7FPtb9M7ul737Jbx9RFV71k4VX7XVuzL7CKQVmbVbSF9aNFH/V2uug5F7RyiTUp47w7YbxkVpczSv9DiRuUcKWNWArF5JtyttneUo/i9CPk1v0pCDk1gyqk7gbMMlwcxbnJ0wtajD/fSH5YvW991uhhi7nC84fdGn9gcEkRXJonC8mWZ44sc+HWzK5is1YlDi2l3lLEmd3cg2d7DW9/mv7z4+D7/6P/huObXwbf//Rh8nHw408P01k35MsdsAxau4bJ/ebHAf37v76Hu/0wHr36dwAAAP//TiBmYw==" + return "eJzMW1uT2zayfvev6PJLxlWSTsVJnTo1p3Kq5PH4RJuJPTuSk90nGgJbInYggAHAkeVfv9UAeBNJzUVSsnqwNSLZ+PqCvqE5hnvcXQLb2lcATjiJlzD9ff4KwKBEZvESlujYK4AULTcid0KrS/i/VwAAv+q0kAgrbSBjKpVCrUHqtYWV0RsiM3kFsBIoU3vpHxiDYhssl6OP2+V4CWujizz+0rMOfT54MhVlv84kXm0u0VyGS12kzjAhq0t9K9Jnn9vyk+KKFdIlfolLWDFpsXW5F2wTsDYe7xVhWRCWFvQ++E0W8AGVSx7QWKFV646Sk3vcbbVJ964dAEafRYZNRJE+6BW4DAlgWJjQb5ib9EIrLJpEpKiccLteaPtC7gIb9yIjyrNIGFDihqBwrRwTykKKjglpgS114TxeWg30qkNrNv0VSoDgMuZgw1L0jxj8o0DrRsBUCttM8Ay4QX8vkxa2aLBDrrCYTmC2AoebXBtmdp1n/D0jv0KJ22Z6ayHTW/q1Q7NDQC+JS0wne7f2GUlTGySDzsXDNtJVR88NQSNRwp6xAZU3drfZt9TnI+kaRgllumHftII7tLowHOEj2yBcTO8+vikB5kYoLnIm93TOmZT7Ym2g5hytTe5xl4g+fKfCH9YhQjB7HxBumfWGA06DFWvVtNBhwBYtbdqENgZ+dYOQ+3bhUwHPVk0sHqgX51a4rLENLPLC9JkEtE2ctlu1MTzrudEPIkULQgVfQ26o3tmRx166lei4QeYw9a7WZdpic8meR4e2UlO4mxVLWOEyosKJeu/dj1vFUwUN0ToemCwQhAVn6P8ofq2dd4qgjXdq/vuWWB0k1uuZoohqhTJptZdhi9egXtYvdvr8+mEKKT4Ijv8L2mVotsLiKETHrsE25ep1RVabMjcEPsj0wA3PESiR8U7eiQ3CNsOwu7q225WYsLboOuKaF6Ee9D2mybLP7k/lLmip0rFRHmHRkNyHwpkteAasz+qh9JzXV29hWjgNc858yhYzlGvJrBMc3iFT1jF53x/20RhtEq7Tfe08PR3pj/pN7vwilflHYzXoCqOs9yF0/RC+DVrL1qeEODsMJgT9BpFSaQegRlpJzgzboEOzr7djRVoTHpEwmdqNotsk+7bkhYMnH4o3NVKba2Uxic771EBL+lVwIEfFOD1jS1O/R+AZU2u0cBFc/qibouW030dk0ilKpK0fiLzp54ulqaDrTCY+4U1ZK/s/lrVpRZ78EGsEtpBeVyFMaUfKctFoOoRKq9OmEtVhk+qkMMdqKAQmDzgkgCuBtmU4/rZyAy9RqHU322VSYgprVGiY888LG0gPbGZf/vTktkft5Db+MrdtqKQEmDZ0ZZBrkw6YUS6OrtEexTm9nVWFGrNWc1HHan99a6e5uGJSdih5DhbE5wFZb5hia7/7wl445T6Ad1pLZGrAjLYZUhbRkLawsL8RoYEw3DW0BViaaCX7a9KjVVFjFRZ0TnZCGiHAfukxLV1fGPSnvnrp96MvqZunIIX1/qOiHUsNTEGoWrTPLSzPV81VJdz07mM3Y2nUZbpQPR7tVDCmYQGqxnQZ4yMwvVVozltyPyIa75yqnWJKTYZaybfqLi/Z1o6j3x17ZJcUbMb0qP97wAK5yAVt9kEBH7Nh7jA3SAlG8F2slrHf+wY5igfvX4U9tJkjX8EjJbH1cz4fW+WftNwIhOKySClH3hJqZ8R6jSaEhX4nG0qNYEOFHGIqYwbTyNNJxf7/n2fvG9FruWt2+ZyGQok/CpS70qSa1/sZii1XLxwqRShLDzlV9OI25JFOQypWKzT0R+ggtz/RBGy/SB5ynqBKcy1OLZI9Df92ewXlQrSbQu8v5jCx8PNllWe7G4PoeaeBKV/3NouWqvgqC635DzWvrZb0ljmevdpn769pSf9OWJ7Vkhb7oSnA7/z8hBa0UA6NQkd5uEFrawfs1YEDdVNu9NqwTdJxbkf3xCPlVtXtu+HKmV1XmyiXT1fjmr1IS9c3756lntMLpS0MlsKSSab4kHaOytZ7ATS7zy0AXkgPb+GGfnwXfxzwMI6ZNbrEq2fSTWqOhDi9+1giDAsFO6jPxR4tsSl9w266cSQubz2Rcif0InCtFPLhPDU32mmu92PD8bvMU+3X6UXmXE6VruP5QNleNU6MpgxXqHXixAYnFnkv0pXUbL+eeYrdacdkaBwKBRa5VqkFKxTHPemFwjzU6MLWMi6UExJEK0xSmbAmlVAisWT8HtVAWRkv/gex2WCDrkSA4ISUrR+sY8bZ2KugBOqRttJfzGHVfmrqrsp7Wly2denFI8Vg+lqLLvC16c9dpe40S57HVdxITU0p2AgpRWR2FLkN8HWOClsMcantfq+78pzSJuTHbMbu8bx8lOcTi5s5VEuSoLne5L6ft8cX6B4rzZiFJaICtI4tpbDZEGvl9hvIaV7s4Wa3+ylNaUS1pYfa5zEPXCLMtXl5R6bfC2vjynryWHTksidVZ9I65gp73BlBL+RAGIhwd3teNK92N2h4gOIh7V+nTeiLfBmE/qW/drMy4SLPTh2o5/MbCHTDSZ1QtAn+y/9cKWEgtSFM54nUtHwVrZ+Ni2faoko4GnfWjCusA7SOWPkjTIjdh3Cm3jCCl8K3aAQ7sXADTVDFZonmzLwIxfXGh1dpEybx1M6EKrk1mtjf1Svvwf06jYi63PWl8OU+Dlz5g6rI8XA4om9p0tcnPZIPgu2Jh/x9AIFhHI/pUvQu7ec4lKsixpd/jKebb2q8oNXGs/QLZMjSoaprQ3U8pokpJKVTQvfMYxydtgeqtZJ97lTI6jTaQ2gfSIsVJbx0T7w8cIISkiT8irzoTlscCTz2q0riYQqgFerqc92LlTZbZtIRrMRXTMdlZBi1RiQmk8mbCcwccKbKyTSw+ICGySCegX1oMBUGuUsKc2Jv8vnuJnpoL/G4jj8sCv9tma1EcODofGKQ2RefY/VCC6f5gW55qF8pI6JbMSGbuKrY9kM4xTh7n2X+g2/koSkntJ7TdFkW/B5d0ndecKxXYEorwZkMQz/1IYVfq/wjnhYEGAOpmr92vs5QoB88wf7MWNxrpO01E2p4ymCjHSYnbDCyPGcmxMoXdRqry3+SUqv1/AgDg7E3z0K1x7PKbvXZRgA6oKdgnSFX2TpbqPrc/UcLyOrOej/U6pT0tOKtT2V96yuFLIxzQYpcMkr8mYX5p+ntpLpzBHfX88Xk58XiNtmgy3Q6KU8B/fjBCH6/fjefLa4P3aINvJsurn6evL++uV5cTz69+9v11aKf9Xs8cXR+fY+7180hkjoGU2hARTVN6kG+Hr8u3XAtqlRjmENxVHIzP6BazQ8dtrTCiNPychcIjz/fzVockewrxxJHvPqhUVWXhGLuhJ0KVWzQCB5wNMvN+uD4wGzOCSbn+iuhahte+0B7pVNs6lnpGIE154Uxg72QnUOb2KHJjxdLLBY3VY/Nr+ML9BHg1/KA1Yu0bgw/oKFst8nGNzR6wI0s/0UplRXf+iV7TFuKiFYh169DdZe3wMGuiH/SN8rOIslW/2s/1AoFKynWmWuc2/q05jsLORqbU1L4MGChrjAqYUYXKv3T4DPXMGCbU7BupOY7XZjHRhNXaMypI3SrxPG2eRfXiVXY43Wqfz+GrY+ZpHoitM8WzXhKKx0sEePw2MkL13Iobfa+bCdWkeepwSaSmKWPhZxMnzizIQa+jtnm21ik47d+FrqyRvzqUKV1wgWz9wMNOLFWzBUGz/OmVkW+lNMI5mL9m0dLX34cdd8jaWaMLSfx4rwydCcTW4jOYP6xXUbkxNxcUwVh4Ybt0MDFfH7zpmyJ1oOyuNZOVK97kPnP+1ijCwOdhpplfyxzttPq6h2L1oLxDbFp4bKf/V4No87te8IutiP4e4FmNw+pN933B/1d5uIXucEx2QamlOK9eblq/a4Ki55WFtWkTWmWsVFJX3umZGCvzXeW3bQwTFl/+BEMbV6+AHKxuJm/qbxZw9Ji33L/oK8xu7SSevv0DkVnoOepPYrfbq+AlnpWb+IsQiQkHwjJjV7bcgn/cuROF6Tt+DqEH1WK44rhFYlSvsLC2+oBSkuoZGTAC+v0ZuiJAVs5wQhjf2bth92q0cXydLJUwVCr3aFZnaNDXPcJFLqtNvf1Wh5bmGXzMyiGrVaCx/NsbdLDfdeztFvL8cC+WfWIbwTTq6vr2wV5rrvr4WJZ6vWhYu7FSKVer8mTxlIuCrdU7wg+/TKCj5/eTxdTH2p/md3S9yG1W8fUWbVeLuFF+11Xsi+wilGZm1W0hfWtRe/1droYmAu6d4k1nKVpf8B4Sa8uZxT+xxIfUMKFNmItFJNvyt5m90g9sjOMMLXuT0GYUjGoQuhuwCzdxUGcDzk/o8X4aVXah9Xb8yf1HrZYKjy9263xhwXOyYLjebKSbH1iz7IUbsPsfSzWqsChpdRb8jiLq1vwy17C25/m//w4+v5/6L/x9OqX0fc/fZh9HP3409180Q/5fAOWQWqXMLt9+HFE//63r+GuP0wnr/4dAAD//xEO3xg=" } From 9b51294e624d28fca9c8e36caf5ce5b4e7f515ed Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 25 Feb 2020 14:21:12 -0700 Subject: [PATCH 2/6] update doc and changelog --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/fields.asciidoc | 2 +- filebeat/docs/modules/aws.asciidoc | 72 ++++++++++--------- x-pack/filebeat/filebeat.reference.yml | 32 ++++++++- x-pack/filebeat/module/aws/_meta/config.yml | 32 ++++++++- .../filebeat/module/aws/_meta/docs.asciidoc | 72 ++++++++++--------- x-pack/filebeat/modules.d/aws.yml.disabled | 32 ++++++++- 7 files changed, 163 insertions(+), 80 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index df1ad669a883..81d93a52d814 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -138,6 +138,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500] - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529] +- Add cloudwatch fileset in aws module. {issue}13716[13716] {pull}16579[16579] *Heartbeat* diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 4e34fa84cb65..be7729c0a726 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1333,7 +1333,7 @@ type: ip *`aws.cloudwatch.program_name`*:: + -- -The internet address of the requester. +The program name of the log entry. type: keyword diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index e8e971b66901..d6c38382da75 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -31,7 +31,7 @@ Example config: [source,yaml] ---- - module: aws - s3access: + cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -40,50 +40,42 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com - elb: + cloudwatch: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws - - vpcflow: + elb: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws - - cloudtrail: + s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials + vpcflow: + enabled: false + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws ---- *`var.queue_url`*:: @@ -120,6 +112,16 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files that are delivered to the S3 bucket when Log File Integrity is turned on, it only reads the CloudTrail logs. +[float] +=== cloudwatch fileset + +Users can use Amazon CloudWatch Logs to monitor, store, and access log files +from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources. +Then export logs from log groups to Amazon S3 bucket which has SQS notification +setup already. Right now, this fileset only parses EC2 logs into fields like `ip` +and `program_name`. For logs from other services, this fileset will store them into +`message` field. + [float] === elb fileset diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 7bf16fa42823..8ba20091b057 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -96,7 +96,33 @@ filebeat.modules: #--------------------------------- AWS Module --------------------------------- - module: aws - s3access: + cloudtrail: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + cloudwatch: enabled: false # AWS SQS queue url @@ -148,7 +174,7 @@ filebeat.modules: # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - vpcflow: + s3access: enabled: false # AWS SQS queue url @@ -174,7 +200,7 @@ filebeat.modules: # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - cloudtrail: + vpcflow: enabled: false # AWS SQS queue url diff --git a/x-pack/filebeat/module/aws/_meta/config.yml b/x-pack/filebeat/module/aws/_meta/config.yml index ee54cc545586..7a338340d045 100644 --- a/x-pack/filebeat/module/aws/_meta/config.yml +++ b/x-pack/filebeat/module/aws/_meta/config.yml @@ -1,5 +1,31 @@ - module: aws - s3access: + cloudtrail: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + cloudwatch: enabled: false # AWS SQS queue url @@ -51,7 +77,7 @@ # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - vpcflow: + s3access: enabled: false # AWS SQS queue url @@ -77,7 +103,7 @@ # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - cloudtrail: + vpcflow: enabled: false # AWS SQS queue url diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index e222f55d23f5..da42283ef49a 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -26,7 +26,7 @@ Example config: [source,yaml] ---- - module: aws - s3access: + cloudtrail: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue #var.shared_credential_file: /etc/filebeat/aws_credentials @@ -35,50 +35,42 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com - elb: + cloudwatch: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws - - vpcflow: + elb: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials - - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws - - cloudtrail: + s3access: enabled: false - - # AWS SQS queue url #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Filename of AWS credential file - # If not set "$HOME/.aws/credentials" is used on Linux/Mac - # "%UserProfile%\.aws\credentials" is used on Windows - # var.shared_credential_file: /etc/filebeat/aws_credentials + vpcflow: + enabled: false + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com - # Profile name for aws credential - # If not set the default profile is used - # var.credential_profile_name: fb-aws ---- *`var.queue_url`*:: @@ -115,6 +107,16 @@ The `cloudtrail` fileset does not read the CloudTrail Digest files that are delivered to the S3 bucket when Log File Integrity is turned on, it only reads the CloudTrail logs. +[float] +=== cloudwatch fileset + +Users can use Amazon CloudWatch Logs to monitor, store, and access log files +from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources. +Then export logs from log groups to Amazon S3 bucket which has SQS notification +setup already. Right now, this fileset only parses EC2 logs into fields like `ip` +and `program_name`. For logs from other services, this fileset will store them into +`message` field. + [float] === elb fileset diff --git a/x-pack/filebeat/modules.d/aws.yml.disabled b/x-pack/filebeat/modules.d/aws.yml.disabled index 47cead91df5b..76cf7aaaf863 100644 --- a/x-pack/filebeat/modules.d/aws.yml.disabled +++ b/x-pack/filebeat/modules.d/aws.yml.disabled @@ -2,7 +2,33 @@ # Docs: https://www.elastic.co/guide/en/beats/filebeat/master/filebeat-module-aws.html - module: aws - s3access: + cloudtrail: + enabled: false + + # AWS SQS queue url + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + + # Filename of AWS credential file + # If not set "$HOME/.aws/credentials" is used on Linux/Mac + # "%UserProfile%\.aws\credentials" is used on Windows + #var.shared_credential_file: /etc/filebeat/aws_credentials + + # Profile name for aws credential + # If not set the default profile is used + #var.credential_profile_name: fb-aws + + # The duration that the received messages are hidden from ReceiveMessage request + # Default to be 300s + #var.visibility_timeout: 300s + + # Maximum duration before AWS API request will be interrupted + # Default to be 120s + #var.api_timeout: 120s + + # Custom endpoint used to access AWS APIs + #var.endpoint: amazonaws.com + + cloudwatch: enabled: false # AWS SQS queue url @@ -54,7 +80,7 @@ # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - vpcflow: + s3access: enabled: false # AWS SQS queue url @@ -80,7 +106,7 @@ # Custom endpoint used to access AWS APIs #var.endpoint: amazonaws.com - cloudtrail: + vpcflow: enabled: false # AWS SQS queue url From 8255e018c80be9d6c29209f10535ddabacd30797 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Thu, 27 Feb 2020 13:06:34 -0700 Subject: [PATCH 3/6] separate cloudwatch fileset and cloudwatch_ec2 fileset --- filebeat/docs/fields.asciidoc | 12 ++- .../module/aws/cloudwatch/_meta/fields.yml | 8 -- .../module/aws/cloudwatch/ingest/pipeline.yml | 4 +- .../module/aws/cloudwatch/manifest.yml | 1 + .../test/cloudwatch_ec2.log-expected.json | 36 +++------ .../aws/cloudwatch_ec2/_meta/fields.yml | 15 ++++ .../cloudwatch_ec2/config/cloudwatch_ec2.yml | 22 ++++++ .../module/aws/cloudwatch_ec2/config/file.yml | 6 ++ .../aws/cloudwatch_ec2/ingest/pipeline.yml | 24 ++++++ .../module/aws/cloudwatch_ec2/manifest.yml | 14 ++++ .../cloudwatch_ec2/test/cloudwatch_ec2.log | 6 ++ .../test/cloudwatch_ec2.log-expected.json | 74 +++++++++++++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- 13 files changed, 185 insertions(+), 39 deletions(-) create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log create mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index be7729c0a726..7c97ac12831b 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1319,18 +1319,24 @@ type: keyword Fields for AWS CloudWatch logs. +[float] +=== cloudwatch_ec2 + +Fields for AWS EC2 logs in CloudWatch. + -*`aws.cloudwatch.ip`*:: + +*`aws.cloudwatch_ec2.ip_address`*:: + -- The internet address of the requester. -type: ip +type: keyword -- -*`aws.cloudwatch.program_name`*:: +*`aws.cloudwatch_ec2.program_name`*:: + -- The program name of the log entry. diff --git a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml index bb1370e29e84..844c13309d62 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/_meta/fields.yml @@ -5,11 +5,3 @@ description: > Fields for AWS CloudWatch logs. fields: - - name: ip - type: ip - description: > - The internet address of the requester. - - name: program_name - type: keyword - description: > - The program name of the log entry. diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index 4ee5b892f338..7503193ce8a0 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -1,11 +1,9 @@ -description: "Pipeline for s3 server access logs" +description: "Pipeline for CloudWatch logs" processors: - grok: field: message patterns: - - >- - %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.cloudwatch.ip} %{NOTSPACE:aws.cloudwatch.program_name}: %{GREEDYDATA:message} - >- %{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message} diff --git a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml index ce14df8f1711..b71b96bbef1d 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/manifest.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/manifest.yml @@ -3,6 +3,7 @@ module_version: 1.0 var: - name: input default: s3 + - name: queue_url - name: shared_credential_file - name: credential_profile_name - name: visibility_timeout diff --git a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json index fdf6779614e4..f80bfe18e014 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json @@ -1,74 +1,62 @@ [ { - "@timestamp": "2020-02-25T20:41:45.297Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "systemd", + "@timestamp": "2020-02-27T19:29:13.341Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 0, - "message": "Stopping User Slice of root.", + "message": "Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root.", "service.type": "aws" }, { - "@timestamp": "2020-02-25T20:41:45.297Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "dhclient[3000]", + "@timestamp": "2020-02-27T19:29:13.348Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 96, - "message": "XMT: Solicit on eth0, interval 125240ms.", + "message": "Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.", "service.type": "aws" }, { - "@timestamp": "2020-02-25T20:41:45.304Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "dhclient[2898]", + "@timestamp": "2020-02-27T19:29:13.355Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 211, - "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", "service.type": "aws" }, { - "@timestamp": "2020-02-25T20:41:45.304Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "dhclient[2898]", + "@timestamp": "2020-02-27T19:29:13.355Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 345, - "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)", "service.type": "aws" }, { - "@timestamp": "2020-02-25T20:41:45.312Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "dhclient[2898]", + "@timestamp": "2020-02-27T19:29:13.355Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 461, - "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.", "service.type": "aws" }, { - "@timestamp": "2020-02-25T20:41:45.312Z", - "aws.cloudwatch.ip": "ip-172-31-81-156", - "aws.cloudwatch.program_name": "ec2net", + "@timestamp": "2020-02-27T19:29:13.355Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 586, - "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "message": "Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", "service.type": "aws" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml b/x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml new file mode 100644 index 000000000000..ff45b02c1d39 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml @@ -0,0 +1,15 @@ +- name: cloudwatch_ec2 + type: group + release: beta + default_field: false + description: > + Fields for AWS EC2 logs in CloudWatch. + fields: + - name: ip_address + type: keyword + description: > + The internet address of the requester. + - name: program_name + type: keyword + description: > + The program name of the log entry. diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml b/x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml new file mode 100644 index 000000000000..2af7bebff30c --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml @@ -0,0 +1,22 @@ +type: s3 +queue_url: {{ .queue_url }} + +{{ if .credential_profile_name }} +credential_profile_name: {{ .credential_profile_name }} +{{ end }} + +{{ if .shared_credential_file }} +shared_credential_file: {{ .shared_credential_file }} +{{ end }} + +{{ if .visibility_timeout }} +visibility_timeout: {{ .visibility_timeout }} +{{ end }} + +{{ if .api_timeout }} +api_timeout: {{ .api_timeout }} +{{ end }} + +{{ if .endpoint }} +endpoint: {{ .endpoint }} +{{ end }} diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml b/x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml new file mode 100644 index 000000000000..8bfbcc9f802b --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml @@ -0,0 +1,6 @@ +type: log +paths: + {{ range $i, $path := .paths }} + - {{$path}} + {{ end }} +exclude_files: [".gz$"] diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml new file mode 100644 index 000000000000..1a5228e6bba7 --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml @@ -0,0 +1,24 @@ +description: "Pipeline for EC2 logs in CloudWatch" + +processors: + - grok: + field: message + patterns: + - >- + %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.cloudwatch_ec2.ip_address} %{NOTSPACE:aws.cloudwatch_ec2.program_name}: %{GREEDYDATA:message} + + - date: + field: "_tmp_.timestamp" + target_field: "@timestamp" + ignore_failure: true + formats: + - "dd/MMM/yyyy:H:m:s Z" + - remove: + field: + - _tmp + ignore_missing: true + +on_failure: + - set: + field: "error.message" + value: "{{ _ingest.on_failure_message }}" diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml b/x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml new file mode 100644 index 000000000000..b71b96bbef1d --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml @@ -0,0 +1,14 @@ +module_version: 1.0 + +var: + - name: input + default: s3 + - name: queue_url + - name: shared_credential_file + - name: credential_profile_name + - name: visibility_timeout + - name: api_timeout + - name: endpoint + +ingest_pipeline: ingest/pipeline.yml +input: config/{{.input}}.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log b/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log new file mode 100644 index 000000000000..4487fdf08d2e --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log @@ -0,0 +1,6 @@ +2020-02-20T07:01:01.000Z Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root. +2020-02-20T07:02:18.000Z Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22) +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds. +2020-02-20T07:02:37.000Z Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json new file mode 100644 index 000000000000..7f4dea707f2f --- /dev/null +++ b/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-02-27T19:53:02.540Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "systemd", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 0, + "message": "Stopping User Slice of root.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-27T19:53:02.547Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "dhclient[3000]", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 96, + "message": "XMT: Solicit on eth0, interval 125240ms.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-27T19:53:02.547Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 211, + "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-27T19:53:02.553Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 345, + "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-27T19:53:02.554Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 461, + "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "service.type": "aws" + }, + { + "@timestamp": "2020-02-27T19:53:02.554Z", + "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", + "aws.cloudwatch_ec2.program_name": "ec2net", + "event.dataset": "aws.cloudwatch_ec2", + "event.module": "aws", + "fileset.name": "cloudwatch_ec2", + "input.type": "log", + "log.offset": 586, + "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "service.type": "aws" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index e77180088a58..a634181e1d09 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMW1uT2zayfvev6PJLxlWSTsVJnTo1p3Kq5PH4RJuJPTuSk90nGgJbInYggAHAkeVfv9UAeBNJzUVSsnqwNSLZ+PqCvqE5hnvcXQLb2lcATjiJlzD9ff4KwKBEZvESlujYK4AULTcid0KrS/i/VwAAv+q0kAgrbSBjKpVCrUHqtYWV0RsiM3kFsBIoU3vpHxiDYhssl6OP2+V4CWujizz+0rMOfT54MhVlv84kXm0u0VyGS12kzjAhq0t9K9Jnn9vyk+KKFdIlfolLWDFpsXW5F2wTsDYe7xVhWRCWFvQ++E0W8AGVSx7QWKFV646Sk3vcbbVJ964dAEafRYZNRJE+6BW4DAlgWJjQb5ib9EIrLJpEpKiccLteaPtC7gIb9yIjyrNIGFDihqBwrRwTykKKjglpgS114TxeWg30qkNrNv0VSoDgMuZgw1L0jxj8o0DrRsBUCttM8Ay4QX8vkxa2aLBDrrCYTmC2AoebXBtmdp1n/D0jv0KJ22Z6ayHTW/q1Q7NDQC+JS0wne7f2GUlTGySDzsXDNtJVR88NQSNRwp6xAZU3drfZt9TnI+kaRgllumHftII7tLowHOEj2yBcTO8+vikB5kYoLnIm93TOmZT7Ym2g5hytTe5xl4g+fKfCH9YhQjB7HxBumfWGA06DFWvVtNBhwBYtbdqENgZ+dYOQ+3bhUwHPVk0sHqgX51a4rLENLPLC9JkEtE2ctlu1MTzrudEPIkULQgVfQ26o3tmRx166lei4QeYw9a7WZdpic8meR4e2UlO4mxVLWOEyosKJeu/dj1vFUwUN0ToemCwQhAVn6P8ofq2dd4qgjXdq/vuWWB0k1uuZoohqhTJptZdhi9egXtYvdvr8+mEKKT4Ijv8L2mVotsLiKETHrsE25ep1RVabMjcEPsj0wA3PESiR8U7eiQ3CNsOwu7q225WYsLboOuKaF6Ee9D2mybLP7k/lLmip0rFRHmHRkNyHwpkteAasz+qh9JzXV29hWjgNc858yhYzlGvJrBMc3iFT1jF53x/20RhtEq7Tfe08PR3pj/pN7vwilflHYzXoCqOs9yF0/RC+DVrL1qeEODsMJgT9BpFSaQegRlpJzgzboEOzr7djRVoTHpEwmdqNotsk+7bkhYMnH4o3NVKba2Uxic771EBL+lVwIEfFOD1jS1O/R+AZU2u0cBFc/qibouW030dk0ilKpK0fiLzp54ulqaDrTCY+4U1ZK/s/lrVpRZ78EGsEtpBeVyFMaUfKctFoOoRKq9OmEtVhk+qkMMdqKAQmDzgkgCuBtmU4/rZyAy9RqHU322VSYgprVGiY888LG0gPbGZf/vTktkft5Db+MrdtqKQEmDZ0ZZBrkw6YUS6OrtEexTm9nVWFGrNWc1HHan99a6e5uGJSdih5DhbE5wFZb5hia7/7wl445T6Ad1pLZGrAjLYZUhbRkLawsL8RoYEw3DW0BViaaCX7a9KjVVFjFRZ0TnZCGiHAfukxLV1fGPSnvnrp96MvqZunIIX1/qOiHUsNTEGoWrTPLSzPV81VJdz07mM3Y2nUZbpQPR7tVDCmYQGqxnQZ4yMwvVVozltyPyIa75yqnWJKTYZaybfqLi/Z1o6j3x17ZJcUbMb0qP97wAK5yAVt9kEBH7Nh7jA3SAlG8F2slrHf+wY5igfvX4U9tJkjX8EjJbH1cz4fW+WftNwIhOKySClH3hJqZ8R6jSaEhX4nG0qNYEOFHGIqYwbTyNNJxf7/n2fvG9FruWt2+ZyGQok/CpS70qSa1/sZii1XLxwqRShLDzlV9OI25JFOQypWKzT0R+ggtz/RBGy/SB5ynqBKcy1OLZI9Df92ewXlQrSbQu8v5jCx8PNllWe7G4PoeaeBKV/3NouWqvgqC635DzWvrZb0ljmevdpn769pSf9OWJ7Vkhb7oSnA7/z8hBa0UA6NQkd5uEFrawfs1YEDdVNu9NqwTdJxbkf3xCPlVtXtu+HKmV1XmyiXT1fjmr1IS9c3756lntMLpS0MlsKSSab4kHaOytZ7ATS7zy0AXkgPb+GGfnwXfxzwMI6ZNbrEq2fSTWqOhDi9+1giDAsFO6jPxR4tsSl9w266cSQubz2Rcif0InCtFPLhPDU32mmu92PD8bvMU+3X6UXmXE6VruP5QNleNU6MpgxXqHXixAYnFnkv0pXUbL+eeYrdacdkaBwKBRa5VqkFKxTHPemFwjzU6MLWMi6UExJEK0xSmbAmlVAisWT8HtVAWRkv/gex2WCDrkSA4ISUrR+sY8bZ2KugBOqRttJfzGHVfmrqrsp7Wly2denFI8Vg+lqLLvC16c9dpe40S57HVdxITU0p2AgpRWR2FLkN8HWOClsMcantfq+78pzSJuTHbMbu8bx8lOcTi5s5VEuSoLne5L6ft8cX6B4rzZiFJaICtI4tpbDZEGvl9hvIaV7s4Wa3+ylNaUS1pYfa5zEPXCLMtXl5R6bfC2vjynryWHTksidVZ9I65gp73BlBL+RAGIhwd3teNK92N2h4gOIh7V+nTeiLfBmE/qW/drMy4SLPTh2o5/MbCHTDSZ1QtAn+y/9cKWEgtSFM54nUtHwVrZ+Ni2faoko4GnfWjCusA7SOWPkjTIjdh3Cm3jCCl8K3aAQ7sXADTVDFZonmzLwIxfXGh1dpEybx1M6EKrk1mtjf1Svvwf06jYi63PWl8OU+Dlz5g6rI8XA4om9p0tcnPZIPgu2Jh/x9AIFhHI/pUvQu7ec4lKsixpd/jKebb2q8oNXGs/QLZMjSoaprQ3U8pokpJKVTQvfMYxydtgeqtZJ97lTI6jTaQ2gfSIsVJbx0T7w8cIISkiT8irzoTlscCTz2q0riYQqgFerqc92LlTZbZtIRrMRXTMdlZBi1RiQmk8mbCcwccKbKyTSw+ICGySCegX1oMBUGuUsKc2Jv8vnuJnpoL/G4jj8sCv9tma1EcODofGKQ2RefY/VCC6f5gW55qF8pI6JbMSGbuKrY9kM4xTh7n2X+g2/koSkntJ7TdFkW/B5d0ndecKxXYEorwZkMQz/1IYVfq/wjnhYEGAOpmr92vs5QoB88wf7MWNxrpO01E2p4ymCjHSYnbDCyPGcmxMoXdRqry3+SUqv1/AgDg7E3z0K1x7PKbvXZRgA6oKdgnSFX2TpbqPrc/UcLyOrOej/U6pT0tOKtT2V96yuFLIxzQYpcMkr8mYX5p+ntpLpzBHfX88Xk58XiNtmgy3Q6KU8B/fjBCH6/fjefLa4P3aINvJsurn6evL++uV5cTz69+9v11aKf9Xs8cXR+fY+7180hkjoGU2hARTVN6kG+Hr8u3XAtqlRjmENxVHIzP6BazQ8dtrTCiNPychcIjz/fzVockewrxxJHvPqhUVWXhGLuhJ0KVWzQCB5wNMvN+uD4wGzOCSbn+iuhahte+0B7pVNs6lnpGIE154Uxg72QnUOb2KHJjxdLLBY3VY/Nr+ML9BHg1/KA1Yu0bgw/oKFst8nGNzR6wI0s/0UplRXf+iV7TFuKiFYh169DdZe3wMGuiH/SN8rOIslW/2s/1AoFKynWmWuc2/q05jsLORqbU1L4MGChrjAqYUYXKv3T4DPXMGCbU7BupOY7XZjHRhNXaMypI3SrxPG2eRfXiVXY43Wqfz+GrY+ZpHoitM8WzXhKKx0sEePw2MkL13Iobfa+bCdWkeepwSaSmKWPhZxMnzizIQa+jtnm21ik47d+FrqyRvzqUKV1wgWz9wMNOLFWzBUGz/OmVkW+lNMI5mL9m0dLX34cdd8jaWaMLSfx4rwydCcTW4jOYP6xXUbkxNxcUwVh4Ybt0MDFfH7zpmyJ1oOyuNZOVK97kPnP+1ijCwOdhpplfyxzttPq6h2L1oLxDbFp4bKf/V4No87te8IutiP4e4FmNw+pN933B/1d5uIXucEx2QamlOK9eblq/a4Ki55WFtWkTWmWsVFJX3umZGCvzXeW3bQwTFl/+BEMbV6+AHKxuJm/qbxZw9Ji33L/oK8xu7SSevv0DkVnoOepPYrfbq+AlnpWb+IsQiQkHwjJjV7bcgn/cuROF6Tt+DqEH1WK44rhFYlSvsLC2+oBSkuoZGTAC+v0ZuiJAVs5wQhjf2bth92q0cXydLJUwVCr3aFZnaNDXPcJFLqtNvf1Wh5bmGXzMyiGrVaCx/NsbdLDfdeztFvL8cC+WfWIbwTTq6vr2wV5rrvr4WJZ6vWhYu7FSKVer8mTxlIuCrdU7wg+/TKCj5/eTxdTH2p/md3S9yG1W8fUWbVeLuFF+11Xsi+wilGZm1W0hfWtRe/1droYmAu6d4k1nKVpf8B4Sa8uZxT+xxIfUMKFNmItFJNvyt5m90g9sjOMMLXuT0GYUjGoQuhuwCzdxUGcDzk/o8X4aVXah9Xb8yf1HrZYKjy9263xhwXOyYLjebKSbH1iz7IUbsPsfSzWqsChpdRb8jiLq1vwy17C25/m//w4+v5/6L/x9OqX0fc/fZh9HP3409180Q/5fAOWQWqXMLt9+HFE//63r+GuP0wnr/4dAAD//xEO3xg=" + return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1Jt+FnyN/+9SxQLRtOHKRusPPU5rrMMyaERfdyj9KZK0vt0Wr0kKjXcSVoGfaUg7k1S8vWWctnHw0oUd5pJoQmv/Z221ZSVPOnb+2SvWznbt496yDk9ELZFQYTMGeKad63O0cVIZ0Amk31HQBBSA9v4YZ+fJd+7HGcntkl+ixsz6idqx0JcXz3sUQYF4p6UB/3Pdo5oKwU21nUkbiC9iTKrYwCgRutkfen37k13nCzH/KOt7JAtXtPL1be51TAe573dCOqfpA1lLhLvcy8XOPIIe9EulCG7ZdpT9E745mK/VCpwSE3WjhwUnPck17sN8TWg3S1jAvtpQK5E/2p+lnSllB+NGf8HnVPtZwu/gex2WCDriSA4KVSOz84z6x3qQVDeeEj3bK/mMOqq9bcuyqd2+Fydy+DeJTszcpr0UW+1t0BVJlWD+h5XCVDau6UhrVUSiZmB4nbCN/kqHGHIa6M22/hV55TuYz8mFuxezwvH+Wxy+xmCtWSJGhu1nloU+7xBaZDS1fMwRxRAzrP5kq6VR9rpfnJ/S7CkR5ucruf0pRKVGt6LOke88AlwtzYlzeaur2wsb4sk49FRy57VDVcnWe+cMcdfXRCjoSBCLfN86J5tW2g8QGKh2S/3tjY7vnSC/1Ld0nqVMZlvjp1oJ5ObyDSjQeQUpMR/Ff4udqEntSGMJ0nUtPyVbR+Ni6+Mg51xtH6s2ZccR2gdeQinMxCaqrEUYGGErwUvkMr2YmFG2mCLtZztGfmRWpu1iG8Kpcxhad2JlTJLdGmtrVZBA8e1mlE1Pm2K4Uv7ThyFc7fEsf94Yi+iayr/XskHwQ7EI/5ew8Cyzge03zpXDqMp2hfRYwv/xiO19/0cEarDSfiC6yQib6qa031PIrMForSKWk6xkyOTtsj1XqTQ+5UqOqQPUDYPWeXC0p46Z50uedgKCZJ+BV50R4iORJ4asOVxONww06oq4+rLxbGbpgVA1jIryiGZWQY7Ex+jEajNyOYeOBMlwN34PABLVNRPD12aFFIi9xnhT2xN/l8d5M8dJB4WiecgcV/NsxVIjgwETCyyNyLj+c6ocUhhUi3nFWoNiOhWzCpmriq2PZDPJw5e59l+kPoT6ItB8+e03SZF/wefdZ1DHKsV2DaaMmZirNM9dlLWKv8Ix2CRBg9qVq4dr7OUKQfPcH+KFyyNdrtJZO6f3hibTxmPcl46+enWH6eMxtj5Ys6jdXlP2lTq/XCZAaDYVDPQu9OnZVN+LNNNrRAj8F5S65y58ikat93n5ggqw8MuqFWh7+nFW992BxaXwJWcUoNBHLFKPFnDqafxrej6s4B3F1PZ6OfZ7PbbI1+ZcSoPNwMUxUD+P363XQyuz50i7Hwbjy7+nn0/vrmenY9+vTub9dXs27W7/HE0fn1PW5fN2dj6hhMoQE11TQigHw9fF264VpUwmAcr/FUcrMwd1uNRR3WtMLK0/JyFwkPP99Ndjgi2VeOJU2udUOjqi6LxdwJOxW6WKOVPOJolpv1efiBkaMTDAR2V0KVGV6HQHtlBDb3WZsUgQ3nhbW9vZCtR5e5voGWF0ssFTdVjy2sEwr0AeDX8tw4iLRuDD+gpWy3ycY3tKbHjcz/RSmVk9+6JXtMW4qIViE3rEN1V9DA3q5IeDI0ys4iyZ3+136olRoWSi5XvnEcHdKa7xzkaF1OSeFDj4b6wuqMWVNo8afBZ76hwC6nYN1IzbemsI9NXC7Q2lNH6J0SJ+jmXVonVWGP16nhtR+2PGZA7InQPju0wzGtdLBETDNxJy9cy1m7yfuynVhFnqcGm0RiIh4LOStz4syGGPg6ZOtvQymGb8OId6WN+NWjFnXCBZP3PQ04udTMFxbP8wJaRb6U0wCmcvlbQEtffhy0X49pZow7TuLFeWXsTmaukK33DY7tMiIn5qaGKggHN2yLFi6m05s3ZUu0nv/FpfGyeouF1H/axRpd6Ok01CyHY5mznVZXr47sLJhefBsXfvVzsNU4wb17T7RiN4C/F2i305h6031/0N9lLn6RWxySbqCgFO/Ny7c2WFVc9LSyqAaISrVMjUr62jH8A3ttvrNY08wy7cLhR1S0afley8XsZvqm8mYNTUt9y/2DvsZI1kKZzdM7FK0hn6f2KH67vQJa6lm9ibMIkZB8ICQ3ZunKJcI7n1tT0G6ntzzC+FKawoxvfpTylQ7eVg9QWkIlIwNeOG/WfU/06MoJJjO7M+sww1dNZJank+UW9LXaPdrFOTrEdZ9Ao98Ye1+vFbDFEb0wg2LZYiF5Os82Vhzuu56l3VpOPXaN4Cd8AxhfXV3fzshz3V33F8vKLA8Vcy9GqsxySZ40lXJJuOX2DuDTLwP4+On9eDYOofaXyS1979t255k+666XSwTRfteW7Au0YlDmZhVt6UJrMXi9rSl65oLufeYsZ0J0B4yX9OpyRuF/qPABFVwYK5dSM/Wm7G22j9QTO/0IhfN/CkJBxaCOobsBs3QXB3E+5PyMGhOGcMkOq/8pwEm9hyvmGk/vdmv8cYFzsuB5ni0UW57Ys8ylXzN3n4q1KnAYpcyGPM7s6hbCspfw9qfpPz8Ovv8f+mc4vvpl8P1PHyYfBz/+dDeddUM+34BllNolTG4ffhzQf/871HDXH8ajV/8OAAD//z2zGtI=" } From 7ecd80510b3e4d3c60a562268f506a45b26caf99 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 3 Mar 2020 08:53:15 -0700 Subject: [PATCH 4/6] change cloudwatch_ec2 fileset name to ec2 --- CHANGELOG.next.asciidoc | 2 +- filebeat/docs/fields.asciidoc | 6 +- filebeat/docs/modules/aws.asciidoc | 25 +++++-- .../filebeat/module/aws/_meta/docs.asciidoc | 25 +++++-- .../test/cloudwatch_ec2.log-expected.json | 12 +-- .../test/cloudwatch_ec2.log-expected.json | 74 ------------------- .../{cloudwatch_ec2 => ec2}/_meta/fields.yml | 2 +- .../cloudwatch_ec2.yml => ec2/config/ec2.yml} | 0 .../{cloudwatch_ec2 => ec2}/config/file.yml | 0 .../ingest/pipeline.yml | 2 +- .../aws/{cloudwatch_ec2 => ec2}/manifest.yml | 0 .../cloudwatch_ec2.log => ec2/test/ec2.log} | 0 .../module/aws/ec2/test/ec2.log-expected.json | 74 +++++++++++++++++++ x-pack/filebeat/module/aws/fields.go | 2 +- 14 files changed, 127 insertions(+), 97 deletions(-) delete mode 100644 x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json rename x-pack/filebeat/module/aws/{cloudwatch_ec2 => ec2}/_meta/fields.yml (93%) rename x-pack/filebeat/module/aws/{cloudwatch_ec2/config/cloudwatch_ec2.yml => ec2/config/ec2.yml} (100%) rename x-pack/filebeat/module/aws/{cloudwatch_ec2 => ec2}/config/file.yml (100%) rename x-pack/filebeat/module/aws/{cloudwatch_ec2 => ec2}/ingest/pipeline.yml (77%) rename x-pack/filebeat/module/aws/{cloudwatch_ec2 => ec2}/manifest.yml (100%) rename x-pack/filebeat/module/aws/{cloudwatch_ec2/test/cloudwatch_ec2.log => ec2/test/ec2.log} (100%) create mode 100644 x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 7b691b94f984..e4f532983eb3 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -149,7 +149,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Improve ECS field mappings in aws module. {issue}16154[16154] {pull}16307[16307] - Improve ECS categorization field mappings in googlecloud module. {issue}16030[16030] {pull}16500[16500] - Improve ECS field mappings in haproxy module. {issue}16162[16162] {pull}16529[16529] -- Add cloudwatch fileset in aws module. {issue}13716[13716] {pull}16579[16579] +- Add cloudwatch fileset and ec2 fileset in aws module. {issue}13716[13716] {pull}16579[16579] - Improve the decode_cef processor by reducing the number of memory allocations. {pull}16587[16587] - Add `cloudfoundry` input to send events from Cloud Foundry. {pull}16586[16586] - Improve ECS categorization field mappings in iis module. {issue}16165[16165] {pull}16618[16618] diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index 7c97ac12831b..c7c57808e61a 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1320,13 +1320,13 @@ Fields for AWS CloudWatch logs. [float] -=== cloudwatch_ec2 +=== ec2 Fields for AWS EC2 logs in CloudWatch. -*`aws.cloudwatch_ec2.ip_address`*:: +*`aws.ec2.ip_address`*:: + -- The internet address of the requester. @@ -1336,7 +1336,7 @@ type: keyword -- -*`aws.cloudwatch_ec2.program_name`*:: +*`aws.ec2.program_name`*:: + -- The program name of the log entry. diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 01cc8709c4d7..2cdfb6c60377 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -51,6 +51,15 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com + ec2: + enabled: false + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com + elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue @@ -118,11 +127,17 @@ on, it only reads the CloudTrail logs. === cloudwatch fileset Users can use Amazon CloudWatch Logs to monitor, store, and access log files -from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources. -Then export logs from log groups to Amazon S3 bucket which has SQS notification -setup already. Right now, this fileset only parses EC2 logs into fields like `ip` -and `program_name`. For logs from other services, this fileset will store them into -`message` field. +from different sources. Export logs from log groups to Amazon S3 bucket which +has SQS notification setup already. This fileset will parse these logs into +`timestamp` and `message` field. + +[float] +=== ec2 fileset + +This fileset is specifically for EC2 logs stored in AWS CloudWatch. Export logs +from log groups to Amazon S3 bucket which has SQS notification setup already. +With this fileset, EC2 logs will be parsed into fields like `ip` +and `program_name`. For logs from other services, please use `cloudwatch` fileset. [float] === elb fileset diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index c9b459a1005b..a352ff797fd5 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -46,6 +46,15 @@ Example config: #var.api_timeout: 120s #var.endpoint: amazonaws.com + ec2: + enabled: false + #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue + #var.shared_credential_file: /etc/filebeat/aws_credentials + #var.credential_profile_name: fb-aws + #var.visibility_timeout: 300s + #var.api_timeout: 120s + #var.endpoint: amazonaws.com + elb: enabled: false #var.queue_url: https://sqs.myregion.amazonaws.com/123456/myqueue @@ -113,11 +122,17 @@ on, it only reads the CloudTrail logs. === cloudwatch fileset Users can use Amazon CloudWatch Logs to monitor, store, and access log files -from Amazon EC2 instances, AWS CloudTrail, Route 53, and other sources. -Then export logs from log groups to Amazon S3 bucket which has SQS notification -setup already. Right now, this fileset only parses EC2 logs into fields like `ip` -and `program_name`. For logs from other services, this fileset will store them into -`message` field. +from different sources. Export logs from log groups to Amazon S3 bucket which +has SQS notification setup already. This fileset will parse these logs into +`timestamp` and `message` field. + +[float] +=== ec2 fileset + +This fileset is specifically for EC2 logs stored in AWS CloudWatch. Export logs +from log groups to Amazon S3 bucket which has SQS notification setup already. +With this fileset, EC2 logs will be parsed into fields like `ip` +and `program_name`. For logs from other services, please use `cloudwatch` fileset. [float] === elb fileset diff --git a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json index f80bfe18e014..90a9c5340436 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json @@ -1,6 +1,6 @@ [ { - "@timestamp": "2020-02-27T19:29:13.341Z", + "@timestamp": "2020-03-03T15:08:08.438Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", @@ -10,7 +10,7 @@ "service.type": "aws" }, { - "@timestamp": "2020-02-27T19:29:13.348Z", + "@timestamp": "2020-03-03T15:08:08.446Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", @@ -20,7 +20,7 @@ "service.type": "aws" }, { - "@timestamp": "2020-02-27T19:29:13.355Z", + "@timestamp": "2020-03-03T15:08:08.453Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", @@ -30,7 +30,7 @@ "service.type": "aws" }, { - "@timestamp": "2020-02-27T19:29:13.355Z", + "@timestamp": "2020-03-03T15:08:08.453Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", @@ -40,7 +40,7 @@ "service.type": "aws" }, { - "@timestamp": "2020-02-27T19:29:13.355Z", + "@timestamp": "2020-03-03T15:08:08.453Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", @@ -50,7 +50,7 @@ "service.type": "aws" }, { - "@timestamp": "2020-02-27T19:29:13.355Z", + "@timestamp": "2020-03-03T15:08:08.453Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json deleted file mode 100644 index 7f4dea707f2f..000000000000 --- a/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log-expected.json +++ /dev/null @@ -1,74 +0,0 @@ -[ - { - "@timestamp": "2020-02-27T19:53:02.540Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "systemd", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 0, - "message": "Stopping User Slice of root.", - "service.type": "aws" - }, - { - "@timestamp": "2020-02-27T19:53:02.547Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "dhclient[3000]", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 96, - "message": "XMT: Solicit on eth0, interval 125240ms.", - "service.type": "aws" - }, - { - "@timestamp": "2020-02-27T19:53:02.547Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "dhclient[2898]", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 211, - "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", - "service.type": "aws" - }, - { - "@timestamp": "2020-02-27T19:53:02.553Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "dhclient[2898]", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 345, - "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", - "service.type": "aws" - }, - { - "@timestamp": "2020-02-27T19:53:02.554Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "dhclient[2898]", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 461, - "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", - "service.type": "aws" - }, - { - "@timestamp": "2020-02-27T19:53:02.554Z", - "aws.cloudwatch_ec2.ip_address": "ip-172-31-81-156", - "aws.cloudwatch_ec2.program_name": "ec2net", - "event.dataset": "aws.cloudwatch_ec2", - "event.module": "aws", - "fileset.name": "cloudwatch_ec2", - "input.type": "log", - "log.offset": 586, - "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", - "service.type": "aws" - } -] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml similarity index 93% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml rename to x-pack/filebeat/module/aws/ec2/_meta/fields.yml index ff45b02c1d39..4c7a71c16aa9 100644 --- a/x-pack/filebeat/module/aws/cloudwatch_ec2/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml @@ -1,4 +1,4 @@ -- name: cloudwatch_ec2 +- name: ec2 type: group release: beta default_field: false diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml b/x-pack/filebeat/module/aws/ec2/config/ec2.yml similarity index 100% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/config/cloudwatch_ec2.yml rename to x-pack/filebeat/module/aws/ec2/config/ec2.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml b/x-pack/filebeat/module/aws/ec2/config/file.yml similarity index 100% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/config/file.yml rename to x-pack/filebeat/module/aws/ec2/config/file.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml similarity index 77% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml rename to x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 1a5228e6bba7..947edfcac3af 100644 --- a/x-pack/filebeat/module/aws/cloudwatch_ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -5,7 +5,7 @@ processors: field: message patterns: - >- - %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.cloudwatch_ec2.ip_address} %{NOTSPACE:aws.cloudwatch_ec2.program_name}: %{GREEDYDATA:message} + %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.ec2.ip_address} %{NOTSPACE:aws.ec2.program_name}: %{GREEDYDATA:message} - date: field: "_tmp_.timestamp" diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml b/x-pack/filebeat/module/aws/ec2/manifest.yml similarity index 100% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/manifest.yml rename to x-pack/filebeat/module/aws/ec2/manifest.yml diff --git a/x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log b/x-pack/filebeat/module/aws/ec2/test/ec2.log similarity index 100% rename from x-pack/filebeat/module/aws/cloudwatch_ec2/test/cloudwatch_ec2.log rename to x-pack/filebeat/module/aws/ec2/test/ec2.log diff --git a/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json b/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json new file mode 100644 index 000000000000..7c38e9da696c --- /dev/null +++ b/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json @@ -0,0 +1,74 @@ +[ + { + "@timestamp": "2020-03-03T15:38:00.278Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "systemd", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 0, + "message": "Stopping User Slice of root.", + "service.type": "aws" + }, + { + "@timestamp": "2020-03-03T15:38:00.285Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "dhclient[3000]", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 96, + "message": "XMT: Solicit on eth0, interval 125240ms.", + "service.type": "aws" + }, + { + "@timestamp": "2020-03-03T15:38:00.291Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 211, + "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-03-03T15:38:00.291Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 345, + "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "service.type": "aws" + }, + { + "@timestamp": "2020-03-03T15:38:00.292Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "dhclient[2898]", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 461, + "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "service.type": "aws" + }, + { + "@timestamp": "2020-03-03T15:38:00.292Z", + "aws.ec2.ip_address": "ip-172-31-81-156", + "aws.ec2.program_name": "ec2net", + "event.dataset": "aws.ec2", + "event.module": "aws", + "fileset.name": "ec2", + "input.type": "log", + "log.offset": 586, + "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "service.type": "aws" + } +] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 2f53db412e39..ec41b0bc81bf 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1Jt+FnyN/+9SxQLRtOHKRusPPU5rrMMyaERfdyj9KZK0vt0Wr0kKjXcSVoGfaUg7k1S8vWWctnHw0oUd5pJoQmv/Z221ZSVPOnb+2SvWznbt496yDk9ELZFQYTMGeKad63O0cVIZ0Amk31HQBBSA9v4YZ+fJd+7HGcntkl+ixsz6idqx0JcXz3sUQYF4p6UB/3Pdo5oKwU21nUkbiC9iTKrYwCgRutkfen37k13nCzH/KOt7JAtXtPL1be51TAe573dCOqfpA1lLhLvcy8XOPIIe9EulCG7ZdpT9E745mK/VCpwSE3WjhwUnPck17sN8TWg3S1jAvtpQK5E/2p+lnSllB+NGf8HnVPtZwu/gex2WCDriSA4KVSOz84z6x3qQVDeeEj3bK/mMOqq9bcuyqd2+Fydy+DeJTszcpr0UW+1t0BVJlWD+h5XCVDau6UhrVUSiZmB4nbCN/kqHGHIa6M22/hV55TuYz8mFuxezwvH+Wxy+xmCtWSJGhu1nloU+7xBaZDS1fMwRxRAzrP5kq6VR9rpfnJ/S7CkR5ucruf0pRKVGt6LOke88AlwtzYlzeaur2wsb4sk49FRy57VDVcnWe+cMcdfXRCjoSBCLfN86J5tW2g8QGKh2S/3tjY7vnSC/1Ld0nqVMZlvjp1oJ5ObyDSjQeQUpMR/Ff4udqEntSGMJ0nUtPyVbR+Ni6+Mg51xtH6s2ZccR2gdeQinMxCaqrEUYGGErwUvkMr2YmFG2mCLtZztGfmRWpu1iG8Kpcxhad2JlTJLdGmtrVZBA8e1mlE1Pm2K4Uv7ThyFc7fEsf94Yi+iayr/XskHwQ7EI/5ew8Cyzge03zpXDqMp2hfRYwv/xiO19/0cEarDSfiC6yQib6qa031PIrMForSKWk6xkyOTtsj1XqTQ+5UqOqQPUDYPWeXC0p46Z50uedgKCZJ+BV50R4iORJ4asOVxONww06oq4+rLxbGbpgVA1jIryiGZWQY7Ex+jEajNyOYeOBMlwN34PABLVNRPD12aFFIi9xnhT2xN/l8d5M8dJB4WiecgcV/NsxVIjgwETCyyNyLj+c6ocUhhUi3nFWoNiOhWzCpmriq2PZDPJw5e59l+kPoT6ItB8+e03SZF/wefdZ1DHKsV2DaaMmZirNM9dlLWKv8Ix2CRBg9qVq4dr7OUKQfPcH+KFyyNdrtJZO6f3hibTxmPcl46+enWH6eMxtj5Ys6jdXlP2lTq/XCZAaDYVDPQu9OnZVN+LNNNrRAj8F5S65y58ikat93n5ggqw8MuqFWh7+nFW992BxaXwJWcUoNBHLFKPFnDqafxrej6s4B3F1PZ6OfZ7PbbI1+ZcSoPNwMUxUD+P363XQyuz50i7Hwbjy7+nn0/vrmenY9+vTub9dXs27W7/HE0fn1PW5fN2dj6hhMoQE11TQigHw9fF264VpUwmAcr/FUcrMwd1uNRR3WtMLK0/JyFwkPP99Ndjgi2VeOJU2udUOjqi6LxdwJOxW6WKOVPOJolpv1efiBkaMTDAR2V0KVGV6HQHtlBDb3WZsUgQ3nhbW9vZCtR5e5voGWF0ssFTdVjy2sEwr0AeDX8tw4iLRuDD+gpWy3ycY3tKbHjcz/RSmVk9+6JXtMW4qIViE3rEN1V9DA3q5IeDI0ys4iyZ3+136olRoWSi5XvnEcHdKa7xzkaF1OSeFDj4b6wuqMWVNo8afBZ76hwC6nYN1IzbemsI9NXC7Q2lNH6J0SJ+jmXVonVWGP16nhtR+2PGZA7InQPju0wzGtdLBETDNxJy9cy1m7yfuynVhFnqcGm0RiIh4LOStz4syGGPg6ZOtvQymGb8OId6WN+NWjFnXCBZP3PQ04udTMFxbP8wJaRb6U0wCmcvlbQEtffhy0X49pZow7TuLFeWXsTmaukK33DY7tMiIn5qaGKggHN2yLFi6m05s3ZUu0nv/FpfGyeouF1H/axRpd6Ok01CyHY5mznVZXr47sLJhefBsXfvVzsNU4wb17T7RiN4C/F2i305h6031/0N9lLn6RWxySbqCgFO/Ny7c2WFVc9LSyqAaISrVMjUr62jH8A3ttvrNY08wy7cLhR1S0afley8XsZvqm8mYNTUt9y/2DvsZI1kKZzdM7FK0hn6f2KH67vQJa6lm9ibMIkZB8ICQ3ZunKJcI7n1tT0G6ntzzC+FKawoxvfpTylQ7eVg9QWkIlIwNeOG/WfU/06MoJJjO7M+sww1dNZJank+UW9LXaPdrFOTrEdZ9Ao98Ye1+vFbDFEb0wg2LZYiF5Os82Vhzuu56l3VpOPXaN4Cd8AxhfXV3fzshz3V33F8vKLA8Vcy9GqsxySZ40lXJJuOX2DuDTLwP4+On9eDYOofaXyS1979t255k+666XSwTRfteW7Au0YlDmZhVt6UJrMXi9rSl65oLufeYsZ0J0B4yX9OpyRuF/qPABFVwYK5dSM/Wm7G22j9QTO/0IhfN/CkJBxaCOobsBs3QXB3E+5PyMGhOGcMkOq/8pwEm9hyvmGk/vdmv8cYFzsuB5ni0UW57Ys8ylXzN3n4q1KnAYpcyGPM7s6hbCspfw9qfpPz8Ovv8f+mc4vvpl8P1PHyYfBz/+dDeddUM+34BllNolTG4ffhzQf/871HDXH8ajV/8OAAD//z2zGtI=" + return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1JV/kYf/vX46YCNhwzSN3g4akddZlnTAiL7uVupDNBltqj1eghUa+DSVAt7KkBc2uWlq2zlqM+GlCivNNBCJ197e22rZmo5k/f2iV72c7dvHvW6cfphbIrDCZgzhTTvG93jqo8OgE0O+k7AIKQHt7CDf34Lv3Y4y09s0v0WdieUTtBOxLi+O5jiTAuFPWgPuN7tF1AqSi2U6cjcQXtSZRbaQQCN1oj78+5c2u84WY/zh1vZYFq955erLzPqWr3PO9pQVRNIGsoW5d6mXm5xpFD3ol0oQzbr82eonfGMxWboFKDQ260cOCk5rgnvdhkiP0G6WoZF9pLBXIn5FPJs6QtoaRozvg96p4SOV38D2KzwQZdSQDBS6V2fnCeWe9S34WSwUdaZH8xh1Urrbl3VQ63w+XuXgbxKNmbiteii3ytuwOoMq3Gz/O4SobU3CkNa6mUTMwOErcRvslR4w5DXBm337evPKdyGfkxt2L3eF4+yrOW2c0UqiVJ0Nys89Cb3OMLTIeWrpiDOaIGdJ7NlXSrPtZK85P7rYMjPdzkdj+lKZWo1vRYxz3mgUuEubEv7y51e2FjfVkbH4uOXPao6rI6z3zhjjvv6IQcCQMRbpvnRfNq20DjAxQPyX69sbHH86UX+pfuOtSpjMt8depAPZ3eQKQbTx2lJiP4r/BztQk9qQ1hOk+kpuWraP1sXHxlHOqMo/VnzbjiOkDryEU4joXUSYnzAQ0leCl8h1ayEws30gRdrOdoz8yL1NysQ3hVLmMKT+1MqJJbok29arMIHjys04io821XCl/aceQqHLoljvvDEX0TWVfP90g+CHYgHvP3HgSWcTym49K5dJhJ0b6KGF/+MRyvv+nhjFYbTsQXWCETfVXXmup5FJktFKVT0nTMlhydtkeq9SaH3KlQ1cl6gLB7uC4XlPDSPelyz2lQTJLwK/KiPTlyJPDUeyuJx4mGnVBXn1FfLIzdMCsGsJBfUQzLyDDYGfcYjUZvRjDxwJkup+zA4QNapqJ4euzQopAWuc8Ke2Jv8vnuJnnoIPG0Tjj4iv9smKtEcGAMYGSRuRefyXVCi5MJkW45oFBtRkK3YFI1cVWx7Yd4InP2Psv0h9CURFtOmz2n6TIv+D36rOvs41ivwLTRkjMVB5jqA5ewVvlHOvmIMHpStXDtfJ2hSD96gv35t2RrtNtLJnX/xMTaeMx6kvHWz0+x/DxnNsbKF3Uaq8t/0qZW64VxDAbDoJ6F3h01KzvvZxtnaIEeg/OWXOXOOUnVs+8+JkFWnxJ0Q61OfE8r3vqEObS+BKziaBoI5IpR4s8cTD+Nb0fVnQO4u57ORj/PZrfZGv3KiFF5ohlGKQbw+/W76WR2fegWY+HdeHb18+j99c317Hr06d3frq9m3azf44mj8+t73L5uDsTUMZhCA2qqaUQA+Xr4unTDtaiEwThT46nkZmHYtpqFOqxphZWn5eUuEh5+vpvscESyrxxLGlfrhkZVXRaLuRN2KnSxRit5xNEsN+tD8ANzRieYAuyuhCozvA6B9soIbO6zNikCG84La3t7IVuPLnN9UywvllgqbqoeW1gnFOgDwK/lYXEQad0YfkBL2W6TjW9oTY8bmf+LUionv3VL9pi2FBGtQm5Yh+quoIG9XZHwZGiUnUWSO/2v/VArNSyUXK584ww6pDXfOcjRupySwoceDfWF1RmzptDiT4PPfEOBXU7BupGab01hHxuzXKC1p47QOyVO0M27tE6qwh6vU8O7Pmx5zFTYE6F9dmiHY1rpYImYBuFOXriWA3aT92U7sYo8Tw02icREPBZyVubEmQ0x8HXI1t+GUgzfhrnuShvxq0ct6oQLJu97GnByqZkvLJ7nrbOKfCmnAUzl8reAlr78OGi/E9PMGHecxIvzytidzFwhWy8ZHNtlRE7MTQ1VEA5u2BYtXEynN2/Klmg99ItL42X16gqp/7SLNbrQ02moWQ7HMmc7ra7eF9lZML3tNi786udgq3Fse/eeaMVuAH8v0G6nMfWm+/6gv8tc/CK3OCTdQEEp3puXb22wqrjoaWVRTQ2VapkalfS1Y+IH9tp8Z7GmmWXahcOPqGjT8mWWi9nN9E3lzRqalvqW+wd9jTmshTKbp3coWkM+T+1R/HZ7BbTUs3oTZxEiIflASG7M0pVLhBc9t6ag3U6vdoTxpTR6GV/3KOUrHbytHqC0hEpGBrxw3qz7nujRlROMY3Zn1mFwrxrDLE8nyy3oa7V7tItzdIjrPoFGvzH2vl4rYItzeWEGxbLFQvJ0nm2sONx3PUu7tRx17Jq7T/gGML66ur6dkee6u+4vlpVZHirmXoxUmeWSPGkq5ZJwy+0dwKdfBvDx0/vxbBxC7S+TW/ret+3OM33WXS+XCKL9ri3ZF2jFoMzNKtrShdZi8HpbU/TMBd37zFnOhOgOGC/p1eWMwv9Q4QMquDBWLqVm6k3Z22wfqSd2+hEK5/8UhIKKQR1DdwNm6S4O4nzI+Rk1Jkzekh1W/yeAk3oPV8w1nt7t1vjjAudkwfM8Wyi2PLFnmUu/Zu4+FWtV4DBKmQ15nNnVLYRlL+HtT9N/fhx8/z/0z3B89cvg+58+TD4Ofvzpbjrrhny+AcsotUuY3D78OKD//neo4a4/jEev/h0AAP///REWRQ==" } From ecf3e6a94fb4f66ff913b5283f10092bb6132220 Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Tue, 10 Mar 2020 11:32:00 -0600 Subject: [PATCH 5/6] add process.name and process.pid into ec2 fileset --- filebeat/docs/fields.asciidoc | 10 ------- filebeat/docs/modules/aws.asciidoc | 2 +- .../filebeat/module/aws/_meta/docs.asciidoc | 2 +- .../module/aws/cloudwatch/ingest/pipeline.yml | 9 +++--- .../test/cloudwatch_ec2.log-expected.json | 24 ++++++++-------- .../filebeat/module/aws/ec2/_meta/fields.yml | 4 --- .../module/aws/ec2/ingest/pipeline.yml | 8 +++--- .../module/aws/ec2/test/ec2.log-expected.json | 28 +++++++++++-------- x-pack/filebeat/module/aws/fields.go | 2 +- 9 files changed, 40 insertions(+), 49 deletions(-) diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc index c7c57808e61a..d8493968f5ce 100644 --- a/filebeat/docs/fields.asciidoc +++ b/filebeat/docs/fields.asciidoc @@ -1332,16 +1332,6 @@ Fields for AWS EC2 logs in CloudWatch. The internet address of the requester. -type: keyword - --- - -*`aws.ec2.program_name`*:: -+ --- -The program name of the log entry. - - type: keyword -- diff --git a/filebeat/docs/modules/aws.asciidoc b/filebeat/docs/modules/aws.asciidoc index 2cdfb6c60377..e266726bdd4a 100644 --- a/filebeat/docs/modules/aws.asciidoc +++ b/filebeat/docs/modules/aws.asciidoc @@ -127,7 +127,7 @@ on, it only reads the CloudTrail logs. === cloudwatch fileset Users can use Amazon CloudWatch Logs to monitor, store, and access log files -from different sources. Export logs from log groups to Amazon S3 bucket which +from different sources. Export logs from log groups to an Amazon S3 bucket which has SQS notification setup already. This fileset will parse these logs into `timestamp` and `message` field. diff --git a/x-pack/filebeat/module/aws/_meta/docs.asciidoc b/x-pack/filebeat/module/aws/_meta/docs.asciidoc index a352ff797fd5..983d8174d85e 100644 --- a/x-pack/filebeat/module/aws/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/aws/_meta/docs.asciidoc @@ -122,7 +122,7 @@ on, it only reads the CloudTrail logs. === cloudwatch fileset Users can use Amazon CloudWatch Logs to monitor, store, and access log files -from different sources. Export logs from log groups to Amazon S3 bucket which +from different sources. Export logs from log groups to an Amazon S3 bucket which has SQS notification setup already. This fileset will parse these logs into `timestamp` and `message` field. diff --git a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml index 7503193ce8a0..d1f65f3ba85d 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/cloudwatch/ingest/pipeline.yml @@ -4,15 +4,16 @@ processors: - grok: field: message patterns: - - >- - %{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message} + - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{GREEDYDATA:message}" + - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{GREEDYDATA:message}" - date: - field: "_tmp_.timestamp" + field: '_tmp.timestamp' target_field: "@timestamp" ignore_failure: true formats: - - "dd/MMM/yyyy:H:m:s Z" + - 'ISO8601' + - remove: field: - _tmp diff --git a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json index 90a9c5340436..11d33c51e0b2 100644 --- a/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json +++ b/x-pack/filebeat/module/aws/cloudwatch/test/cloudwatch_ec2.log-expected.json @@ -1,62 +1,62 @@ [ { - "@timestamp": "2020-03-03T15:08:08.438Z", + "@timestamp": "2020-02-20T07:01:01.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 0, - "message": "Feb 20 07:01:01 ip-172-31-81-156 systemd: Stopping User Slice of root.", + "message": "ip-172-31-81-156 systemd: Stopping User Slice of root.", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:08:08.446Z", + "@timestamp": "2020-02-20T07:02:18.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 96, - "message": "Feb 20 07:02:18 ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.", + "message": "ip-172-31-81-156 dhclient[3000]: XMT: Solicit on eth0, interval 125240ms.", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:08:08.453Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 211, - "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "message": "ip-172-31-81-156 dhclient[2898]: DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:08:08.453Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 345, - "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "message": "ip-172-31-81-156 dhclient[2898]: DHCPACK from 172.31.80.1 (xid=0x4575af22)", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:08:08.453Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 461, - "message": "Feb 20 07:02:37 ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.", + "message": "ip-172-31-81-156 dhclient[2898]: bound to 172.31.81.156 -- renewal in 1599 seconds.", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:08:08.453Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "event.dataset": "aws.cloudwatch", "event.module": "aws", "fileset.name": "cloudwatch", "input.type": "log", "log.offset": 586, - "message": "Feb 20 07:02:37 ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "message": "ip-172-31-81-156 ec2net: [get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", "service.type": "aws" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/ec2/_meta/fields.yml b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml index 4c7a71c16aa9..f6c21a4d7b6e 100644 --- a/x-pack/filebeat/module/aws/ec2/_meta/fields.yml +++ b/x-pack/filebeat/module/aws/ec2/_meta/fields.yml @@ -9,7 +9,3 @@ type: keyword description: > The internet address of the requester. - - name: program_name - type: keyword - description: > - The program name of the log entry. diff --git a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml index 947edfcac3af..0ada24c6f77d 100644 --- a/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml +++ b/x-pack/filebeat/module/aws/ec2/ingest/pipeline.yml @@ -4,15 +4,15 @@ processors: - grok: field: message patterns: - - >- - %{TIMESTAMP_ISO8601:_tmp.timestamp} %{MONTH:_tmp.month} %{MONTHDAY:_tmp.day} %{TIME:_tmp.time} %{IPORHOST:aws.ec2.ip_address} %{NOTSPACE:aws.ec2.program_name}: %{GREEDYDATA:message} + - "%{TIMESTAMP_ISO8601:_tmp.timestamp} %{SYSLOGTIMESTAMP:_tmp.syslog_timestamp} %{IPORHOST:aws.ec2.ip_address} %{DATA:process.name}(?:\\[%{POSINT:process.pid}\\])?: %{GREEDYDATA:message}" - date: - field: "_tmp_.timestamp" + field: '_tmp.timestamp' target_field: "@timestamp" ignore_failure: true formats: - - "dd/MMM/yyyy:H:m:s Z" + - 'ISO8601' + - remove: field: - _tmp diff --git a/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json b/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json index 7c38e9da696c..c2635e6a802b 100644 --- a/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json +++ b/x-pack/filebeat/module/aws/ec2/test/ec2.log-expected.json @@ -1,74 +1,78 @@ [ { - "@timestamp": "2020-03-03T15:38:00.278Z", + "@timestamp": "2020-02-20T07:01:01.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "systemd", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 0, "message": "Stopping User Slice of root.", + "process.name": "systemd", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:38:00.285Z", + "@timestamp": "2020-02-20T07:02:18.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "dhclient[3000]", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 96, "message": "XMT: Solicit on eth0, interval 125240ms.", + "process.name": "dhclient", + "process.pid": "3000", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:38:00.291Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "dhclient[2898]", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 211, "message": "DHCPREQUEST on eth0 to 172.31.80.1 port 67 (xid=0x4575af22)", + "process.name": "dhclient", + "process.pid": "2898", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:38:00.291Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "dhclient[2898]", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 345, "message": "DHCPACK from 172.31.80.1 (xid=0x4575af22)", + "process.name": "dhclient", + "process.pid": "2898", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:38:00.292Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "dhclient[2898]", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 461, "message": "bound to 172.31.81.156 -- renewal in 1599 seconds.", + "process.name": "dhclient", + "process.pid": "2898", "service.type": "aws" }, { - "@timestamp": "2020-03-03T15:38:00.292Z", + "@timestamp": "2020-02-20T07:02:37.000Z", "aws.ec2.ip_address": "ip-172-31-81-156", - "aws.ec2.program_name": "ec2net", "event.dataset": "aws.ec2", "event.module": "aws", "fileset.name": "ec2", "input.type": "log", "log.offset": 586, "message": "[get_meta] Trying to get http://169.254.169.254/latest/meta-data/network/interfaces/macs/12:e2:a9:95:8b:97/local-ipv4s", + "process.name": "ec2net", "service.type": "aws" } ] \ No newline at end of file diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index ec41b0bc81bf..61defc5b5f3f 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1JV/kYf/vX46YCNhwzSN3g4akddZlnTAiL7uVupDNBltqj1eghUa+DSVAt7KkBc2uWlq2zlqM+GlCivNNBCJ197e22rZmo5k/f2iV72c7dvHvW6cfphbIrDCZgzhTTvG93jqo8OgE0O+k7AIKQHt7CDf34Lv3Y4y09s0v0WdieUTtBOxLi+O5jiTAuFPWgPuN7tF1AqSi2U6cjcQXtSZRbaQQCN1oj78+5c2u84WY/zh1vZYFq955erLzPqWr3PO9pQVRNIGsoW5d6mXm5xpFD3ol0oQzbr82eonfGMxWboFKDQ260cOCk5rgnvdhkiP0G6WoZF9pLBXIn5FPJs6QtoaRozvg96p4SOV38D2KzwQZdSQDBS6V2fnCeWe9S34WSwUdaZH8xh1Urrbl3VQ63w+XuXgbxKNmbiteii3ytuwOoMq3Gz/O4SobU3CkNa6mUTMwOErcRvslR4w5DXBm337evPKdyGfkxt2L3eF4+yrOW2c0UqiVJ0Nys89Cb3OMLTIeWrpiDOaIGdJ7NlXSrPtZK85P7rYMjPdzkdj+lKZWo1vRYxz3mgUuEubEv7y51e2FjfVkbH4uOXPao6rI6z3zhjjvv6IQcCQMRbpvnRfNq20DjAxQPyX69sbHH86UX+pfuOtSpjMt8depAPZ3eQKQbTx2lJiP4r/BztQk9qQ1hOk+kpuWraP1sXHxlHOqMo/VnzbjiOkDryEU4joXUSYnzAQ0leCl8h1ayEws30gRdrOdoz8yL1NysQ3hVLmMKT+1MqJJbok29arMIHjys04io821XCl/aceQqHLoljvvDEX0TWVfP90g+CHYgHvP3HgSWcTym49K5dJhJ0b6KGF/+MRyvv+nhjFYbTsQXWCETfVXXmup5FJktFKVT0nTMlhydtkeq9SaH3KlQ1cl6gLB7uC4XlPDSPelyz2lQTJLwK/KiPTlyJPDUeyuJx4mGnVBXn1FfLIzdMCsGsJBfUQzLyDDYGfcYjUZvRjDxwJkup+zA4QNapqJ4euzQopAWuc8Ke2Jv8vnuJnnoIPG0Tjj4iv9smKtEcGAMYGSRuRefyXVCi5MJkW45oFBtRkK3YFI1cVWx7Yd4InP2Psv0h9CURFtOmz2n6TIv+D36rOvs41ivwLTRkjMVB5jqA5ewVvlHOvmIMHpStXDtfJ2hSD96gv35t2RrtNtLJnX/xMTaeMx6kvHWz0+x/DxnNsbKF3Uaq8t/0qZW64VxDAbDoJ6F3h01KzvvZxtnaIEeg/OWXOXOOUnVs+8+JkFWnxJ0Q61OfE8r3vqEObS+BKziaBoI5IpR4s8cTD+Nb0fVnQO4u57ORj/PZrfZGv3KiFF5ohlGKQbw+/W76WR2fegWY+HdeHb18+j99c317Hr06d3frq9m3azf44mj8+t73L5uDsTUMZhCA2qqaUQA+Xr4unTDtaiEwThT46nkZmHYtpqFOqxphZWn5eUuEh5+vpvscESyrxxLGlfrhkZVXRaLuRN2KnSxRit5xNEsN+tD8ANzRieYAuyuhCozvA6B9soIbO6zNikCG84La3t7IVuPLnN9UywvllgqbqoeW1gnFOgDwK/lYXEQad0YfkBL2W6TjW9oTY8bmf+LUionv3VL9pi2FBGtQm5Yh+quoIG9XZHwZGiUnUWSO/2v/VArNSyUXK584ww6pDXfOcjRupySwoceDfWF1RmzptDiT4PPfEOBXU7BupGab01hHxuzXKC1p47QOyVO0M27tE6qwh6vU8O7Pmx5zFTYE6F9dmiHY1rpYImYBuFOXriWA3aT92U7sYo8Tw02icREPBZyVubEmQ0x8HXI1t+GUgzfhrnuShvxq0ct6oQLJu97GnByqZkvLJ7nrbOKfCmnAUzl8reAlr78OGi/E9PMGHecxIvzytidzFwhWy8ZHNtlRE7MTQ1VEA5u2BYtXEynN2/Klmg99ItL42X16gqp/7SLNbrQ02moWQ7HMmc7ra7eF9lZML3tNi786udgq3Fse/eeaMVuAH8v0G6nMfWm+/6gv8tc/CK3OCTdQEEp3puXb22wqrjoaWVRTQ2VapkalfS1Y+IH9tp8Z7GmmWXahcOPqGjT8mWWi9nN9E3lzRqalvqW+wd9jTmshTKbp3coWkM+T+1R/HZ7BbTUs3oTZxEiIflASG7M0pVLhBc9t6ag3U6vdoTxpTR6GV/3KOUrHbytHqC0hEpGBrxw3qz7nujRlROMY3Zn1mFwrxrDLE8nyy3oa7V7tItzdIjrPoFGvzH2vl4rYItzeWEGxbLFQvJ0nm2sONx3PUu7tRx17Jq7T/gGML66ur6dkee6u+4vlpVZHirmXoxUmeWSPGkq5ZJwy+0dwKdfBvDx0/vxbBxC7S+TW/ret+3OM33WXS+XCKL9ri3ZF2jFoMzNKtrShdZi8HpbU/TMBd37zFnOhOgOGC/p1eWMwv9Q4QMquDBWLqVm6k3Z22wfqSd2+hEK5/8UhIKKQR1DdwNm6S4O4nzI+Rk1Jkzekh1W/yeAk3oPV8w1nt7t1vjjAudkwfM8Wyi2PLFnmUu/Zu4+FWtV4DBKmQ15nNnVLYRlL+HtT9N/fhx8/z/0z3B89cvg+58+TD4Ofvzpbjrrhny+AcsotUuY3D78OKD//neo4a4/jEev/h0AAP///REWRQ==" + return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1JV/kYf/vX46YCNhwzSN3g4akddZlnTAiL7uVupDNBltqj1eghUa+DSVAt7KkBc2tC37HlqI8GlCjvdBBCZ197uz0MJu+xQGVaSf0zgEhxAEalYWr+dA1bspcp0M27Zx3CnH5vdveECZgzxTTvU5KjCqBOAM2G/g6AIKSHt3BDP75LP/Y4bc/sEn0WtmfUzhOPhDi++1gijAtFPaiPGh/tWlBGjO0M7khcQXsS5VY2g8CN1sj7U//cGm+42Q+3xxt7oNq9pxcr73MwFjzPezohVS8qGqvUy8zLNY4c8k6kC2XYfon4FL0znqnYi5UaHHKjhQMnNcc96cVeR2x7SFfLuNBeKpA7mQdVXkvaEsrN5ozfo+6p1NPF/yA2G2zQlQQQvFRq5wfnmfUutX8oJ32kU/cXc1h19Jp7V6WSO1zu7mUQj5K9FUEtusjXujuOvyxUNbhKhtTcKQ1rqZRMzA4StxG+yVHjDkNcGbd/fFB5TuUy8mNuxe7xvHyURz6zmylUS5KguVnnoUW6xxeYDi1dMQdzRA3oPJsr6VZ9rJXmJ/c7GEd6uMntfmZVKlGt6bGcfMwDlwhzY1/e5Or2wsb6skQ/Fh257FHV7HWe+cIdd+zSCTkSBiLcNs+L5tW2gcYHKB6S/XpjY6vpSy/0L93lsFMZl/nq1IF6Or2BSDcefkpNRvBf4edqE3pSG8J0nkhNy1fR+tm4+Mo41BlH68+accV1gNaRi3AqDKmhE8cUGkrwUvgOrWQnFm6kCbpYz9GemRepuVmH8KpcxhSe2plQQblEm1rmZhE8eFinEVHn264UvrTjyFU4+0sc94cj+iayrtbzkXwQ7EA85u89CCzjeEzjp3PpMBqjfRUxvvxjOF5/08MZrTaciC+wQib6qq4183yFIrOFonRKmo4Rl6PT9ki13uSQOxWqOuAPEHbP+OWCEl66J13uOZSKSRJ+RV60B1iOBJ5agCXxOFixE+rqo/KLhbEbZsUAFvIrimEZGQY7Uyej0ejNCCYeONPlsB84fEDLVBRPjx1aFNIi91lhT+xNPt/dJA8dJJ7WCedv8Z8Nc5UIDkwjjCwy9+KjwU5ocUAi0i3nJKrNSOgWTKomriq2/RAPhs7eZ5n+EHqjaMuht+c0XeYFv0efdR3BHOsVmDZacqbiHFV97hPWKv9IBzARRk+qFq6drzMU6UdPsD+Gl2yNdnvJpO4f3Fgbj1lPMt76+SmWn+fMxlj5ooZndflP2tRqvTAVwmAY1LPQuxNv5QHA2aYqWqDH4LwlV7lzXFMdHXSf1iCrDyu6oVYHz6cVb33QHVpfAlZxQg4EcsUo8WcOpp/Gt6PqzgHcXU9no59ns9tsjX5lxKg8WA0THQP4/frddDK7PnSLsfBuPLv6efT++uZ6dj369O5v11ezbtbv8cTR+fU9bl8353LqGEyhATXVNCKAfD18XbrhWlTCYBzt8VRyszDzW41kHda0wsrT8nIXCQ8/3012OCLZV44lTc11Q6OqLovF3Ak7FbpYo5U84miWm/VZ/IFxpxMMI3ZXQpUZXodAe2UENvdZmxSBDeeFtb29kK1Hl7m+YZoXSywVN1WPLawTCvQB4NfyzDqItG4MP6ClbLfJxje0pseNzP9FKZWT37ole0xbiohWITesQ3VX0MDerkh4MjTKziLJnf7XfqiVGhZKLle+cRQe0prvHORoXU5J4UOPhvrC6oxZU2jxp8FnvqHALqdg3UjNt6awj017LtDaU0fonRIn6OZdWidVYY/XqeGVI7Y8ZjjtidA+O7TDMa10sERM83gnL1zLOb/J+7KdWEWepwabRGIiHgs5K3PizIYY+Dpk629DKYZvw3h5pY341aMWdcIFk/c9DTi51MwXFs/z8ltFvpTTAKZy+VtAS19+HLRfzWlmjDtO4sV5ZexOZq6QrXcdju0yIifmpoYqCAc3bIsWLqbTmzdlS7SePcal8bJ6g4bUf9rFGl3o6TTULIdjmbOdVlevrewsmF66Gxd+9XOw1Tg9vntPtGI3gL8XaLfTmHrTfX/Q32UufpFbHJJuoKAU783LtzZYVVz0tLKohpdKtUyNSvraMXgEe22+s1jTzDLtwuFHVLRp+U7Nxexm+qbyZg1NS33L/YO+xjjYQpnN0zsUrVmjp/Yofru9AlrqWb2JswiRkHwgJDdm6colwvumW1PQbqc3TMIUVZoAjW+dlPKVDt5WD1BaQiUjA144b9Z9T/ToygmmQrsz6zA/WE2DlqeT5Rb0tdo92sU5OsR1n0Cj3xh7X68VsMXxwDCDYtliIXk6zzZWHO67nqXdWk5cdo3/J3wDGF9dXd/OyHPdXfcXy8osDxVzL0aqzHJJnjSVckm45fYO4NMvA/j46f14Ng6h9pfJLX3v23bnmT7rrpdLBNF+15bsC7RiUOZmFW3pQmsxeL2tKXrmgu595ixnQnQHjJf06nJG4X+o8AEVXBgrl1Iz9absbbaP1BM7/QiF838KQkHFoI6huwGzdBcHcT7k/IwaEwaAyQ6r/yHBSb2HK+YaT+92a/xxgXOy4HmeLRRbntizzKVfM3efirUqcBilzIY8zuzqFsKyl/D2p+k/Pw6+/x/6Zzi++mXw/U8fJh8HP/50N511Qz7fgGWU2iVMbh9+HNB//zvUcNcfxqNX/w4AAP//gcM4yw==" } From 5607f2418e2dc8719eb60e46286338f848e5834e Mon Sep 17 00:00:00 2001 From: kaiyan-sheng Date: Wed, 11 Mar 2020 13:38:25 -0600 Subject: [PATCH 6/6] rerun mage fmt update --- x-pack/filebeat/module/aws/fields.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/filebeat/module/aws/fields.go b/x-pack/filebeat/module/aws/fields.go index 61defc5b5f3f..f3d84e531194 100644 --- a/x-pack/filebeat/module/aws/fields.go +++ b/x-pack/filebeat/module/aws/fields.go @@ -19,5 +19,5 @@ func init() { // AssetAws returns asset data. // This is the base64 encoded gzipped contents of module/aws. func AssetAws() string { - return "eJzMXN1zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9iijPT+HWjv9GTIdzj9hLYxr0C8NIrvITx79NXABYVMoeXMEfPXgEIdNzK3EujL+H/XgEA/GpEoRAWxsKKaaGkXoIySwcLa9ZEZvQKYCFRCXcZHhiCZmssl6OP3+Z4CUtrijz90rEOfT4EMhXlsM4oXW0u0VyGK1MIb5lU1aWuFemzz235EbhghfJZWOISFkw53LncCbYJ2NiA94qwzAjLDvQu+E0W8AG1zx7QOmn0zh0lJ/e43Rgr9q4dAEaf2QqbiBJ9MAvwKySAcWFCv2Z+1AmtcGgzKVB76bed0PaF3AY27ERGlCeJMKDCNUHhRnsmtQOBnknlgM1N4QNeWg3MokVrMv4VSoDgV8zDmgkMj1j8o0DnB8C0gM1K8hVwi+Fephxs0GKLXOFQjGCyAI/r3Fhmt61nwj2DsEKJ263MxsHKbOjXFs0WATMnLlGM9m7tUpLmbpAMWhcP60h7OzpuiDuSJBwY69nyhnXbfU19PpK2YpRQxmv2zWi4Q2cKyxE+sjXCxfju45sSYG6l5jJnam/POVNqX6wN1Jyjc9k9bjPZhe9U+OM6RAgm7yPCDXNBccAbcHKpmxraD9ihI6PNyDDwq++F3GWFTwU8WTSxBKBBnBvpVw0zcMgL26USsKviZG6VYQTWc2sepEAHUkdfQ26otuzEYyfdSnTcIvMogqv1K+OwuWTHo32m1BTuesEyVvgVUeFEvfPux7XiqYKGpB0PTBUI0oG39G8SvzE+OEUwNji18H1DrPYS6/RMSUT1hjLlTJDhDq9xe1m32Onz64cxCHyQHP8XjF+h3UiHgxgd2wrblGvYK9JawXwf+CjTAzc8R6BEJjh5L9cImxVG62rrblti0rmi7YhrXqR+MPcosnmX3p/KXdBSpWOjPMKhJbn3hTNX8BWwLq2H0nNeX72FceENTDkLKVvKUK4Vc15yeIdMO8/UfXfYR2uNzbgR+7vz9HSkO+o3uQuLVOqflNWiL6x2wYfQ9UP41ugcW54S4uQwmBj0G0TKTTsANdHKcmbZGj3a/X07VqQ14QEJk+ntILlN0m9HXjh68r54UyN1udEOs+S8Tw20pF8FB3JUjNMzrlT1ewS+YnqJDi6iyx+0U7Sc7H1AKi1QIZl+JPKmmy8mhKTrTGUh4RVsJ/s/lrVxRZ78EGsEtpheVyFMG0+b5ZPStAiVWmdsJarDKtVKYY7doRiYAuCYAC4kuh3FCbeVBjxHqZftbJcphQKWqNEyH56XLpLuMeZQ/nTktkdZ8i7+MrdtbEkJUDT2yiI3VvSoUS6PrtEexTm+nVSFGnPOcFnH6nB948a5vGJKtSgFDmbE5wFZr5lmy2B90RZOaQfwzhiFTPeo0WaFlEU0pC0d7BsiNBDGu/pMgInMaNVdkx69FTVW6cDkpCe0IwQ4LD2kpesLvf40VC/dfvQldfMYlHTBf1S0U6mBAqSuRfvcwvJ81VxVwo3vPrYzlkZdZgrd4dFOBWMcF6BqzJQxPgEzG432vCX3I6IJzqmyFFvuZKyVQqvu8pJt3DD53WFAdknBZkiPhr97NJDLXJKx9wr4GIO5w9wiJRjRd7FaxsH2LXKUD8G/SnfImBNf0SNlqfVzPh9b5Z+03ACk5qoQlCNvCLW3crlEG8NCt5ONpUbUoUL1MbViFkXi6aRi///Pk/eN6DXfNrt83kCh5R8Fqm2pUs3r3QyllmsQDpUilKXHnCp5cRfzSG9AyMUCLf0RO8i7n6QCrlskDznPUIvcyFOLZG+Hf7u9gnIhsqbY+0s5TCr8QlkV2G7HIHreG2A61L3NoqUqvspCa/pDzetOS3rDPF+92mfvr2lJ/05YHm1JV/kYf/vX46YCNhwzSN3g4akddZlnTAiL7uVupDNBltqj1eghUa+DSVAt7KkBc2tC37HlqI8GlCjvdBBCZ197uz0MJu+xQGVaSf0zgEhxAEalYWr+dA1bspcp0M27Zx3CnH5vdveECZgzxTTvU5KjCqBOAM2G/g6AIKSHt3BDP75LP/Y4bc/sEn0WtmfUzhOPhDi++1gijAtFPaiPGh/tWlBGjO0M7khcQXsS5VY2g8CN1sj7U//cGm+42Q+3xxt7oNq9pxcr73MwFjzPezohVS8qGqvUy8zLNY4c8k6kC2XYfon4FL0znqnYi5UaHHKjhQMnNcc96cVeR2x7SFfLuNBeKpA7mQdVXkvaEsrN5ozfo+6p1NPF/yA2G2zQlQQQvFRq5wfnmfUutX8oJ32kU/cXc1h19Jp7V6WSO1zu7mUQj5K9FUEtusjXujuOvyxUNbhKhtTcKQ1rqZRMzA4StxG+yVHjDkNcGbd/fFB5TuUy8mNuxe7xvHyURz6zmylUS5KguVnnoUW6xxeYDi1dMQdzRA3oPJsr6VZ9rJXmJ/c7GEd6uMntfmZVKlGt6bGcfMwDlwhzY1/e5Or2wsb6skQ/Fh257FHV7HWe+cIdd+zSCTkSBiLcNs+L5tW2gcYHKB6S/XpjY6vpSy/0L93lsFMZl/nq1IF6Or2BSDcefkpNRvBf4edqE3pSG8J0nkhNy1fR+tm4+Mo41BlH68+accV1gNaRi3AqDKmhE8cUGkrwUvgOrWQnFm6kCbpYz9GemRepuVmH8KpcxhSe2plQQblEm1rmZhE8eFinEVHn264UvrTjyFU4+0sc94cj+iayrtbzkXwQ7EA85u89CCzjeEzjp3PpMBqjfRUxvvxjOF5/08MZrTaciC+wQib6qq4183yFIrOFonRKmo4Rl6PT9ki13uSQOxWqOuAPEHbP+OWCEl66J13uOZSKSRJ+RV60B1iOBJ5agCXxOFixE+rqo/KLhbEbZsUAFvIrimEZGQY7Uyej0ejNCCYeONPlsB84fEDLVBRPjx1aFNIi91lhT+xNPt/dJA8dJJ7WCedv8Z8Nc5UIDkwjjCwy9+KjwU5ocUAi0i3nJKrNSOgWTKomriq2/RAPhs7eZ5n+EHqjaMuht+c0XeYFv0efdR3BHOsVmDZacqbiHFV97hPWKv9IBzARRk+qFq6drzMU6UdPsD+Gl2yNdnvJpO4f3Fgbj1lPMt76+SmWn+fMxlj5ooZndflP2tRqvTAVwmAY1LPQuxNv5QHA2aYqWqDH4LwlV7lzXFMdHXSf1iCrDyu6oVYHz6cVb33QHVpfAlZxQg4EcsUo8WcOpp/Gt6PqzgHcXU9no59ns9tsjX5lxKg8WA0THQP4/frddDK7PnSLsfBuPLv6efT++uZ6dj369O5v11ezbtbv8cTR+fU9bl8353LqGEyhATXVNCKAfD18XbrhWlTCYBzt8VRyszDzW41kHda0wsrT8nIXCQ8/3012OCLZV44lTc11Q6OqLovF3Ak7FbpYo5U84miWm/VZ/IFxpxMMI3ZXQpUZXodAe2UENvdZmxSBDeeFtb29kK1Hl7m+YZoXSywVN1WPLawTCvQB4NfyzDqItG4MP6ClbLfJxje0pseNzP9FKZWT37ole0xbiohWITesQ3VX0MDerkh4MjTKziLJnf7XfqiVGhZKLle+cRQe0prvHORoXU5J4UOPhvrC6oxZU2jxp8FnvqHALqdg3UjNt6awj017LtDaU0fonRIn6OZdWidVYY/XqeGVI7Y8ZjjtidA+O7TDMa10sERM83gnL1zLOb/J+7KdWEWepwabRGIiHgs5K3PizIYY+Dpk629DKYZvw3h5pY341aMWdcIFk/c9DTi51MwXFs/z8ltFvpTTAKZy+VtAS19+HLRfzWlmjDtO4sV5ZexOZq6QrXcdju0yIifmpoYqCAc3bIsWLqbTmzdlS7SePcal8bJ6g4bUf9rFGl3o6TTULIdjmbOdVlevrewsmF66Gxd+9XOw1Tg9vntPtGI3gL8XaLfTmHrTfX/Q32UufpFbHJJuoKAU783LtzZYVVz0tLKohpdKtUyNSvraMXgEe22+s1jTzDLtwuFHVLRp+U7Nxexm+qbyZg1NS33L/YO+xjjYQpnN0zsUrVmjp/Yofru9AlrqWb2JswiRkHwgJDdm6colwvumW1PQbqc3TMIUVZoAjW+dlPKVDt5WD1BaQiUjA144b9Z9T/ToygmmQrsz6zA/WE2DlqeT5Rb0tdo92sU5OsR1n0Cj3xh7X68VsMXxwDCDYtliIXk6zzZWHO67nqXdWk5cdo3/J3wDGF9dXd/OyHPdXfcXy8osDxVzL0aqzHJJnjSVckm45fYO4NMvA/j46f14Ng6h9pfJLX3v23bnmT7rrpdLBNF+15bsC7RiUOZmFW3pQmsxeL2tKXrmgu595ixnQnQHjJf06nJG4X+o8AEVXBgrl1Iz9absbbaP1BM7/QiF838KQkHFoI6huwGzdBcHcT7k/IwaEwaAyQ6r/yHBSb2HK+YaT+92a/xxgXOy4HmeLRRbntizzKVfM3efirUqcBilzIY8zuzqFsKyl/D2p+k/Pw6+/x/6Zzi++mXw/U8fJh8HP/50N511Qz7fgGWU2iVMbh9+HNB//zvUcNcfxqNX/w4AAP//gcM4yw==" + return "eJzMW91zGzeSf/df0eWXyFUkr+Kkrq50lauiZfnCjWJrRTrZfRqDQJPECgQmAEY0/ddvNYD5IGeG+iCZLB9sijPT+HWjv9EzhHvcXgLbuFcAXnqFlzD+ffoKwKJC5vAS5ujZKwCBjluZe2n0JfzfKwCAX40oFMLCWFgxLZTUS1Bm6WBhzZrIjF4BLCQq4S7DA0PQbI3lcvTx2xwvYWlNkadfOtahz4dApqIc1hmlq80lmstwZQrhLZOqutS1In32uS0/AhesUD4LS1zCgimHO5c7wTYBGxvwXhGWGWHZgd4Fv8kCPqD22QNaJ43euaPk5B63G2PF3rUDwOgzW2ETUaIPZgF+hQQwLkzo18yPOqEVDm0mBWov/bYT2r6Q28CGnciI8iQRBlS4JijcaM+kdiDQM6kcsLkpfMBLq4FZtGhNxr9CCRD8inlYM4HhEYt/FOj8AJgWsFlJvgJuMdzLlIMNWmyRKxyKEUwW4HGdG8vstvVMuGcQVihxu5XZOFiZDf3aotkiYObEJYrR3q1dStLcDZJB6+JhHWlvR8cNcUeShANjPVvesG67r6nPR9JWjBLKeM2+GQ136ExhOcJHtka4GN99fFMCzK3UXOZM7e05Z0rti7WBmnN0LrvHbSa78J0Kf1yHCMHkfUS4YS4oDngDTi51U0P7ATt0ZLQZGQZ+9b2Qu6zwqYAniyaWADSIcyP9qmEGDnlhu1QCdlWczK0yjMB6bs2DFOhA6uhryA3Vlp147KRbiY5bZB5FcLV+ZRw2l+x4tM+UmsJdL1jGCr8iKpyod979uFY8VdCQtOOBqQJBOvCW/k/iN8YHpwjGBqcWvm+I1V5inZ4piajeUKacCTLc4TVuL+sWO31+/TAGgQ+S4/+C8Su0G+lwEKNjW2Gbcg17RVormO8DH2V64IbnCJTIBCfv5Rphs8JoXW3dbUtMOle0HXHNi9QP5h5FNu/S+1O5C1qqdGyURzi0JPe+cOYKvgLWpfVQes7rq7cwLryBKWchZUsZyrVizksO75Bp55m67w77aK2xGTdif3eeno50R/0md2GRSv2Tslr0hdUu+BC6fgjfGp1jy1NCnBwGE4N+g0i5aQegJlpZzixbo0e7v2/HirQmPCBhMr0dJLdJ+u3IC0dP3hdvaqQuN9phlpz3qYGW9KvgQI6KcXrGlap+j8BXTC/RwUV0+YN2ipaTvQ9IpQUqJNOPRN5088WEkHSdqSwkvILtZP/HsjauyJMfYo3AFtPrKoRp42mzfFKaFqFS64ytRHVYpVopzLE7FANTABwTwIVEt6M44bbSgOco9bKd7TKlUMASNVrmw/PSRdI9xhzKn47c9ihL3sVf5raNLSkBisZeWeTGih41yuXRNdqjOMe3k6pQY84ZLutYHa5v3DiXV0ypFqXAwYz4PCDrNdNsGawv2sIp7QDeGaOQ6R412qyQsoiGtKWDfUOEBsJ4V58JMJEZrbpr0qO3osYqHZic9IR2hACHpYe0dH2h15+G6qXbj76kbh6Dki74j4p2KjVQgNS1aJ9bWJ6vmqtKuPHdx3bG0qjLTKE7PNqpYIzjAlSNmTLGJ2Bmo9Get+R+RDTBOVWWYsudjLVSaNVdXrKNGya/OwzILinYDOnR8HePBnKZSzL2XgEfYzB3mFukBCP6LlbLONi+RY7yIfhX6Q4Zc+IreqQstX7O52Or/JOWG4DUXBWCcuQNofZWLpdoY1jodrKx1Ig6VKg+plbMokg8nVTs//958r4RvebbZpfPGyi0/KNAtS1Vqnm9m6HUcg3CoVKEsvSYUyUv7mIe6Q0IuVigpT9iB3n3k1TAdYvkIecZapEbeWqR7O3wb7dXUC5E1hR7fymHSYVfKKsC2+0YRM97A0yHurdZtFTFV1loTX+oed1pSW+Y56tX++z9NS3p3wnLoy3pKh/jb/963FTAhmMGqRs8PLWjLvOMCWHRvdyNdCbIUnu0Gj0k6nUwCaqFtq0MqOZPl+aSvUxYN++edeDQChhHC6bZwVCGCZgzxTTHnqL4qGS/E0Czeb0DIAjp4S3c0I/v0o89Dsozu0Sfhe0ZtXOiIyGO7z6WCONCUQ/qY7VHK3TK/rCdrRyJK2hPotyK3AjcaI28P83NrfGGm/3QciSokmr3nl6svM+pUPY876n6q76LNZQgS73MvFzjyCHvRLpQhu2XQ0/RO+OZin1HqcEhN1o4cFJz3JNerOtjiS9dLeNCe6lA7kRZqjKWtCWUh8wZv0fdU5Wmi/9BbDbYoCsJIHip1M4PzjPrXWp1UP71SFfqL+aw6l41965Km3a43N3LIB4le7PfWnSRr3V3zFKm1Wt5HlfJkJo7pWEtlZKJ2UHiNsI3OWrcYYgr4/Zb5ZXnVC4jP+ZW7B7Py0d5vDG7mUK1JAmam3Ue2oF7fIHp0NIVczBH1IDOs7mSbtXHWml+cr9aP9LDTW73s4hSiWpNj6XTYx64RJgb+/KGTrcXNtaX5eix6Mhlj6rGpvPMF+64I4ZOyJEwEOG2eV40r7YNND5A8ZDs1xsb2ypfeqF/6S79nMq4zFenDtTT6Q1EuvGgT2oygv8KP1eb0JPaEKbzRGpavorWz8bFV8ahzjhaf9aMK64DtI5chBNQSM2LeCTfUIKXwndoJTuxcCNN0MV6jvbMvEjNzTqEV+UypvDUzoSKpyXa1B42i+DBwzqNiDrfdqXwpR1HrsI5V+K4PxzRN5F1tVmP5INgB+Ixf+9BYBnHY5ocnUuHMRDtq4jx5R/D8fqbHs5oteFEfIEVMtFXda2phEaR2UJROiVNxzjH0Wl7pFpvcsidClUdZgcIu+fZckEJL92TLvccwMQkCb8iL9rDGkcCT+2ukngcItgJdfWx8MXC2A2zYgAL+RXFsIwMg50Ji9Fo9GYEEw+c6XKwDRw+oGUqiqfHDi0KaZH7rLAn9iaf726Shw4ST+uEs6b434a5SgQHTt5HFpl78TFYJ7Q4DBDpljMB1WYkdAsmVRNXFdt+iIcgZ++zTH8IfUC05YDXc5ou84Lfo8+6jhuO9QpMGy05U3FmqD7jCGuVf6TDhgijJ1UL187XGYr0oyfYHzlLtka7vWRS9w8prI3HrCcZb/38FMvPc2ZjrHxycw/ajYY/bVOr9cIEBINhUM9C7053lc3us00QtECPwXlLrnLnaKJqk3efTCCrG/PdUKtD1tOKtz7UDa0vAas4DQYCuWKU+DMH00/j21F15wDurqez0c+z2W22Rr8yYlQeIobphQH8fv1uOpldH7rFWHg3nl39PHp/fXM9ux59eve366tZN+v3eOLo/Poet6+bMyh1DKbQgJpqGhFAvh6+Lt1wLSphMI6xeCq5WZhvrcaPDmtaYeVpebmLhIef7yY7HJHsK8eSJsS6oVFVl8Vi7oSdCl2s0UoecTTLzfrc+cBozwkG77orocoMr0OgvTICm/usTYrAhvPC2t5eyNajy1zf4MiLJZaKm6rHFtYJBfoA8Gt5PhtEWjeGH9BStttk4xta0+NG5v+ilMrJb92SPaYtRUSrkBvWoboraGBvVyQ8GRplZ5HkTv9rP9RKDQsllyvfOPYNac13DnK0Lqek8KFHQ31hdcasKbT40+Az31Bgl1OwbqTmW1PYxyYbF2jtqSP0TokTdPMurZOqsMfr1PB6DVseM4j1RGifHdrhmFY6WCKm2bOTF67lTNvkfdlOrCLPU4NNIjERj4WclTlxZkMMfB2y9behFMO3YZS60kb86lGLOuGCyfueBpxcauYLi+d50asiX8ppAFO5/C2gpS8/DtqvoTQzxh0n8eK8MnYnM1fI1lz/sV1G5MTc1FAF4eCGbdHCxXR686ZsidZztrg0XlZvi5D6T7tYows9nYaa5XAsc7bT6uoVjZ0F0wtm48Kvfg62Gield++JVuwG8PcC7XYaU2+67w/6u8zFL3KLQ9INFJTivXn51garioueVhbVoE6plqlRSV87hmxgr813FmuaWaZdOPyIijYt3x+5mN1M31TerKFpqW+5f9DXGH1aKLN5eoeiNVfz1B7Fb7dXQEs9qzdxFiESkg+E5MYsXblEeLdyawra7fQ2RZgYStOO8Q2LUr7SwdvqAUpLqGRkwAvnzbrviR5dOcEEZHdmHWblqsnH8nSy3IK+VrtHuzhHh7juE2j0G2Pv67UCtjgKF2ZQLFssJE/n2caKw33Xs7Rby+nCrlH3hG8A46ur69sZea676/5iWZnloWLuxUiVWS7Jk6ZSLgm33N4BfPplAB8/vR/PxiHU/jK5pe992+4802fd9XKJINrv2pJ9gVYMytysoi1daC0Gr7c1Rc9c0L3PnOVMiO6A8ZJeXc4o/A8VPqCCC2PlUmqm3pS9zfaRemKnH6Fw/k9BKKgY1DF0N2CW7uIgzoecn1FjwrAr2WH18v1JvYcr5hpP73Zr/HGBc7LgeZ4tFFue2LPMpV8zd5+KtSpwGKXMhjzO7OoWwrKX8Pan6T8/Dr7/H/pvOL76ZfD9Tx8mHwc//nQ3nXVDPt+AZZTaJUxuH34c0L//HWq46w/j0at/BwAA//+LQ/E/" }