Merge branch 'hopefuly-last-update-of-python3-branch' into HEAD

.ci/apm-beats-update.groovy

@@ -33,7 +33,7 @@ pipeline {
       options { skipDefaultCheckout() }
       steps {
         deleteDir()
-        gitCheckout(basedir: "beats")
+        gitCheckout(basedir: "beats", githubNotifyFirstTimeContributor: false)
         script {
           dir("beats"){
             env.GO_VERSION = readFile(".go-version").trim()
@@ -73,15 +73,9 @@ pipeline {
       steps {
         withGithubNotify(context: 'Check Apm Server Beats Update') {
           deleteDir()
-          gitCheckout(basedir: "${BASE_DIR}",
-            repo: "git@github.com:elastic/${REPO}.git",
-            branch: 'master',
-            credentialsId: 'credentials-id',
-            githubNotifyFirstTimeContributor: false,
-            depth: 1,
-            reference: "/var/lib/jenkins/.git-references/${REPO}.git"
-          )
           dir("${BASE_DIR}"){
+            git(credentialsId: 'f6c7695a-671e-4f4f-a331-acdce44ff9ba',
+              url:  "git@github.com:elastic/${REPO}.git")
             sh(label: 'Update Beats script', script: """
               export BEATS_VERSION=${env.GIT_BASE_COMMIT}
               git config --global --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pr/*"
@@ -99,9 +93,9 @@ pipeline {
       }
     }
   }
-  post {
-    cleanup {
-      notifyBuildResult()
-    }
-  }
+  // post {
+  //   cleanup {
+  //     notifyBuildResult()
+  //   }
+  // }
 }

.go-version

@@ -1 +1 @@
-1.13.6
+1.13.7

CHANGELOG.asciidoc

@@ -13,6 +13,7 @@ https://github.com/elastic/beats/compare/v7.5.0...v7.5.1[View commits]
 
 - Fix `proxy_url` option in Elasticsearch output. {pull}14950[14950]
 - Fix bug with potential concurrent reads and writes from event.Meta map by Kafka output. {issue}14542[14542] {pull}14568[14568]
+- Fix license detection, when a beats successfully connect to Elasticsearch the detected license will be show in the log at info level. {pull}15834[15834]
 
 *Filebeat*
 
@@ -186,6 +187,7 @@ processing events. (CVE-2019-17596) See https://www.elastic.co/community/securit
 
 - Fill `event.provider`. {pull}13937[13937]
 - Add support for user management events to the Security module. {pull}13530[13530]
+- Made the event parser more lenient w.r.t. invalid event log definition version numbers. {issue}15838[15838]
 
 ==== Deprecated
 

CHANGELOG.next.asciidoc

@@ -10,6 +10,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- The document id fields has been renamed from @metadata.id to @metadata._id {pull}15859[15859]
+
 
 *Auditbeat*
 
@@ -39,16 +41,25 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 *Affecting all Beats*
 
 - TLS or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
+- Fix panics that could result from invalid TLS certificates. This can affect Beats that connect over TLS, or Beats that accept connections over TLS and validate client certificates. {pull}14146[14146]
 - Fix panic in the Logstash output when trying to send events to closed connection. {pull}15568[15568]
 - Fix missing output in dockerlogbeat {pull}15719[15719]
+- Fix logging target settings being ignored when Beats are started via systemd or docker. {issue}12024[12024] {pull}15422[15442]
+- Do not load dashboards where not available. {pull}15802[15802]
+- Fix issue where TLS settings would be ignored when a forward proxy was in use. {pull}15516{15516}
+- Update replicaset group to apps/v1 {pull}15854[15802]
+- Fix issue where default go logger is not discarded when either * or stdout is selected. {issue}10251[10251] {pull}15708[15708]
 
 *Auditbeat*
 
 - system/socket: Fixed compatibility issue with kernel 5.x. {pull}15771[15771]
 
 *Filebeat*
 
+- Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
 - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
+- Fix typos in zeek notice fileset config file. {issue}15764[15764] {pull}15765[15765]
+- Improve `elasticsearch/audit` fileset to handle timestamps correctly. {pull}15942[15942]
 
 *Heartbeat*
 
@@ -59,6 +70,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Metricbeat*
 
+- Add dedot for tags in ec2 metricset and cloudwatch metricset. {issue}15843[15843] {pull}15844[15844]
+- Use RFC3339 format for timestamps collected using the SQL module. {pull}15847[15847]
+- Avoid parsing errors returned from prometheus endpoints. {pull}15712[15712]
+- Change lookup_fields from metricset.host to service.address {pull}15883[15883]
+- Add dedot for cloudwatch metric name. {issue}15916[15916] {pull}15917[15917]
 
 *Packetbeat*
 
@@ -73,12 +89,16 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Affecting all Beats*
 
+- Add document_id setting to decode_json_fields processor. {pull}15859[15859]
+
 
 *Auditbeat*
 
 
 *Filebeat*
 - Set event.outcome field based on googlecloud audit log output. {pull}15731[15731]
+- Add dashboard for AWS ELB fileset. {pull}15804[15804]
+
 
 *Heartbeat*
 
@@ -97,8 +117,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add DynamoDB AWS Metricbeat light module {pull}15097[15097]
 - Release elb module as GA. {pull}15485[15485]
 - Add a `system/network_summary` metricset {pull}15196[15196]
-- Add mesh metricset for Istio Metricbeat module{pull}15535[15535]
+- Add mesh metricset for Istio Metricbeat module {pull}15535[15535]
+- Add mixer metricset for Istio Metricbeat module {pull}15696[15696]
+- Add pilot metricset for Istio Metricbeat module {pull}15761[15761]
 - Make the `system/cpu` metricset collect normalized CPU metrics by default. {issue}15618[15618] {pull}15729[15729]
+- Add galley metricset for Istio Metricbeat module {pull}15857[15857]
+- Add `key/value` mode for SQL module. {issue}15770[15770] {pull]15845[15845]
+- Add STAN dashboard {pull}15654[15654]
+- Add support for Unix socket in Memcached metricbeat module. {issue}13685[13685] {pull}15822[15822]
 
 *Packetbeat*
 

README.md

@@ -85,3 +85,18 @@ your dev environment to build Beats from the source.
 ## Snapshots
 
 For testing purposes, we generate snapshot builds that you can find [here](https://beats-ci.elastic.co/job/elastic+beats+master+multijob-package-linux/lastSuccessfulBuild/gcsObjects/). Please be aware that these are built on top of master and are not meant for production.
+
+## CI
+
+It is possible to trigger some jobs by putting a comment on a GitHub PR.
+(This service is only available for users affiliated with Elastic and not for open-source contributors.)
+
+* [beats][]
+  * `jenkins run the tests please`
+  * `jenkins run tests`
+* [apm-beats-update][]
+  * `/run apm-beats-update`
+
+
+[beats]: https://beats-ci.elastic.co/job/Beats/job/beats-beats-mbp/
+[apm-beats-update]: https://beats-ci.elastic.co/job/Beats/job/apm-beats-update/

auditbeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.6
+FROM golang:1.13.7
 
 RUN \
     apt-get update \

auditbeat/cmd/root.go

@@ -47,6 +47,11 @@ func init() {
 		),
 	)
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
-	RootCmd = cmd.GenRootCmdWithSettings(create, instance.Settings{RunFlags: runFlags, Name: Name})
+	settings := instance.Settings{
+		RunFlags:      runFlags,
+		Name:          Name,
+		HasDashboards: true,
+	}
+	RootCmd = cmd.GenRootCmdWithSettings(create, settings)
 	RootCmd.AddCommand(ShowCmd)
 }

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -65,7 +65,7 @@ data:
         - fsstat
       processors:
       - drop_event.when.regexp:
-          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
+          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)'
   kubernetes.yml: |-
     - module: kubernetes
       metricsets:

deploy/kubernetes/metricbeat/metricbeat-daemonset-configmap.yaml

@@ -65,7 +65,7 @@ data:
         - fsstat
       processors:
       - drop_event.when.regexp:
-          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib)($|/)'
+          system.filesystem.mount_point: '^/(sys|cgroup|proc|dev|etc|host|lib|snap)($|/)'
   kubernetes.yml: |-
     - module: kubernetes
       metricsets:

dev-tools/find_pr_jenkins_jobs.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Description:
+#    This script finds Jenkins jobs for a given Beats PR.
+#
+# Usage:
+#    ./find_pr_jenkins_job.sh PR_NUMBER
+#
+# Example:
+#    ./find_pr_jenkins_job.sh 15790
+#
+# Dependencies:
+#    curl, jq
+
+set -e
+
+NUM_JOBS_TO_SEARCH=100
+
+get_pr_from_input() {
+    pr=$1
+    if [ -z $pr ]; then
+        echo "Usage: ./find_jenkins_job.sh PR_NUMBER" >&2
+        exit 1
+    fi
+
+    echo $pr
+}
+
+find_latest_beats_pr_job() {
+    curl -s 'https://beats-ci.elastic.co/job/elastic+beats+pull-request/api/json' | jq '.builds[0].number'
+}
+
+find_job_for_pr() {
+    job=$1
+    pr=$2
+
+    found=$(curl -s "https://beats-ci.elastic.co/job/elastic+beats+pull-request/$job/api/json" \
+                | jq -c ".actions[] | select(._class == \"org.jenkinsci.plugins.ghprb.GhprbParametersAction\").parameters[] | select(.name == \"ghprbPullId\" and .value == \"$pr\")" \
+                | wc -l)
+
+    echo $found
+}
+
+main() {
+    pr=$(get_pr_from_input $1)
+    echo "Searching last $NUM_JOBS_TO_SEARCH Jenkins jobs for PR number: $pr..."
+
+    n=$(find_latest_beats_pr_job $pr)
+    let e=$n-$NUM_JOBS_TO_SEARCH
+
+    while [ $n -gt $e ]; do
+        found=$(find_job_for_pr $n $pr)
+        if [ $found -gt 0 ]; then
+            echo "https://beats-ci.elastic.co/job/elastic+beats+pull-request/$n/"
+        fi
+
+        let n=$n-1
+    done
+}
+
+main $1

dev-tools/packaging/templates/darwin/launchd-daemon.plist.tmpl

@@ -2,23 +2,25 @@
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
 <dict>
-	<key>Label</key>
+    <key>Label</key>
         <string>{{.identifier}}</string>
-	<key>ProgramArguments</key>
-	<array>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
-            <string>-c</string>
-            <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
-            <string>--path.home</string>
-            <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
-            <string>--path.config</string>
-            <string>/etc/{{.BeatName}}</string>
-            <string>--path.data</string>
-            <string>/var/lib/{{.BeatName}}</string>
-            <string>--path.logs</string>
-            <string>/var/log/{{.BeatName}}</string>
-        </array>
-	<key>RunAtLoad</key>
-        <true/>
+    <key>ProgramArguments</key>
+    <array>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}/bin/{{.BeatName}}</string>
+        <string>-environment</string>
+        <string>macOS_service</string>
+        <string>-c</string>
+        <string>/etc/{{.BeatName}}/{{.BeatName}}.yml</string>
+        <string>--path.home</string>
+        <string>{{.install_path}}/{{.BeatVendor}}/{{.BeatName}}</string>
+        <string>--path.config</string>
+        <string>/etc/{{.BeatName}}</string>
+        <string>--path.data</string>
+        <string>/var/lib/{{.BeatName}}</string>
+        <string>--path.logs</string>
+        <string>/var/log/{{.BeatName}}</string>
+    </array>
+    <key>RunAtLoad</key>
+    <true/>
 </dict>
 </plist>

dev-tools/packaging/templates/docker/Dockerfile.tmpl

@@ -50,4 +50,4 @@ EXPOSE {{ $port }}
 
 WORKDIR {{ $beatHome }}
 ENTRYPOINT ["/usr/local/bin/docker-entrypoint"]
-CMD ["-e"]
+CMD ["-environment", "container"]

dev-tools/packaging/templates/linux/systemd.unit.tmpl

@@ -9,10 +9,10 @@ After=network-online.target
 User={{ .BeatUser }}
 Group={{ .BeatUser }}
 {{- end }}
-Environment="BEAT_LOG_OPTS=-e"
+Environment="BEAT_LOG_OPTS="
 Environment="BEAT_CONFIG_OPTS=-c /etc/{{.BeatName}}/{{.BeatName}}.yml"
 Environment="BEAT_PATH_OPTS=-path.home /usr/share/{{.BeatName}} -path.config /etc/{{.BeatName}} -path.data /var/lib/{{.BeatName}} -path.logs /var/log/{{.BeatName}}"
-ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
+ExecStart=/usr/share/{{.BeatName}}/bin/{{.BeatName}} -environment systemd $BEAT_LOG_OPTS $BEAT_CONFIG_OPTS $BEAT_PATH_OPTS
 Restart=always
 
 [Install]

dev-tools/packaging/templates/windows/install-service.ps1.tmpl

@@ -11,7 +11,7 @@ $workdir = Split-Path $MyInvocation.MyCommand.Path
 # Create the new service.
 New-Service -name {{.BeatName}} `
   -displayName {{.BeatName | title}} `
-  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
+  -binaryPathName "`"$workdir\{{.BeatName}}.exe`" -environment=windows_service -c `"$workdir\{{.BeatName}}.yml`" -path.home `"$workdir`" -path.data `"C:\ProgramData\{{.BeatName}}`" -path.logs `"C:\ProgramData\{{.BeatName}}\logs`" -E logging.files.redirect_stderr=true"
 
 # Attempt to set the service to delayed start using sc config.
 Try {

filebeat/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.13.6
+FROM golang:1.13.7
 
 RUN \
     apt-get update \

filebeat/cmd/root.go

@@ -42,7 +42,12 @@ func init() {
 	var runFlags = pflag.NewFlagSet(Name, pflag.ExitOnError)
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("once"))
 	runFlags.AddGoFlag(flag.CommandLine.Lookup("modules"))
-	RootCmd = cmd.GenRootCmdWithSettings(beater.New, instance.Settings{RunFlags: runFlags, Name: Name})
+	settings := instance.Settings{
+		RunFlags:      runFlags,
+		Name:          Name,
+		HasDashboards: true,
+	}
+	RootCmd = cmd.GenRootCmdWithSettings(beater.New, settings)
 	RootCmd.PersistentFlags().AddGoFlag(flag.CommandLine.Lookup("M"))
 	RootCmd.TestCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))
 	RootCmd.SetupCmd.Flags().AddGoFlag(flag.CommandLine.Lookup("modules"))

filebeat/docs/filebeat-options.asciidoc

@@ -54,6 +54,7 @@ You can configure {beatname_uc} to use the following inputs:
 * <<{beatname_lc}-input-s3>>
 * <<{beatname_lc}-input-netflow>>
 * <<{beatname_lc}-input-google-pubsub>>
+* <<{beatname_lc}-input-azure-eventhub>>
 
 
 include::inputs/input-log.asciidoc[]
@@ -79,3 +80,5 @@ include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[]
 include::../../x-pack/filebeat/docs/inputs/input-netflow.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-google-pubsub.asciidoc[]
+
+include::../../x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc[]

filebeat/docs/inputs/input-common-harvester-options.asciidoc

@@ -193,7 +193,7 @@ occur.
 
 *`document_id`*:: Option configuration setting that specifies the JSON key to
 set the document id. If configured, the field will be removed from the original
-json document and stored in `@metadata.id`
+json document and stored in `@metadata._id`
 
 *`ignore_decoding_error`*:: An optional configuration setting that specifies if
 JSON decoding errors should be logged or not. If set to true, errors will not

filebeat/input/log/harvester.go

@@ -444,7 +444,7 @@ func (h *Harvester) onMessage(
 
 		if id != "" {
 			meta = common.MapStr{
-				"id": id,
+				"_id": id,
 			}
 		}
 	} else if &text != nil {

filebeat/module/elasticsearch/audit/config/audit.yml

@@ -6,5 +6,4 @@ paths:
 exclude_files: [".gz$"]
 
 processors:
-# Locale for timezone is only needed in non-json logs
-- add_locale.when.not.regexp.message: "^{"
+- add_locale: ~