Merge branch 'feature/support-flaky-test-analyser' of github.com:v1v/…

…beats into feature/support-flaky-test-analyser

* 'feature/support-flaky-test-analyser' of github.com:v1v/beats: (40 commits)
  [CI] support windows-10 (elastic#19804)
  Use default config when creating the input (elastic#22126)
  Change x509 mappings from file. to tls.server. (elastic#22097)
  Add fleet settings image (elastic#22065)
  Edit 7.9.3 changelog (elastic#22117)
  Edit 6.8.13 release notes (elastic#22120)
  Incorporate librpm fix feedback (elastic#22098)
  [libbeat] Add more disk queue unit tests and fix a size-check bug (elastic#22107)
  docs: move kerberos include (elastic#22109)
  Check context.Canceled and fix s3 input config (elastic#22036)
  Add max_number_of_messages into aws filebeat fileset vars (elastic#22057)
  Remove suricata.eve.timestamp alias (elastic#22095)
  [Ingest Manager] Use symlink path for reexecutions (elastic#21835)
  chore: use ubuntu 18 as linux agent (elastic#22084)
  docs: Prepare Changelog for 7.9.3 (elastic#22073) (elastic#22075)
  docs: Prepare Changelog for 6.8.13 (elastic#22072) (elastic#22079)
  [build][packaging] Add resilience when docker build (elastic#22050)
  Fix the url of reviewdog (elastic#21981)
  revert WSS process reporting for windows (elastic#22055)
  Fix typo (elastic#19585) (elastic#22061)
  ...

.ci/beats-tester.groovy

@@ -54,13 +54,15 @@ pipeline {
  options { skipDefaultCheckout() }
  when { branch 'master' }
  steps {
+ // TODO: to use the git commit that triggered the upstream build
  runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
  }
  }
  stage('Build *.x branch') {
  options { skipDefaultCheckout() }
  when { branch '*.x' }
  steps {
+ // TODO: to use the git commit that triggered the upstream build
  runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
  }
  }
@@ -84,6 +86,7 @@ pipeline {
  }
  }
  steps {
+ // TODO: to use the git commit that triggered the upstream build
  runBeatsTesterJob(version: "${env.VERSION}-SNAPSHOT")
  }
  }

.ci/packaging.groovy

@@ -43,7 +43,7 @@ pipeline {
  }
  stages {
  stage('Filter build') {
- agent { label 'ubuntu && immutable' }
+ agent { label 'ubuntu-18 && immutable' }
  when {
  beforeAgent true
  anyOf {
@@ -98,7 +98,7 @@ pipeline {
  }
  stages {
  stage('Package Linux'){
- agent { label 'ubuntu && immutable' }
+ agent { label 'ubuntu-18 && immutable' }
  options { skipDefaultCheckout() }
  when {
  beforeAgent true
@@ -160,7 +160,7 @@ pipeline {
  }
  }
  stage('Run E2E Tests for Packages'){
- agent { label 'ubuntu && immutable' }
+ agent { label 'ubuntu-18 && immutable' }
  options { skipDefaultCheckout() }
  steps {
  runE2ETests()
@@ -191,10 +191,14 @@ def pushCIDockerImages(){
  }
 }
 
-def tagAndPush(name){
+def tagAndPush(beatName){
  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+ def aliasVersion = ""
  if("${env.SNAPSHOT}" == "true"){
+ aliasVersion = libbetaVer.substring(0, libbetaVer.lastIndexOf(".")) // remove third number in version
+
  libbetaVer += "-SNAPSHOT"
+ aliasVersion += "-SNAPSHOT"
  }
 
  def tagName = "${libbetaVer}"
@@ -207,25 +211,37 @@ def tagAndPush(name){
  // supported image flavours
  def variants = ["", "-oss", "-ubi8"]
  variants.each { variant ->
- def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
- def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
- def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
-
- def iterations = 0
- retryWithSleep(retries: 3, seconds: 5, backoff: true) {
- iterations++
- def status = sh(label:'Change tag and push', script: """
- docker tag ${oldName} ${newName}
- docker push ${newName}
- docker tag ${oldName} ${commitName}
- docker push ${commitName}
- """, returnStatus: true)
-
- if ( status > 0 && iterations < 3) {
- error('tag and push failed, retry')
- } else if ( status > 0 ) {
- log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
- }
+ doTagAndPush(beatName, variant, libbetaVer, tagName)
+ doTagAndPush(beatName, variant, libbetaVer, "${env.GIT_BASE_COMMIT}")
+
+ if (!isPR() && aliasVersion != "") {
+ doTagAndPush(beatName, variant, libbetaVer, aliasVersion)
+ }
+ }
+}
+
+/**
+* @param beatName name of the Beat
+* @param variant name of the variant used to build the docker image name
+* @param sourceTag tag to be used as source for the docker tag command, usually under the 'beats' namespace
+* @param targetTag tag to be used as target for the docker tag command, usually under the 'observability-ci' namespace
+*/
+def doTagAndPush(beatName, variant, sourceTag, targetTag) {
+ def sourceName = "${DOCKER_REGISTRY}/beats/${beatName}${variant}:${sourceTag}"
+ def targetName = "${DOCKER_REGISTRY}/observability-ci/${beatName}${variant}:${targetTag}"
+
+ def iterations = 0
+ retryWithSleep(retries: 3, seconds: 5, backoff: true) {
+ iterations++
+ def status = sh(label: "Change tag and push ${targetName}", script: """
+ docker tag ${sourceName} ${targetName}
+ docker push ${targetName}
+ """, returnStatus: true)
+
+ if ( status > 0 && iterations < 3) {
+ error("tag and push failed for ${beatName}, retry")
+ } else if ( status > 0 ) {
+ log(level: 'WARN', text: "${beatName} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
  }
  }
 }
@@ -327,7 +343,16 @@ def publishPackages(baseDir){
  bucketUri = "gs://${JOB_GCS_BUCKET}/pull-requests/pr-${env.CHANGE_ID}"
  }
  def beatsFolderName = getBeatsName(baseDir)
- googleStorageUpload(bucket: "${bucketUri}/${beatsFolderName}",
+ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+
+ // Copy those files to another location with the sha commit to test them
+ // aftewords.
+ bucketUri = "gs://${JOB_GCS_BUCKET}/commits/${env.GIT_BASE_COMMIT}"
+ uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
+}
+
+def uploadPackages(bucketUri, baseDir){
+ googleStorageUpload(bucket: bucketUri,
  credentialsId: "${JOB_GCS_CREDENTIALS}",
  pathPrefix: "${baseDir}/build/distributions/",
  pattern: "${baseDir}/build/distributions/**/*",

CHANGELOG.asciidoc

@@ -3,6 +3,31 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.3]]
+=== Beats version 7.9.3
+https://github.com/elastic/beats/compare/v7.9.2...v7.9.3[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- The `o365audit` input and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258]
+
+*Auditbeat*
+
+- system/socket: Fix a crash due to concurrent map read and write. {issue}21192[21192] {pull}21690[21690]
+
+*Filebeat*
+
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
+
+*Metricbeat*
+
+- Fix `remote_write` flaky test. {pull}21173[21173]
+- Fix panic in Kubernetes autodiscovery caused by storing stateless keystores. {issue}21843[21843] {pull}21880[21880]
+- Remove redundant dockersock volume mount to avoid problems on Kubernetes deployments that do not use docker as the container runtime. {pull}22009[22009]
+
+
 [[release-notes-7.9.2]]
 === Beats version 7.9.2
 https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
@@ -2575,6 +2600,46 @@ https://github.com/elastic/beats/compare/v6.5.0...v7.0.0-alpha1[View commits]
 - Added support to calculate certificates' fingerprints (MD5, SHA-1, SHA-256). {issue}8180[8180]
 - Support new TLS version negotiation introduced in TLS 1.3. {issue}8647[8647].
 
+[[release-notes-6.8.13]]
+=== Beats version 6.8.13
+https://github.com/elastic/beats/compare/v6.8.12...v6.8.13[View commits]
+
+==== Added
+
+*Filebeat*
+
+- Add container image in Kubernetes metadata. {pull}13356[13356] {issue}12688[12688]
+
+[[release-notes-6.8.12]]
+=== Beats version 6.8.12
+https://github.com/elastic/beats/compare/v6.8.11...v6.8.12[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Fix Filebeat OOMs on very long lines {issue}19500[19500], {pull}19552[19552]
+
+[[release-notes-6.8.11]]
+=== Beats version 6.8.11
+https://github.com/elastic/beats/compare/v6.8.10...v6.8.11[View commits]
+
+==== Bugfixes
+
+*Metricbeat*
+
+- Fix bug incorrect parsing of float numbers as integers in Couchbase module {issue}18949[18949] {pull}19055[19055]
+
+[[release-notes-6.8.10]]
+=== Beats version 6.8.10
+https://github.com/elastic/beats/compare/v6.8.9...v6.8.10[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix `add_cloud_metadata` to better support modifying sub-fields with other processors. {pull}13808[13808]
+
 [[release-notes-6.8.9]]
 === Beats version 6.8.9
 https://github.com/elastic/beats/compare/v6.8.8...v6.8.9[View commits]

CHANGELOG.next.asciidoc

@@ -80,6 +80,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 - API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
+- Remove `suricata.eve.timestamp` alias field. {issue}10535[10535] {pull}22095[22095]
 
 *Heartbeat*
 
@@ -375,6 +376,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix retrieving resources by ID for the azure module. {pull}21711[21711] {issue}21707[21707]
 - Use timestamp from CloudWatch API when creating events. {pull}21498[21498]
 - Report the correct windows events for system/filesystem {pull}21758[21758]
+- Fix azure storage event format. {pull}21845[21845]
+- Fix panic in kubernetes autodiscover related to keystores {issue}21843[21843] {pull}21880[21880]
+- [Kubernetes] Remove redundant dockersock volume mount {pull}22009[22009]
+- Revert change to report `process.memory.rss` as `process.memory.wss` on Windows. {pull}22055[22055]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -630,6 +636,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - New juniper.srx dataset for Juniper SRX logs. {pull}20017[20017]
 - Adding support for Microsoft 365 Defender (Microsoft Threat Protection) {pull}21446[21446]
 - Adding support for FIPS in s3 input {pull}21446[21446]
+- Add SSL option to checkpoint module {pull}19560[19560]
+- Add max_number_of_messages config into s3 input. {pull}21993[21993]
 
 *Heartbeat*
 
@@ -815,3 +823,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 ==== Known Issue
 
 *Journalbeat*
+
+
+

Makefile

@@ -12,7 +12,7 @@ GOLINT=golint
 GOLINT_REPO=golang.org/x/lint/golint
 REVIEWDOG=reviewdog
 REVIEWDOG_OPTIONS?=-diff "git diff master"
-REVIEWDOG_REPO=github.com/haya14busa/reviewdog/cmd/reviewdog
+REVIEWDOG_REPO=github.com/reviewdog/reviewdog/cmd/reviewdog
 XPACK_SUFFIX=x-pack/
 
 # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support

NOTICE.txt

@@ -2183,12 +2183,12 @@ Contents of probable licence file $GOMODCACHE/github.com/!azure/go-autorest/auto
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/Microsoft/go-winio
-Version: v0.4.15-0.20190919025122-fc70bd9a86b5
+Dependency : github.com/bi-zone/go-winio
+Version: v0.4.15
 Licence type (autodetected): MIT
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/!microsoft/go-winio@v0.4.15-0.20190919025122-fc70bd9a86b5/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/bi-zone/go-winio@v0.4.15/LICENSE:
 
 The MIT License (MIT)
 

auditbeat/Jenkinsfile.yml

@@ -69,3 +69,14 @@ stages:
  - "windows-2012"
  branches: true ## for all the branches
  tags: true ## for all the tags
+ windows-10:
+ mage: "mage build unitTest"
+ platforms: ## override default labels in this specific stage.
+ - "windows-10"
+ when: ## Override the top-level when.
+ comments:
+ - "/test auditbeat for windows-10"
+ labels:
+ - "windows-10"
+ branches: true ## for all the branches
+ tags: true ## for all the tags

auditbeat/docs/configuring-howto.asciidoc

@@ -42,6 +42,10 @@ include::./reload-configuration.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::{libbeat-dir}/shared-ilm.asciidoc[]

deploy/kubernetes/metricbeat-kubernetes.yaml

@@ -189,8 +189,6 @@ spec:
  - name: modules
  mountPath: /usr/share/metricbeat/modules.d
  readOnly: true
- - name: dockersock
- mountPath: /var/run/docker.sock
  - name: proc
  mountPath: /hostfs/proc
  readOnly: true
@@ -204,9 +202,6 @@ spec:
  - name: cgroup
  hostPath:
  path: /sys/fs/cgroup
- - name: dockersock
- hostPath:
- path: /var/run/docker.sock
  - name: config
  configMap:
  defaultMode: 0640

deploy/kubernetes/metricbeat/metricbeat-daemonset.yaml

@@ -64,8 +64,6 @@ spec:
  - name: modules
  mountPath: /usr/share/metricbeat/modules.d
  readOnly: true
- - name: dockersock
- mountPath: /var/run/docker.sock
  - name: proc
  mountPath: /hostfs/proc
  readOnly: true
@@ -79,9 +77,6 @@ spec:
  - name: cgroup
  hostPath:
  path: /sys/fs/cgroup
- - name: dockersock
- hostPath:
- path: /var/run/docker.sock
  - name: config
  configMap:
  defaultMode: 0640

dev-tools/mage/dockerbuilder.go

@@ -26,6 +26,7 @@ import (
  "os/exec"
  "path/filepath"
  "strings"
+ "time"
 
  "github.com/magefile/mage/sh"
  "github.com/pkg/errors"
@@ -71,7 +72,13 @@ func (b *dockerBuilder) Build() error {
 
  tag, err := b.dockerBuild()
  if err != nil {
- return errors.Wrap(err, "failed to build docker")
+ fmt.Println(">> Building docker images again (after 10 seconds)")
+ // This sleep is to avoid hitting the docker build issues when resources are not available.
+ time.Sleep(10)
+ tag, err = b.dockerBuild()
+ if err != nil {
+ return errors.Wrap(err, "failed to build docker")
+ }
  }
 
  if err := b.dockerSave(tag); err != nil {

filebeat/Jenkinsfile.yml

@@ -57,3 +57,16 @@ stages:
  # - "windows-2016"
  # branches: true ## for all the branches
  # tags: true ## for all the tags
+ windows-10:
+ mage: "mage build unitTest"
+ platforms: ## override default labels in this specific stage.
+ - "windows-10"
+ when: ## Override the top-level when.
+ comments:
+ - "/test filebeat for windows-10"
+ labels:
+ - "windows-10"
+ branches: true ## for all the branches
+ tags: true ## for all the tags
+ branches: true ## for all the branches
+ tags: true ## for all the tags

filebeat/docs/configuring-howto.asciidoc

@@ -44,6 +44,10 @@ include::./reload-configuration.asciidoc[]
 
 include::{libbeat-dir}/outputconfig.asciidoc[]
 
+ifndef::no_kerberos[]
+include::{libbeat-dir}/shared-kerberos-config.asciidoc[]
+endif::[]
+
 include::{libbeat-dir}/shared-ssl-config.asciidoc[]
 
 include::../../libbeat/docs/shared-ilm.asciidoc[]