Merge branch '7.11' into backport_24156_7.11

elastic · Mar 10, 2021 · c4fedf7 · c4fedf7
2 parents 48b2ecb + 01930e9
commit c4fedf7
Show file tree

Hide file tree

Showing 21 changed files with 77 additions and 33 deletions.
diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -347,9 +347,7 @@ def triggerE2ETests(String suite) {
  string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
  ]
  if (isPR()) {
- def version = "pr-${env.CHANGE_ID}"
- parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
- parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+ parameters.push(string(name: 'BEAT_VERSION', value: "pr-${env.CHANGE_ID}"))
  }
 
  build(job: "${e2eTestsPipeline}",

diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,26 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.11.2]]
+=== Beats version 7.11.2
+https://github.com/elastic/beats/compare/v7.11.1...v7.11.2[View commits]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318]
+
+*Filebeat*
+
+- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025]
+- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167]
+- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
+
+*Metricbeat*
+
+- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
+
 [[release-notes-7.11.1]]
 === Beats version 7.11.1
 https://github.com/elastic/beats/compare/v7.11.0...v7.11.1[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -41,7 +41,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Disable the option of running --machine-learning on its own. {pull}20241[20241]
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
-- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052]
 
 *Heartbeat*
 
@@ -230,7 +229,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
 - Add support for azure light metricset app_stats. {pull}20639[20639]
 - Fix remote_write flaky test. {pull}21173[21173]
-- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647]
 
 *Packetbeat*
 
@@ -470,3 +468,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 
 
+
+
+
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -286,8 +286,12 @@ def target(Map args = [:]) {
 */
 def withNode(String label, Closure body) {
  sleep randomNumber(min: 10, max: 200)
+ // this should workaround the existing issue with reusing workers with the Gobld
+ def uuid = UUID.randomUUID().toString()
  node(label) {
- body()
+ ws("workspace/${JOB_BASE_NAME}-${BUILD_NUMBER}-${uuid}") {
+ body()
+ }
  }
 }
 
@@ -330,7 +334,12 @@ def withBeatsEnv(Map args = [:], Closure body) {
  gox_flags = '-arch 386'
  }
 
- deleteDir()
+ // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace.
+ // Windows workers are ephemerals, so this should not really affect us.
+ if(isUnix()) {
+ deleteDir()
+ }
+
  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
  // NOTE: This is required to run after the unstash
  def module = withModule ? getCommonModuleInTheChangeSet(directory) : ''
@@ -374,11 +383,24 @@ def withBeatsEnv(Map args = [:], Closure body) {
  if (archive) {
  archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload)
  }
- // Tear down the setup for the permamnent workers.
- catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
- fixPermissions("${WORKSPACE}")
- deleteDir()
- }
+ tearDown()
+ }
+ }
+ }
+}
+
+/**
+* Tear down the setup for the permanent workers.
+*/
+def tearDown() {
+ catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+ cmd(label: 'Remove the entire module cache', script: 'go clean -modcache', returnStatus: true)
+ fixPermissions("${WORKSPACE}")
+ // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace.
+ // Windows workers are ephemerals, so this should not really affect us.
+ if (isUnix()) {
+ dir("${WORKSPACE}") {
+ deleteDir()
  }
  }
  }

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -40,7 +40,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test auditbeat for macos"

diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go
@@ -130,10 +130,8 @@ func TestActions(t *testing.T) {
  }
 
  // Create some files in first directory
- go func() {
- ioutil.WriteFile(createdFilepath, []byte("hello world"), 0600)
- ioutil.WriteFile(updatedFilepath, []byte("hello world"), 0600)
- }()
+ ioutil.WriteFile(createdFilepath, []byte("hello world"), 0600)
+ ioutil.WriteFile(updatedFilepath, []byte("hello world"), 0600)
 
  ms := mbtest.NewPushMetricSetV2(t, getConfig(dir, newDir))
  events := mbtest.RunPushMetricSetV2(10*time.Second, 5, ms)

diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -39,7 +39,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test filebeat for macos"

diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc
@@ -87,6 +87,8 @@ include::multiline.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc[]
 
+include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[]
+
 include::../../x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc[]
 
 include::../../x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc[]
@@ -115,8 +117,6 @@ include::../../x-pack/filebeat/docs/inputs/input-o365audit.asciidoc[]
 
 include::inputs/input-redis.asciidoc[]
 
-include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[]
-
 include::inputs/input-stdin.asciidoc[]
 
 include::inputs/input-syslog.asciidoc[]

diff --git a/filebeat/docs/inputs/input-log.asciidoc b/filebeat/docs/inputs/input-log.asciidoc
@@ -60,7 +60,7 @@ because this can lead to unexpected behaviour.
 [[file-identity]]
 ==== Reading files on network shares and cloud providers
 
-:WARNING: Filebeat does not support reading from network shares and cloud providers.
+WARNING: Filebeat does not support reading from network shares and cloud providers.
 
 However, one of the limitations of these data sources can be mitigated
 if you configure Filebeat adequately.

diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml
@@ -22,7 +22,7 @@ stages:
  macos-metricbeat:
  make: "make -C generator/_templates/metricbeat test"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test generator for macos"
@@ -35,7 +35,7 @@ stages:
  macos-beat:
  make: "make -C generator/_templates/beat test"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test generator for macos"

diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -38,7 +38,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test heartbeat for macos"

diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go
@@ -276,7 +276,11 @@ func (w *watcher) watch() {
  select {
  case event := <-events:
  w.log.Debugf("Got a new docker event: %v", event)
- lastValidTimestamp = time.Unix(event.Time, event.TimeNano)
+ if event.TimeNano > 0 {
+ lastValidTimestamp = time.Unix(0, event.TimeNano)
+ } else {
+ lastValidTimestamp = time.Unix(event.Time, 0)
+ }
  lastReceivedEventTime = w.clock.Now()
 
  switch event.Action {

diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc
@@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read
 <<breaking-changes>> for more detail about changes that affect
 upgrade.
 
+* <<release-notes-7.11.2>>
 * <<release-notes-7.11.1>>
 * <<release-notes-7.11.0>>
 * <<release-notes-7.10.2>>

diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
@@ -33,7 +33,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test metricbeat for macos"

diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -39,7 +39,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test packetbeat for macos"

diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -39,7 +39,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/auditbeat for macos"

diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -36,7 +36,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/elastic-agent for macos"

diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -39,7 +39,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/filebeat for macos"

diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml
@@ -34,7 +34,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/functionbeat for macos"

diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
@@ -37,7 +37,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/metricbeat for macos"

diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
@@ -39,7 +39,7 @@ stages:
  macos:
  mage: "mage build unitTest"
  platforms: ## override default label in this specific stage.
- - "macosx"
+ - "macosx&&x86_64"
  when: ## Override the top-level when.
  comments:
  - "/test x-pack/packetbeat for macos"