From 3c13d2029ab5fe9815f6aeb5f7d59bae89a0bd8a Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 23 Feb 2021 15:32:37 +0200 Subject: [PATCH 1/8] Skip flaky TestActions on MacOSx (#23966) (#24177) --- auditbeat/module/file_integrity/metricset_test.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/auditbeat/module/file_integrity/metricset_test.go b/auditbeat/module/file_integrity/metricset_test.go index 43fa02d15f7..2c1d2093e89 100644 --- a/auditbeat/module/file_integrity/metricset_test.go +++ b/auditbeat/module/file_integrity/metricset_test.go @@ -130,10 +130,8 @@ func TestActions(t *testing.T) { } // Create some files in first directory - go func() { - ioutil.WriteFile(createdFilepath, []byte("hello world"), 0600) - ioutil.WriteFile(updatedFilepath, []byte("hello world"), 0600) - }() + ioutil.WriteFile(createdFilepath, []byte("hello world"), 0600) + ioutil.WriteFile(updatedFilepath, []byte("hello world"), 0600) ms := mbtest.NewPushMetricSetV2(t, getConfig(dir, newDir)) events := mbtest.RunPushMetricSetV2(10*time.Second, 5, ms) From d4b955a42db64ebd69c66c9e1ecc47c71709367d Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Tue, 2 Mar 2021 16:45:08 -0800 Subject: [PATCH 2/8] Fix admonition tag to render text correctly (#24258) (#24303) --- filebeat/docs/inputs/input-log.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/filebeat/docs/inputs/input-log.asciidoc b/filebeat/docs/inputs/input-log.asciidoc index 8d4ef24a55a..95c037707b3 100644 --- a/filebeat/docs/inputs/input-log.asciidoc +++ b/filebeat/docs/inputs/input-log.asciidoc @@ -60,7 +60,7 @@ because this can lead to unexpected behaviour. [[file-identity]] ==== Reading files on network shares and cloud providers -:WARNING: Filebeat does not support reading from network shares and cloud providers. +WARNING: Filebeat does not support reading from network shares and cloud providers. However, one of the limitations of these data sources can be mitigated if you configure Filebeat adequately. From f0da8cfbd2d23ac462773ca2832d55c423b2dfb0 Mon Sep 17 00:00:00 2001 From: DeDe Morton Date: Tue, 2 Mar 2021 16:46:08 -0800 Subject: [PATCH 3/8] Alphabetize AWS S3 input in list (#24257) (#24300) --- filebeat/docs/filebeat-options.asciidoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/filebeat/docs/filebeat-options.asciidoc b/filebeat/docs/filebeat-options.asciidoc index b8e230bfd0e..c706d44e6eb 100644 --- a/filebeat/docs/filebeat-options.asciidoc +++ b/filebeat/docs/filebeat-options.asciidoc @@ -87,6 +87,8 @@ include::multiline.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-aws-cloudwatch.asciidoc[] +include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[] + include::../../x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc[] include::../../x-pack/filebeat/docs/inputs/input-cloudfoundry.asciidoc[] @@ -115,8 +117,6 @@ include::../../x-pack/filebeat/docs/inputs/input-o365audit.asciidoc[] include::inputs/input-redis.asciidoc[] -include::../../x-pack/filebeat/docs/inputs/input-aws-s3.asciidoc[] - include::inputs/input-stdin.asciidoc[] include::inputs/input-syslog.asciidoc[] From 93b1f61573ece09bee03d53fa2ed6d9173e12fe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Manuel=20de=20la=20Pe=C3=B1a?= Date: Wed, 3 Mar 2021 10:13:24 +0100 Subject: [PATCH 4/8] chore(ci): rename variable for E2E tests (#24306) (#24309) --- .ci/packaging.groovy | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy index 3a7dd3ab425..d6eecd9bc70 100644 --- a/.ci/packaging.groovy +++ b/.ci/packaging.groovy @@ -347,9 +347,7 @@ def triggerE2ETests(String suite) { string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT), ] if (isPR()) { - def version = "pr-${env.CHANGE_ID}" - parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}")) - parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}")) + parameters.push(string(name: 'BEAT_VERSION', value: "pr-${env.CHANGE_ID}")) } build(job: "${e2eTestsPipeline}", From 02b421cdeb36d6bfe9228eae0de7786e791fb92c Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Wed, 3 Mar 2021 23:00:31 +0200 Subject: [PATCH 5/8] Cherry-pick #24318 to 7.11: Fix event time calculation on docker events (#24321) --- CHANGELOG.next.asciidoc | 1 + libbeat/common/docker/watcher.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index bcdde1dafe0..32f39ecd27f 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -112,6 +112,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add service resource in k8s cluster role. {pull}20546[20546] - [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571] - The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258] +- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318] *Auditbeat* diff --git a/libbeat/common/docker/watcher.go b/libbeat/common/docker/watcher.go index 4145423209a..5b490b1e13e 100644 --- a/libbeat/common/docker/watcher.go +++ b/libbeat/common/docker/watcher.go @@ -276,7 +276,11 @@ func (w *watcher) watch() { select { case event := <-events: w.log.Debugf("Got a new docker event: %v", event) - lastValidTimestamp = time.Unix(event.Time, event.TimeNano) + if event.TimeNano > 0 { + lastValidTimestamp = time.Unix(0, event.TimeNano) + } else { + lastValidTimestamp = time.Unix(event.Time, 0) + } lastReceivedEventTime = w.clock.Now() switch event.Action { From 1d9cced55410003f5d0b4594ff5471d15a4e2900 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 4 Mar 2021 12:26:32 +0000 Subject: [PATCH 6/8] CI: add resilience when ephemeral windows workers are reused (#24316) (#24326) --- Jenkinsfile | 36 +++++++++++++++++++++++++++++------- 1 file changed, 29 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e1ba54ffe34..f019201d828 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -286,8 +286,12 @@ def target(Map args = [:]) { */ def withNode(String label, Closure body) { sleep randomNumber(min: 10, max: 200) + // this should workaround the existing issue with reusing workers with the Gobld + def uuid = UUID.randomUUID().toString() node(label) { - body() + ws("workspace/${JOB_BASE_NAME}-${BUILD_NUMBER}-${uuid}") { + body() + } } } @@ -330,7 +334,12 @@ def withBeatsEnv(Map args = [:], Closure body) { gox_flags = '-arch 386' } - deleteDir() + // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. + // Windows workers are ephemerals, so this should not really affect us. + if(isUnix()) { + deleteDir() + } + unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") // NOTE: This is required to run after the unstash def module = withModule ? getCommonModuleInTheChangeSet(directory) : '' @@ -374,11 +383,24 @@ def withBeatsEnv(Map args = [:], Closure body) { if (archive) { archiveTestOutput(testResults: testResults, artifacts: artifacts, id: args.id, upload: upload) } - // Tear down the setup for the permamnent workers. - catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { - fixPermissions("${WORKSPACE}") - deleteDir() - } + tearDown() + } + } + } +} + +/** +* Tear down the setup for the permanent workers. +*/ +def tearDown() { + catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') { + cmd(label: 'Remove the entire module cache', script: 'go clean -modcache', returnStatus: true) + fixPermissions("${WORKSPACE}") + // IMPORTANT: Somehow windows workers got a different opinion regarding removing the workspace. + // Windows workers are ephemerals, so this should not really affect us. + if (isUnix()) { + dir("${WORKSPACE}") { + deleteDir() } } } From 17663fe240557b76619a00eaa0dac916eabb45eb Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Mon, 8 Mar 2021 10:44:45 +0000 Subject: [PATCH 7/8] [CI] Use MacOS for x86_64 arch (#24362) (#24375) --- auditbeat/Jenkinsfile.yml | 2 +- filebeat/Jenkinsfile.yml | 2 +- generator/Jenkinsfile.yml | 4 ++-- heartbeat/Jenkinsfile.yml | 2 +- metricbeat/Jenkinsfile.yml | 2 +- packetbeat/Jenkinsfile.yml | 2 +- x-pack/auditbeat/Jenkinsfile.yml | 2 +- x-pack/elastic-agent/Jenkinsfile.yml | 2 +- x-pack/filebeat/Jenkinsfile.yml | 2 +- x-pack/functionbeat/Jenkinsfile.yml | 2 +- x-pack/metricbeat/Jenkinsfile.yml | 2 +- x-pack/packetbeat/Jenkinsfile.yml | 2 +- 12 files changed, 13 insertions(+), 13 deletions(-) diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 168ac26edd5..03b48c68c11 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -40,7 +40,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test auditbeat for macos" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index ea1215b6514..fd66161a889 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -39,7 +39,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test filebeat for macos" diff --git a/generator/Jenkinsfile.yml b/generator/Jenkinsfile.yml index c4295799cd0..f3210ee482a 100644 --- a/generator/Jenkinsfile.yml +++ b/generator/Jenkinsfile.yml @@ -22,7 +22,7 @@ stages: macos-metricbeat: make: "make -C generator/_templates/metricbeat test" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test generator for macos" @@ -35,7 +35,7 @@ stages: macos-beat: make: "make -C generator/_templates/beat test" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test generator for macos" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index de73acd5be7..753073b5274 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -38,7 +38,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test heartbeat for macos" diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 2bdb1218a5d..72bbe8107a5 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -33,7 +33,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test metricbeat for macos" diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 0225058ab1c..068625e656c 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -39,7 +39,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test packetbeat for macos" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index deb1fdddd3a..6c9dd2a2544 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -39,7 +39,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/auditbeat for macos" diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 0e6ecef9e4c..b0b62f492ca 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -36,7 +36,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/elastic-agent for macos" diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 0581f2cd860..e5bf7315884 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -39,7 +39,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/filebeat for macos" diff --git a/x-pack/functionbeat/Jenkinsfile.yml b/x-pack/functionbeat/Jenkinsfile.yml index ec9a4ec57f0..d151193f3b6 100644 --- a/x-pack/functionbeat/Jenkinsfile.yml +++ b/x-pack/functionbeat/Jenkinsfile.yml @@ -34,7 +34,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/functionbeat for macos" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 90dcd4ca560..417ad60706f 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -37,7 +37,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/metricbeat for macos" diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index 7cc39a8b501..475b9e0df33 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -39,7 +39,7 @@ stages: macos: mage: "mage build unitTest" platforms: ## override default label in this specific stage. - - "macosx" + - "macosx&&x86_64" when: ## Override the top-level when. comments: - "/test x-pack/packetbeat for macos" From 01930e9f9b09c281bf3798e9c477e8f40770f243 Mon Sep 17 00:00:00 2001 From: Elastic Machine Date: Tue, 9 Mar 2021 15:46:59 -0500 Subject: [PATCH 8/8] docs: Prepare Changelog for 7.11.2 (#24437) * docs: Close changelog for 7.11.2 * Remove empty sections. Fix breaking change entry Co-authored-by: Andres Rodriguez --- CHANGELOG.asciidoc | 20 ++++++++++++++++++++ CHANGELOG.next.asciidoc | 8 +++----- libbeat/docs/release.asciidoc | 1 + 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc index 0d1ea0d2ebe..51c0f1f8aea 100644 --- a/CHANGELOG.asciidoc +++ b/CHANGELOG.asciidoc @@ -3,6 +3,26 @@ :issue: https://github.com/elastic/beats/issues/ :pull: https://github.com/elastic/beats/pull/ +[[release-notes-7.11.2]] +=== Beats version 7.11.2 +https://github.com/elastic/beats/compare/v7.11.1...v7.11.2[View commits] + +==== Bugfixes + +*Affecting all Beats* + +- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318] + +*Filebeat* + +- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025] +- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167] +- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052] + +*Metricbeat* + +- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647] + [[release-notes-7.11.1]] === Beats version 7.11.1 https://github.com/elastic/beats/compare/v7.11.0...v7.11.1[View commits] diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 32f39ecd27f..e7b38b33bcd 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -41,7 +41,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Disable the option of running --machine-learning on its own. {pull}20241[20241] - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808] - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993] -- Add `nodes` to filebeat-kubernetes.yaml ClusterRole. {issue}24051[24051] {pull}24052[24052] *Heartbeat* @@ -112,7 +111,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add service resource in k8s cluster role. {pull}20546[20546] - [Metricbeat][Kubernetes] Change cluster_ip field from ip to keyword. {pull}20571[20571] - The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21258[21258] -- Fix issue discovering docker containers and metadata after reconnections {pull}24318[24318] *Auditbeat* @@ -167,8 +165,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Fix `cisco` asa and ftd parsing of messages 106102 and 106103. {pull}20469[20469] - Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390] - Fix event.type for zeek/ssl and duplicate event.category for zeek/connection {pull}20696[20696] -- Fix Okta default date formatting. {issue}24018[24018] {pull}24025[24025] -- Fix aws/vpcflow generating errors for empty logs or unidentified formats. {pull}24167[24167] *Heartbeat* @@ -230,7 +226,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403] - Add support for azure light metricset app_stats. {pull}20639[20639] - Fix remote_write flaky test. {pull}21173[21173] -- Add check for iis/application_pool metricset for nil worker process id values. {issue}23605[23605] {pull}23647[23647] *Packetbeat* @@ -470,3 +465,6 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d + + + diff --git a/libbeat/docs/release.asciidoc b/libbeat/docs/release.asciidoc index b84a4aa0ae9..7c6f3aa2a16 100644 --- a/libbeat/docs/release.asciidoc +++ b/libbeat/docs/release.asciidoc @@ -8,6 +8,7 @@ This section summarizes the changes in each release. Also read <> for more detail about changes that affect upgrade. +* <> * <> * <> * <>