Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0-arm * upstream/master: (54 commits) [CI] Change x-pack/auditbeat build events (comments, labels) (elastic#21463) [CI] changeset from elastic#20603 was not added to CI2.0 (elastic#21464) Add new log file reader for filestream input (elastic#21450) [CI] Send slack message with build status (elastic#21428) Remove duplicated sources url in dependencies report (elastic#21462) Add implementation of FSWatcher and FSScanner for filestream (elastic#21444) [Ingest Manager] Split index restrictions into type,dataset, namespace parts (elastic#21406) Update Filebeat module expected logs files (elastic#21454) Edit SQL module docs and fix broken doc structure (elastic#21233) [Ingest Manager] Send snapshot flag together with metadata (elastic#21285) Revert "[JJBB] Set shallow cloning to 10 (elastic#21409)" (elastic#21447) [JJBB] Use reference repo for fast checkouts (elastic#21410) Add initial skeleton of filestream input (elastic#21427) Initial spec file for apm-server (elastic#21225) [Ingest Manager] Upgrade Action: make source URI optional (elastic#21372) Add field limit check for AWS Cloudtrail flattened fields (elastic#21388) [Winlogbeat] Move winlogbeat javascript processor to libbeat (elastic#21402) ci: pipeline to generate the changelog (elastic#21426) [JJBB] Set shallow cloning to 10 (elastic#21409) docs: add link to release notes for 7.9.2 (elastic#21405) (elastic#21419) ...
v1v · Oct 2, 2020 · 83f8acf · 83f8acf
2 parents 7f2fb7f + 35a76df
commit 83f8acf
Show file tree

Hide file tree

Showing 324 changed files with 21,865 additions and 17,466 deletions.
diff --git a/.ci/jobs/beats-release-changelog.yml b/.ci/jobs/beats-release-changelog.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-changelog
+    display-name: 'Prepare the Changelog for a Release'
+    description: 'Automate the steps to prepare the Changelog for a Release'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-changelog.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master
diff --git a/.ci/jobs/beats-release-minor-major.yml b/.ci/jobs/beats-release-minor-major.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-minor-major
+    display-name: 'Prepare Major/minor Release'
+    description: 'Automate the steps to prepare a new Release branch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-minor-major.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master
diff --git a/.ci/jobs/beats-release-patch.yml b/.ci/jobs/beats-release-patch.yml
@@ -0,0 +1,20 @@
+---
+- job:
+    name: Beats/Release/beats-release-patch
+    display-name: 'Prepare Patch Release'
+    description: 'Automate the steps to prepare a new Patch'
+    view: Beats
+    project-type: pipeline
+    pipeline-scm:
+        script-path: release_scripts/pipeline-release-patch.groovy
+        scm:
+        - git:
+            url: git@github.com:elastic/ingest-dev.git
+            refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+            branches:
+                - master
diff --git a/.ci/jobs/beats.yml b/.ci/jobs/beats.yml
@@ -53,6 +53,7 @@
             recursive: true
             parent-credentials: true
             timeout: 100
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
         timeout: '15'
         use-author: true
         wipe-workspace: true
diff --git a/.ci/jobs/folders.yml b/.ci/jobs/folders.yml
@@ -4,3 +4,8 @@
     name: Beats
     description: Beats
     project-type: folder
+
+- job:
+    name: Beats/Release
+    description: Jobs for release preparation
+    project-type: folder
diff --git a/CHANGELOG.asciidoc b/CHANGELOG.asciidoc
@@ -3,6 +3,47 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.2]]
+=== Beats version 7.9.2
+https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
+- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+
+*Filebeat*
+
+- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
+- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
+- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+
+*Metricbeat*
+
+- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095]
+- Handle missing counters in the application_pool metricset. {pull}21071[21071]
+
+*Functionbeat*
+
+- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
+
 [[release-notes-7.9.1]]
 === Beats version 7.9.1
 https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]

diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -22,6 +22,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
 - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 - Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]
+- Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
+- Update to Golang 1.12.1. {pull}11330[11330]
+- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 
 *Auditbeat*
 
@@ -73,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
+- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 
 *Heartbeat*
 
@@ -174,6 +178,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
 - Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 - Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21259[21258]
 
 *Auditbeat*
 
@@ -267,7 +272,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
 - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
 - Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+- Handle multiple upstreams in ingress-controller. {pull}21215[21215]
 - Provide backwards compatibility for the `append` processor when Elasticsearch is less than 7.10.0. {pull}21159[21159]
+- Fix checkpoint module when logs contain time field. {pull}20567[20567]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 
@@ -352,6 +360,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix timestamp handling in remote_write. {pull}21166[21166]
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -363,6 +372,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
 - Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983]
+- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 
 *Functionbeat*
 
@@ -437,6 +447,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
 - Add Cloud Foundry tags in related events. {pull}21177[21177]
+- Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
 
 *Auditbeat*
 
@@ -591,6 +602,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
 - Always attempt community_id processor on zeek module {pull}21155[21155]
 - Add related.hosts ecs field to all modules {pull}21160[21160]
+- Keep cursor state between httpjson input restarts {pull}20751[20751]
 
 *Heartbeat*
 
@@ -716,6 +728,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Sanitize `event.host`. {pull}21022[21022]
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
+- Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
 
 *Packetbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -12,15 +12,18 @@ pipeline {
   agent { label 'ubuntu-18 && immutable' }
   environment {
     AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
-    BASE_DIR = 'src/github.com/elastic/beats'
+    REPO = 'beats'
+    BASE_DIR = "src/github.com/elastic/${env.REPO}"
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_COMPOSE_VERSION = "1.21.0"
     DOCKER_REGISTRY = 'docker.elastic.co'
     JOB_GCS_BUCKET = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
     OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
     PIPELINE_LOG_LEVEL = 'INFO'
+    PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
     RUNBLD_DISABLE_NOTIFICATIONS = 'true'
+    SLACK_CHANNEL = "#beats-ci-builds"
     TERRAFORM_VERSION = "0.12.24"
     XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
   }
@@ -44,6 +47,7 @@ pipeline {
     booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
     booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.')
     booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
+    string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
   }
   stages {
     stage('Checkout') {
@@ -104,26 +108,21 @@ pipeline {
                 mapParallelTasks["${k}"] = v
               }
             }
+            notifyBuildReason()
             parallel(mapParallelTasks)
           }
         }
       }
-      post {
-        always {
-          dir("${BASE_DIR}"){
-            // Archive the markdown files that contain the build reasons
-            archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md')
-          }
-        }
-      }
     }
   }
   post {
     always {
-      runbld()
+      deleteDir()
+      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+      runbld(stashedTestReports: stashedTestReports, project: env.REPO)
     }
     cleanup {
-      notifyBuildResult(prComment: true)
+      notifyBuildResult(prComment: true, slackComment: true)
     }
   }
 }
@@ -355,7 +354,7 @@ def archiveTestOutput(Map args = [:]) {
     }
     cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
     dir('build') {
-      junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id)
+      junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id)
       archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
     }
     catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
@@ -371,50 +370,6 @@ def archiveTestOutput(Map args = [:]) {
   }
 }
 
-/**
-* This method wraps the junit built-in step to archive the test reports that gonna be populated later on
-* with the runbld post build step.
-*/
-def junitAndStore(Map args = [:]) {
-  junit(args)
-  // args.id could be null in some cases, so let's use the currentmilliseconds
-  def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}"
-  stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true)
-  stashedTestReports[stageName] = stageName
-}
-
-/**
-* This method populates the test output using the runbld approach. For such it requires the
-* global variable stashedTestReports.
-* TODO: should be moved to the shared library
-*/
-def runbld() {
-  catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') {
-    if (stashedTestReports) {
-      def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats'
-      deleteDir()
-      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
-      dir("${env.BASE_DIR}") {
-        // Unstash the test reports
-        stashedTestReports.each { k, v ->
-          dir(k) {
-            unstash(v)
-          }
-        }
-      }
-      sh(label: 'Process JUnit reports with runbld',
-        script: """\
-        ## for debugging purposes
-        find . -name "TEST-*.xml"
-        cat >./runbld-script <<EOF
-        echo "Processing JUnit reports with runbld..."
-        EOF
-        /usr/local/bin/runbld ./runbld-script --job-name ${jobName}
-        """.stripIndent())  // stripIdent() requires '''/
-    }
-  }
-}
-
 /**
 * This method executes a closure with credentials for cloud test
 * environments.
@@ -598,6 +553,17 @@ def isDockerInstalled(){
   }
 }
 
+/**
+* Notify the build reason.
+*/
+def notifyBuildReason() {
+  // Archive the build reason here, since the workspace can be deleted when running the parallel stages.
+  archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.*')
+  if (isPR()) {
+    echo 'TODO: Add a comment with the build reason (this is required to be implemented in the shared library)'
+  }
+}
+
 /**
 * This class is the one used for running the parallel stages, therefore
 * its arguments are passed by the beatsStages step.

diff --git a/Jenkinsfile.yml b/Jenkinsfile.yml
@@ -28,11 +28,13 @@ changeset:
         - "^\\.ci/scripts/.*"
     oss:
         - "^go.mod"
+        - "^pytest.ini"
         - "^dev-tools/.*"
         - "^libbeat/.*"
         - "^testing/.*"
     xpack:
         - "^go.mod"
+        - "^pytest.ini"
         - "^dev-tools/.*"
         - "^libbeat/.*"
         - "^testing/.*"