Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0-arm

* upstream/master: (54 commits)
  [CI] Change x-pack/auditbeat build events (comments, labels) (elastic#21463)
  [CI] changeset from elastic#20603 was not added to CI2.0 (elastic#21464)
  Add new log file reader for filestream input (elastic#21450)
  [CI] Send slack message with build status (elastic#21428)
  Remove duplicated sources url in dependencies report (elastic#21462)
  Add implementation of FSWatcher and FSScanner for filestream (elastic#21444)
  [Ingest Manager] Split index restrictions into type,dataset, namespace parts (elastic#21406)
  Update Filebeat module expected logs files (elastic#21454)
  Edit SQL module docs and fix broken doc structure (elastic#21233)
  [Ingest Manager] Send snapshot flag together with metadata (elastic#21285)
  Revert "[JJBB] Set shallow cloning to 10 (elastic#21409)" (elastic#21447)
  [JJBB] Use reference repo for fast checkouts (elastic#21410)
  Add initial skeleton of filestream input (elastic#21427)
  Initial spec file for apm-server (elastic#21225)
  [Ingest Manager] Upgrade Action: make source URI optional (elastic#21372)
  Add field limit check for AWS Cloudtrail flattened fields (elastic#21388)
  [Winlogbeat] Move winlogbeat javascript processor to libbeat (elastic#21402)
  ci: pipeline to generate the changelog (elastic#21426)
  [JJBB] Set shallow cloning to 10 (elastic#21409)
  docs: add link to release notes for 7.9.2 (elastic#21405) (elastic#21419)
  ...

.ci/jobs/beats-release-changelog.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-changelog
+ display-name: 'Prepare the Changelog for a Release'
+ description: 'Automate the steps to prepare the Changelog for a Release'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-changelog.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats-release-minor-major.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-minor-major
+ display-name: 'Prepare Major/minor Release'
+ description: 'Automate the steps to prepare a new Release branch'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-minor-major.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats-release-patch.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-patch
+ display-name: 'Prepare Patch Release'
+ description: 'Automate the steps to prepare a new Patch'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-patch.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats.yml

@@ -53,6 +53,7 @@
  recursive: true
  parent-credentials: true
  timeout: 100
+ reference-repo: /var/lib/jenkins/.git-references/beats.git
  timeout: '15'
  use-author: true
  wipe-workspace: true

.ci/jobs/folders.yml

@@ -4,3 +4,8 @@
  name: Beats
  description: Beats
  project-type: folder
+
+- job:
+ name: Beats/Release
+ description: Jobs for release preparation
+ project-type: folder

CHANGELOG.asciidoc

@@ -3,6 +3,47 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-7.9.2]]
+=== Beats version 7.9.2
+https://github.com/elastic/beats/compare/v7.9.1...v7.9.2[View commits]
+
+==== Breaking changes
+
+*Affecting all Beats*
+
+- Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
+
+==== Bugfixes
+
+*Affecting all Beats*
+
+- Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
+- Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+
+*Filebeat*
+
+- Provide backwards compatibility for the `set` processor when Elasticsearch is less than 7.9.0. {pull}20908[20908]
+- Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
+- Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+
+*Metricbeat*
+
+- The Kibana collector applies backoff when errored at getting usage stats {pull}20772[20772]
+- The `elasticsearch/index` metricset only requests wildcard expansion for hidden indices if the monitored Elasticsearch cluster supports it. {pull}20938[20938]
+- Fix panic index out of range error when getting AWS account name. {pull}21101[21101] {issue}21095[21095]
+- Handle missing counters in the application_pool metricset. {pull}21071[21071]
+
+*Functionbeat*
+
+- Do not need Google credentials if not required for the operation. {issue}17329[17329] {pull}21072[21072]
+- Fix dependency issues of GCP functions. {issue}20830[20830] {pull}21070[21070]
+
+==== Added
+
+*Affecting all Beats*
+
+- Add container ECS fields in kubernetes metadata. {pull}20984[20984]
+
 [[release-notes-7.9.1]]
 === Beats version 7.9.1
 https://github.com/elastic/beats/compare/v7.9.0...v7.9.1[View commits]

CHANGELOG.next.asciidoc

@@ -22,6 +22,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added `certificate` TLS verification mode to ignore server name mismatch. {issue}12283[12283] {pull}20293[20293]
 - Autodiscover doesn't generate any configuration when a variable is missing. Previously it generated an incomplete configuration. {pull}20898[20898]
 - Remove redundant `cloudfoundry.*.timestamp` fields. This value is set in `@timestamp`. {pull}21175[21175]
+- Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
+- Update to Golang 1.12.1. {pull}11330[11330]
+- Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
 
 *Auditbeat*
 
@@ -73,6 +76,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
+- Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
 
 *Heartbeat*
 
@@ -174,6 +178,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - [Autodiscover] Handle input-not-finished errors in config reload. {pull}20915[20915]
 - Explicitly detect missing variables in autodiscover configuration, log them at the debug level. {issue}20568[20568] {pull}20898[20898]
 - Fix `libbeat.output.write.bytes` and `libbeat.output.read.bytes` metrics of the Elasticsearch output. {issue}20752[20752] {pull}21197[21197]
+- The `o365input` and `o365` module now recover from an authentication problem or other fatal errors, instead of terminating. {pull}21259[21258]
 
 *Auditbeat*
 
@@ -267,7 +272,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Remove wrongly mapped `tls.client.server_name` from `fortinet/firewall` fileset. {pull}20983[20983]
 - Fix an error updating file size being logged when EOF is reached. {pull}21048[21048]
 - Fix error when processing AWS Cloudtrail Digest logs. {pull}21086[21086] {issue}20943[20943]
+- Handle multiple upstreams in ingress-controller. {pull}21215[21215]
 - Provide backwards compatibility for the `append` processor when Elasticsearch is less than 7.10.0. {pull}21159[21159]
+- Fix checkpoint module when logs contain time field. {pull}20567[20567]
+- Add field limit check for AWS Cloudtrail flattened fields. {pull}21388[21388] {issue}21382[21382]
 
 *Heartbeat*
 
@@ -352,6 +360,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix timestamp handling in remote_write. {pull}21166[21166]
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
+- Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
 
 *Packetbeat*
 
@@ -363,6 +372,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 - Fix invalid IP addresses in DNS query results from Sysmon data. {issue}18432[18432] {pull}18436[18436]
 - Fields from Winlogbeat modules were not being included in index templates and patterns. {pull}18983[18983]
+- Add source.ip validation for event ID 4778 in the Security module. {issue}19627[19627]
 
 *Functionbeat*
 
@@ -437,6 +447,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Added experimental dataset `juniper/netscreen`. {pull}20820[20820]
 - Added experimental dataset `sophos/utm`. {pull}20820[20820]
 - Add Cloud Foundry tags in related events. {pull}21177[21177]
+- Add option to select the type of index template to load: legacy, component, index. {pull}21212[21212]
 
 *Auditbeat*
 
@@ -591,6 +602,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add type and sub_type to panw panos fileset {pull}20912[20912]
 - Always attempt community_id processor on zeek module {pull}21155[21155]
 - Add related.hosts ecs field to all modules {pull}21160[21160]
+- Keep cursor state between httpjson input restarts {pull}20751[20751]
 
 *Heartbeat*
 
@@ -716,6 +728,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Sanitize `event.host`. {pull}21022[21022]
 - Add overview and platform health dashboards to Cloud Foundry module. {pull}21124[21124]
 - Release lambda metricset in aws module as GA. {issue}21251[21251] {pull}21255[21255]
+- Add dashboard for pubsub metricset in googlecloud module. {pull}21326[21326] {issue}17137[17137]
 
 *Packetbeat*
 

Jenkinsfile

@@ -12,15 +12,18 @@ pipeline {
  agent { label 'ubuntu-18 && immutable' }
  environment {
  AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
- BASE_DIR = 'src/github.com/elastic/beats'
+ REPO = 'beats'
+ BASE_DIR = "src/github.com/elastic/${env.REPO}"
  DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
  DOCKER_COMPOSE_VERSION = "1.21.0"
  DOCKER_REGISTRY = 'docker.elastic.co'
  JOB_GCS_BUCKET = 'beats-ci-temp'
  JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
  OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
  PIPELINE_LOG_LEVEL = 'INFO'
+ PYTEST_ADDOPTS = "${params.PYTEST_ADDOPTS}"
  RUNBLD_DISABLE_NOTIFICATIONS = 'true'
+ SLACK_CHANNEL = "#beats-ci-builds"
  TERRAFORM_VERSION = "0.12.24"
  XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
  }
@@ -44,6 +47,7 @@ pipeline {
  booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
  booleanParam(name: 'armTest', defaultValue: false, description: 'Allow arm stages.')
  booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
+ string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
  }
  stages {
  stage('Checkout') {
@@ -104,26 +108,21 @@ pipeline {
  mapParallelTasks["${k}"] = v
  }
  }
+ notifyBuildReason()
  parallel(mapParallelTasks)
  }
  }
  }
- post {
- always {
- dir("${BASE_DIR}"){
- // Archive the markdown files that contain the build reasons
- archiveArtifacts(allowEmptyArchive: false, artifacts: 'build-reasons/*.md')
- }
- }
- }
  }
  }
  post {
  always {
- runbld()
+ deleteDir()
+ unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+ runbld(stashedTestReports: stashedTestReports, project: env.REPO)
  }
  cleanup {
- notifyBuildResult(prComment: true)
+ notifyBuildResult(prComment: true, slackComment: true)
  }
  }
 }
@@ -355,7 +354,7 @@ def archiveTestOutput(Map args = [:]) {
  }
  cmd(label: 'Prepare test output', script: 'python .ci/scripts/pre_archive_test.py')
  dir('build') {
- junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, id: args.id)
+ junitAndStore(allowEmptyResults: true, keepLongStdio: true, testResults: args.testResults, stashedTestReports: stashedTestReports, id: args.id)
  archiveArtifacts(allowEmptyArchive: true, artifacts: args.artifacts)
  }
  catchError(buildResult: 'SUCCESS', message: 'Failed to archive the build test results', stageResult: 'SUCCESS') {
@@ -371,50 +370,6 @@ def archiveTestOutput(Map args = [:]) {
  }
 }
 
-/**
-* This method wraps the junit built-in step to archive the test reports that gonna be populated later on
-* with the runbld post build step.
-*/
-def junitAndStore(Map args = [:]) {
- junit(args)
- // args.id could be null in some cases, so let's use the currentmilliseconds
- def stageName = args.id ? args.id?.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}"
- stash(includes: args.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true)
- stashedTestReports[stageName] = stageName
-}
-
-/**
-* This method populates the test output using the runbld approach. For such it requires the
-* global variable stashedTestReports.
-* TODO: should be moved to the shared library
-*/
-def runbld() {
- catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') {
- if (stashedTestReports) {
- def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats'
- deleteDir()
- unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
- dir("${env.BASE_DIR}") {
- // Unstash the test reports
- stashedTestReports.each { k, v ->
- dir(k) {
- unstash(v)
- }
- }
- }
- sh(label: 'Process JUnit reports with runbld',
- script: """\
- ## for debugging purposes
- find . -name "TEST-*.xml"
- cat >./runbld-script <<EOF
- echo "Processing JUnit reports with runbld..."
- EOF
- /usr/local/bin/runbld ./runbld-script --job-name ${jobName}
- """.stripIndent()) // stripIdent() requires '''/
- }
- }
-}
-
 /**
 * This method executes a closure with credentials for cloud test
 * environments.
@@ -598,6 +553,17 @@ def isDockerInstalled(){
  }
 }
 
+/**
+* Notify the build reason.
+*/
+def notifyBuildReason() {
+ // Archive the build reason here, since the workspace can be deleted when running the parallel stages.
+ archiveArtifacts(allowEmptyArchive: true, artifacts: 'build-reasons/*.*')
+ if (isPR()) {
+ echo 'TODO: Add a comment with the build reason (this is required to be implemented in the shared library)'
+ }
+}
+
 /**
 * This class is the one used for running the parallel stages, therefore
 * its arguments are passed by the beatsStages step.

Jenkinsfile.yml

@@ -28,11 +28,13 @@ changeset:
  - "^\\.ci/scripts/.*"
  oss:
  - "^go.mod"
+ - "^pytest.ini"
  - "^dev-tools/.*"
  - "^libbeat/.*"
  - "^testing/.*"
  xpack:
  - "^go.mod"
+ - "^pytest.ini"
  - "^dev-tools/.*"
  - "^libbeat/.*"
  - "^testing/.*"