Merge remote-tracking branch 'upstream/master' into feature/windows-7-32

* upstream/master: (156 commits)
  [CI] Enable build stage for arm architecture (elastic#21284)
  [BUILD][CI] fetch dependencies with retry (elastic#21614)
  Add proxy metricset (elastic#21751)
  [Filebeat][httpjson] Fix date_cursor validation (elastic#21756)
  Update endpoint-related terminology in Elastic Agent docs (elastic#21458)
  Move s3_daily_storage and s3_request metricsets to use cloudwatch input (elastic#21703)
  [Agent] Remove heartbeat from bundled beats in agent. (elastic#21602)
  [Elastic Agent] Don't perform install until after enroll (elastic#21772)
  [Elastic Agent] Update path from policy change for Kibana connection. (elastic#21804)
  Change cloud.provider from googlecloud to gcp (elastic#21775)
  Fix stress test package list (elastic#21719)
  [Elastic Agent] Remove the service installer scripts from packaging (elastic#21694)
  [CI] Support Windows-2012 in pipeline 2.0 (elastic#21338)
  Fix non-windows fields on system/filesystem (elastic#21758)
  disable TestReceiveEventsAndMetadata/TestSocketCleanup/TestReceiveNewEventsConcurrently in Windows (elastic#21750)
  Use badger code from upstream repository (elastic#21705)
  Disable writes sync in persistent cache (elastic#21754)
  Make API address and Shard ID required in Cloud Foundry settings (elastic#21759)
  [CI] Support skip-ci label (elastic#21377)
  Increase recommended memory when deploying in Cloud foundry (elastic#21755)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
  "upstream": "elastic/beats",
- "branches": [{ "name": "7.9"}, { "name": "7.8"}, { "name": "7.7"}, { "name": "7.x"}],
+ "branches": [ { "name": "7.x", "checked": true }, "7.10", "7.9", "7.8", "7.7"],
  "labels": ["backport"],
  "autoAssign": true,
  "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/jobs/apm-beats-update.yml

@@ -18,7 +18,7 @@
  discover-pr-forks-trust: 'permission'
  discover-pr-origin: 'merge-current'
  discover-tags: true
- head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+ head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
  disable-pr-notifications: true
  notification-context: 'apm-beats-update'
  repo: 'beats'
@@ -38,6 +38,9 @@
  - regex-name:
  regex: '7\.[x789]'
  case-sensitive: true
+ - regex-name:
+ regex: '7\.1\d'
+ case-sensitive: true
  - regex-name:
  regex: '8\.\d+'
  case-sensitive: true
@@ -48,7 +51,7 @@
  before: true
  prune: true
  shallow-clone: true
- depth: 3
+ depth: 10
  do-not-fetch-tags: true
  submodule:
  disable: false

.ci/jobs/beats-release-changelog.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-changelog
+ display-name: 'Prepare the Changelog for a Release'
+ description: 'Automate the steps to prepare the Changelog for a Release'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-changelog.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats-release-minor-major.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-minor-major
+ display-name: 'Prepare Major/minor Release'
+ description: 'Automate the steps to prepare a new Release branch'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-minor-major.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats-release-patch.yml

@@ -0,0 +1,20 @@
+---
+- job:
+ name: Beats/Release/beats-release-patch
+ display-name: 'Prepare Patch Release'
+ description: 'Automate the steps to prepare a new Patch'
+ view: Beats
+ project-type: pipeline
+ pipeline-scm:
+ script-path: release_scripts/pipeline-release-patch.groovy
+ scm:
+ - git:
+ url: git@github.com:elastic/ingest-dev.git
+ refspec: +refs/heads/*:refs/remotes/origin/* +refs/pull/*/head:refs/remotes/origin/pr/*
+ wipe-workspace: 'True'
+ name: origin
+ shallow-clone: true
+ credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+ reference-repo: /var/lib/jenkins/.git-references/ingest-dev.git
+ branches:
+ - master

.ci/jobs/beats-tester.yml

@@ -44,7 +44,7 @@
  before: true
  prune: true
  shallow-clone: true
- depth: 3
+ depth: 10
  do-not-fetch-tags: true
  submodule:
  disable: false

.ci/jobs/beats.yml

@@ -17,7 +17,7 @@
  discover-pr-forks-strategy: 'merge-current'
  discover-pr-forks-trust: 'permission'
  discover-pr-origin: 'merge-current'
- head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+ head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
  discover-tags: true
  notification-context: "beats-ci"
  repo: 'beats'
@@ -38,6 +38,9 @@
  - regex-name:
  regex: '7\.[x789]'
  case-sensitive: true
+ - regex-name:
+ regex: '7\.1\d'
+ case-sensitive: true
  - regex-name:
  regex: '8\.\d+'
  case-sensitive: true
@@ -46,13 +49,14 @@
  before: true
  prune: true
  shallow-clone: true
- depth: 3
+ depth: 10
  do-not-fetch-tags: true
  submodule:
  disable: false
  recursive: true
  parent-credentials: true
  timeout: 100
+ reference-repo: /var/lib/jenkins/.git-references/beats.git
  timeout: '15'
  use-author: true
  wipe-workspace: true

.ci/jobs/folders.yml

@@ -4,3 +4,8 @@
  name: Beats
  description: Beats
  project-type: folder
+
+- job:
+ name: Beats/Release
+ description: Jobs for release preparation
+ project-type: folder

.ci/jobs/golang-crossbuild-mbp.yml

@@ -31,7 +31,7 @@
  before: true
  prune: true
  shallow-clone: true
- depth: 4
+ depth: 10
  do-not-fetch-tags: true
  submodule:
  disable: false

.ci/jobs/packaging.yml

@@ -14,7 +14,7 @@
  discover-pr-forks-trust: 'permission'
  discover-pr-origin: 'merge-current'
  discover-tags: true
- head-filter-regex: '(master|7\.[x789]|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
+ head-filter-regex: '(master|7\.[x789]|7\.1\d|8\.\d+|PR-.*|v\d+\.\d+\.\d+)'
  disable-pr-notifications: true
  notification-context: 'beats-packaging'
  repo: 'beats'
@@ -34,6 +34,9 @@
  - regex-name:
  regex: '7\.[x789]'
  case-sensitive: true
+ - regex-name:
+ regex: '7\.1\d'
+ case-sensitive: true
  - regex-name:
  regex: '8\.\d+'
  case-sensitive: true
@@ -44,7 +47,7 @@
  before: true
  prune: true
  shallow-clone: true
- depth: 3
+ depth: 10
  do-not-fetch-tags: true
  submodule:
  disable: false

.ci/packaging.groovy

@@ -5,12 +5,14 @@
 pipeline {
  agent none
  environment {
- BASE_DIR = 'src/github.com/elastic/beats'
+ REPO = 'beats'
+ BASE_DIR = "src/github.com/elastic/${env.REPO}"
  JOB_GCS_BUCKET = 'beats-ci-artifacts'
  JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
  JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
  DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
  DOCKER_REGISTRY = 'docker.elastic.co'
+ GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
  SNAPSHOT = "true"
  PIPELINE_LOG_LEVEL = "INFO"
  }
@@ -119,6 +121,7 @@ pipeline {
  release()
  pushCIDockerImages()
  }
+ runE2ETestForPackages()
  }
  }
  stage('Package Mac OS'){
@@ -195,29 +198,97 @@ def tagAndPush(name){
  tagName = "pr-${env.CHANGE_ID}"
  }
 
- def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
- def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${tagName}"
- def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
- retry(3){
- sh(label:'Change tag and push', script: """
- docker tag ${oldName} ${newName}
- docker push ${newName}
- docker tag ${oldName} ${commitName}
- docker push ${commitName}
- """)
+
+ // supported image flavours
+ def variants = ["", "-oss", "-ubi8"]
+ variants.each { variant ->
+ def oldName = "${DOCKER_REGISTRY}/beats/${name}${variant}:${libbetaVer}"
+ def newName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${tagName}"
+ def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}${variant}:${env.GIT_BASE_COMMIT}"
+
+ def iterations = 0
+ retryWithSleep(retries: 3, seconds: 5, backoff: true)
+ iterations++
+ def status = sh(label:'Change tag and push', script: """
+ docker tag ${oldName} ${newName}
+ docker push ${newName}
+ docker tag ${oldName} ${commitName}
+ docker push ${commitName}
+ """, returnStatus: true)
+
+ if ( status > 0 && iterations < 3) {
+ error('tag and push failed, retry')
+ } else if ( status > 0 ) {
+ log(level: 'WARN', text: "${name} doesn't have ${variant} docker images. See https://github.com/elastic/beats/pull/21621")
+ }
+ }
+}
+
+def runE2ETestForPackages(){
+ def suite = ''
+
+ catchError(buildResult: 'UNSTABLE', message: 'Unable to run e2e tests', stageResult: 'FAILURE') {
+ if ("${env.BEATS_FOLDER}" == "filebeat" || "${env.BEATS_FOLDER}" == "x-pack/filebeat") {
+ suite = 'helm,fleet'
+ } else if ("${env.BEATS_FOLDER}" == "metricbeat" || "${env.BEATS_FOLDER}" == "x-pack/metricbeat") {
+ suite = ''
+ } else if ("${env.BEATS_FOLDER}" == "x-pack/elastic-agent") {
+ suite = 'fleet'
+ } else {
+ echo("Skipping E2E tests for ${env.BEATS_FOLDER}.")
+ return
+ }
+
+ triggerE2ETests(suite)
  }
 }
 
 def release(){
  withBeatsEnv(){
- dir("${env.BEATS_FOLDER}") {
- sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+ withEnv([
+ "DEV=true"
+ ]) {
+ dir("${env.BEATS_FOLDER}") {
+ sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+ }
  }
  publishPackages("${env.BEATS_FOLDER}")
  }
 }
 
+def triggerE2ETests(String suite) {
+ echo("Triggering E2E tests for ${env.BEATS_FOLDER}. Test suite: ${suite}.")
+
+ def branchName = isPR() ? "${env.CHANGE_TARGET}" : "${env.JOB_BASE_NAME}"
+ def e2eTestsPipeline = "e2e-tests/e2e-testing-mbp/${branchName}"
+
+ def parameters = [
+ booleanParam(name: 'forceSkipGitChecks', value: true),
+ booleanParam(name: 'forceSkipPresubmit', value: true),
+ booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+ string(name: 'runTestsSuites', value: suite),
+ string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
+ string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
+ string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
+ ]
+ if (isPR()) {
+ def version = "pr-${env.CHANGE_ID}"
+ parameters.push(booleanParam(name: 'USE_CI_SNAPSHOTS', value: true))
+ parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
+ parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
+ }
+
+ build(job: "${e2eTestsPipeline}",
+ parameters: parameters,
+ propagate: false,
+ wait: false
+ )
+
+ def notifyContext = "${env.GITHUB_CHECK_E2E_TESTS_NAME} for ${env.BEATS_FOLDER}"
+ githubNotify(context: "${notifyContext}", description: "${notifyContext} ...", status: 'PENDING', targetUrl: "${env.JENKINS_URL}search/?q=${e2eTestsPipeline.replaceAll('/','+')}")
+}
+
 def withMacOSEnv(Closure body){
  withEnvMask( vars: [
  [var: "KEYCHAIN_PASS", password: getVaultSecret(secret: "secret/jenkins-ci/macos-codesign-keychain").data.password],