Merge remote-tracking branch 'upstream/master' into feature/retry-win…

…dows-7

* upstream/master: (332 commits)
  Use ECS v1.8.0 (elastic#24086)
  Add support for postgresql csv logs (elastic#23334)
  [Heartbeat] Refactor config system (elastic#23467)
  [CI] install docker-compose with retry (elastic#24069)
  Add nodes to filebeat-kubernetes.yaml ClusterRole - fixes elastic#24051 (elastic#24052)
  updating manifest files for filebeat threatintel module (elastic#24074)
  Add Zeek Signatures (elastic#23772)
  Update Beats to ECS 1.8.0 (elastic#23465)
  Support running Docker logging plugin on ARM64 (elastic#24034)
  Fix ec2 metricset fields.yml and add integration test (elastic#23726)
  Only build targz and zip versions of Beats if PACKAGES is set in agent (elastic#24060)
  [Filebeat] Add field definitions for known Netflow/IPFIX vendor fields (elastic#23773)
  [Elastic Agent] Enroll with Fleet Server (elastic#23865)
  [Filebeat] Convert logstash logEvent.action objects to strings (elastic#23944)
  [Ingest Management] Fix reloading of log level for services (elastic#24055)
  Add Agent standalone k8s manifest (elastic#23679)
  [Metricbeat][Kubernetes] Extend state_node with more conditions (elastic#23905)
  [CI] googleStorageUploadExt step (elastic#24048)
  Check fields are documented for aws metricsets (elastic#23887)
  Update go-concert to 0.1.0 (elastic#23770)
  ...

.backportrc.json

@@ -1,6 +1,6 @@
 {
   "upstream": "elastic/beats",
-  "branches": [ { "name": "7.x", "checked": true }, "7.10" ],
+  "branches": [ { "name": "7.x", "checked": true }, "7.11" ],
   "labels": ["backport"],
   "autoAssign": true,
   "prTitle": "Cherry-pick to {targetBranch}: {commitMessages}"

.ci/heartbeat-synthetics.groovy

@@ -0,0 +1,122 @@
+#!/usr/bin/env groovy
+
+@Library('apm@current') _
+
+pipeline {
+  agent { label 'ubuntu-18 && immutable' }
+  environment {
+    BASE_DIR = 'src/github.com/elastic/beats'
+    DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
+    DOCKER_REGISTRY = 'docker.elastic.co'
+    SYNTHETICS = "-synthetics"
+    PIPELINE_LOG_LEVEL = "INFO"
+    BEATS_FOLDER = "x-pack/heartbeat"
+  }
+  options {
+    timeout(time: 3, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+    disableConcurrentBuilds()
+  }
+  triggers {
+    issueCommentTrigger('(?i)^\\/packag[ing|e] synthetics$')
+  }
+  parameters {
+    booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+  }
+  stages {
+    stage('Checkout') {
+      options { skipDefaultCheckout() }
+      steps {
+        deleteDir()
+        gitCheckout(basedir: "${BASE_DIR}")
+        setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+      }
+    }
+    stage('Build and test'){
+      steps {
+        withGithubNotify(context: "Build and test") {
+          withBeatsEnv{
+            dir("${env.BEATS_FOLDER}") {
+              sh(label: 'Build and test', script: 'mage build test')
+            }
+          }
+        }
+      }
+    }
+    stage('Package Linux'){
+      environment {
+        HOME = "${env.WORKSPACE}"
+        PLATFORMS = [
+          '+all',
+          'linux/amd64'
+        ].join(' ')
+      }
+      steps {
+        withGithubNotify(context: "Packaging Linux ${BEATS_FOLDER}") {
+          release()
+          pushCIDockerImages()
+        }
+      }
+    }
+  }
+}
+
+def pushCIDockerImages(){
+  catchError(buildResult: 'UNSTABLE', message: 'Unable to push Docker images', stageResult: 'FAILURE') {
+    tagAndPush('heartbeat')
+  }
+}
+
+def tagAndPush(name){
+  def libbetaVer = sh(label: 'Get libbeat version', script: 'grep defaultBeatVersion ${BASE_DIR}/libbeat/version/version.go|cut -d "=" -f 2|tr -d \\"', returnStdout: true)?.trim()
+
+  def tagName = "${libbetaVer}"
+  def oldName = "${DOCKER_REGISTRY}/beats/${name}:${libbetaVer}"
+  def newName = "${DOCKER_REGISTRY}/observability-ci/${name}:${libbetaVer}${env.SYNTHETICS}"
+  def commitName = "${DOCKER_REGISTRY}/observability-ci/${name}:${env.GIT_BASE_COMMIT}"
+  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
+  retry(3){
+    sh(label:'Change tag and push', script: """
+      docker tag ${oldName} ${newName}
+      docker push ${newName}
+      docker tag ${oldName} ${commitName}
+      docker push ${commitName}
+    """)
+  }
+}
+
+def release(){
+  withBeatsEnv(){
+    dir("${env.BEATS_FOLDER}") {
+      sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+    }
+  }
+}
+
+/**
+* There is a specific folder structure in https://staging.elastic.co/ and https://artifacts.elastic.co/downloads/
+* therefore the storage bucket in GCP should follow the same folder structure.
+* This is required by https://github.com/elastic/beats-tester
+* e.g.
+* baseDir=name -> return name
+* baseDir=name1/name2/name3-> return name2
+*/
+def getBeatsName(baseDir) {
+  return baseDir.replace('x-pack/', '')
+}
+
+def withBeatsEnv(Closure body) {
+  withMageEnv(){
+    withEnv([
+      "PYTHON_ENV=${WORKSPACE}/python-env"
+    ]) {
+      dir("${env.BASE_DIR}"){
+        body()
+      }
+    }
+  }
+}

.ci/jobs/beats-schedule-weekly.yml

@@ -0,0 +1,27 @@
+---
+- job:
+    name: beats-schedule-weekly
+    display-name: Jobs scheduled weekly (Sunday)
+    description: Jobs scheduled weekly (Sunday)
+    view: Beats
+    project-type: pipeline
+    parameters:
+      - string:
+          name: branch_specifier
+          default: master
+          description: the Git branch specifier to build
+    pipeline-scm:
+      script-path: .ci/schedule-weekly.groovy
+      scm:
+        - git:
+            url: git@github.com:elastic/beats.git
+            refspec: +refs/heads/*:refs/remotes/origin/*
+            wipe-workspace: 'True'
+            name: origin
+            shallow-clone: true
+            credentials-id: f6c7695a-671e-4f4f-a331-acdce44ff9ba
+            reference-repo: /var/lib/jenkins/.git-references/beats.git
+            branches:
+              - $branch_specifier
+    triggers:
+      - timed: 'H H(1-4) * * 0'

.ci/packaging.groovy

@@ -17,6 +17,7 @@ pipeline {
     JOB_GCS_BUCKET = 'beats-ci-artifacts'
     JOB_GCS_BUCKET_STASH = 'beats-ci-temp'
     JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
+    JOB_GCS_EXT_CREDENTIALS = 'beats-ci-gcs-plugin-file-credentials'
     DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
     DOCKER_REGISTRY = 'docker.elastic.co'
     GITHUB_CHECK_E2E_TESTS_NAME = 'E2E Tests'
@@ -40,6 +41,7 @@ pipeline {
   parameters {
     booleanParam(name: 'macos', defaultValue: false, description: 'Allow macOS stages.')
     booleanParam(name: 'linux', defaultValue: true, description: 'Allow linux stages.')
+    booleanParam(name: 'arm', defaultValue: true, description: 'Allow ARM stages.')
   }
   stages {
     stage('Filter build') {
@@ -83,12 +85,13 @@ pipeline {
               }
             }
             setEnvVar("GO_VERSION", readFile("${BASE_DIR}/.go-version").trim())
+            // Stash without any build/dependencies context to support different architectures.
+            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
             withMageEnv(){
               dir("${BASE_DIR}"){
                 setEnvVar('BEAT_VERSION', sh(label: 'Get beat version', script: 'make get-version', returnStdout: true)?.trim())
               }
             }
-            stashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
           }
         }
         stage('Build Packages'){
@@ -172,12 +175,74 @@ pipeline {
                 }
                 steps {
                   withGithubNotify(context: "Packaging MacOS ${BEATS_FOLDER}") {
-                    deleteDir()
+                    deleteWorkspace()
                     withMacOSEnv(){
                       release()
                     }
                   }
                 }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
+              }
+            }
+          }
+        }
+        stage('Build Packages ARM'){
+          matrix {
+            axes {
+              axis {
+                name 'BEATS_FOLDER'
+                values (
+                  'auditbeat',
+                  'filebeat',
+                  'heartbeat',
+                  'journalbeat',
+                  'metricbeat',
+                  'packetbeat',
+                  'x-pack/auditbeat',
+                  'x-pack/dockerlogbeat',
+                  'x-pack/elastic-agent',
+                  'x-pack/filebeat',
+                  'x-pack/heartbeat',
+                  'x-pack/metricbeat',
+                  'x-pack/packetbeat'
+                )
+              }
+            }
+            stages {
+              stage('Package Docker images for linux/arm64'){
+                agent { label 'arm' }
+                options { skipDefaultCheckout() }
+                when {
+                  beforeAgent true
+                  expression {
+                    return params.arm
+                  }
+                }
+                environment {
+                  HOME = "${env.WORKSPACE}"
+                  PACKAGES = "docker"
+                  PLATFORMS = [
+                    'linux/arm64',
+                  ].join(' ')
+                }
+                steps {
+                  withGithubNotify(context: "Packaging linux/arm64 ${BEATS_FOLDER}") {
+                    deleteWorkspace()
+                    release()
+                    pushCIDockerImages()
+                  }
+                }
+                post {
+                  always {
+                    // static workers require this
+                    deleteWorkspace()
+                  }
+                }
               }
             }
           }
@@ -339,14 +404,14 @@ def triggerE2ETests(String suite) {
     booleanParam(name: 'forceSkipGitChecks', value: true),
     booleanParam(name: 'forceSkipPresubmit', value: true),
     booleanParam(name: 'notifyOnGreenBuilds', value: !isPR()),
+    booleanParam(name: 'BEATS_USE_CI_SNAPSHOTS', value: true),
     string(name: 'runTestsSuites', value: suite),
     string(name: 'GITHUB_CHECK_NAME', value: env.GITHUB_CHECK_E2E_TESTS_NAME),
     string(name: 'GITHUB_CHECK_REPO', value: env.REPO),
     string(name: 'GITHUB_CHECK_SHA1', value: env.GIT_BASE_COMMIT),
   ]
   if (isPR()) {
     def version = "pr-${env.CHANGE_ID}"
-    parameters.push(booleanParam(name: 'ELASTIC_AGENT_USE_CI_SNAPSHOTS', value: true))
     parameters.push(string(name: 'ELASTIC_AGENT_VERSION', value: "${version}"))
     parameters.push(string(name: 'METRICBEAT_VERSION', value: "${version}"))
   }
@@ -385,14 +450,11 @@ def publishPackages(baseDir){
   uploadPackages("${bucketUri}/${beatsFolderName}", baseDir)
 }
 
-def uploadPackages(bucketUri, baseDir){
-  googleStorageUpload(bucket: bucketUri,
-    credentialsId: "${JOB_GCS_CREDENTIALS}",
-    pathPrefix: "${baseDir}/build/distributions/",
-    pattern: "${baseDir}/build/distributions/**/*",
-    sharedPublicly: true,
-    showInline: true
-  )
+def uploadPackages(bucketUri, beatsFolder){
+  googleStorageUploadExt(bucket: bucketUri,
+    credentialsId: "${JOB_GCS_EXT_CREDENTIALS}",
+    pattern: "${beatsFolder}/build/distributions/**/*",
+    sharedPublicly: true)
 }
 
 /**
@@ -408,14 +470,43 @@ def getBeatsName(baseDir) {
 }
 
 def withBeatsEnv(Closure body) {
+  unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
+  fixPermissions()
   withMageEnv(){
     withEnv([
       "PYTHON_ENV=${WORKSPACE}/python-env"
     ]) {
-      unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET_STASH}", credentialsId: "${JOB_GCS_CREDENTIALS}")
       dir("${env.BASE_DIR}"){
         body()
       }
     }
   }
 }
+
+/**
+* This method fixes the filesystem permissions after the build has happenend. The reason is to
+* ensure any non-ephemeral workers don't have any leftovers that could cause some environmental
+* issues.
+*/
+def deleteWorkspace() {
+  catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+    fixPermissions()
+    deleteDir()
+  }
+}
+
+def fixPermissions() {
+  if(isUnix()) {
+    catchError(buildResult: 'SUCCESS', stageResult: 'SUCCESS') {
+      dir("${env.BASE_DIR}") {
+        if (fileExists('script/fix_permissions.sh')) {
+          sh(label: 'Fix permissions', script: """#!/usr/bin/env bash
+            set +x
+            source ./dev-tools/common.bash
+            docker_setup
+            script/fix_permissions.sh ${WORKSPACE}""", returnStatus: true)
+        }
+      }
+    }
+  }
+}

.ci/schedule-weekly.groovy

@@ -0,0 +1,33 @@
+@Library('apm@current') _
+
+pipeline {
+  agent none
+  environment {
+    NOTIFY_TO = credentials('notify-to')
+    PIPELINE_LOG_LEVEL = 'INFO'
+  }
+  options {
+    timeout(time: 1, unit: 'HOURS')
+    buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20'))
+    timestamps()
+    ansiColor('xterm')
+    disableResume()
+    durabilityHint('PERFORMANCE_OPTIMIZED')
+  }
+  triggers {
+    cron('H H(1-4) * * 0')
+  }
+  stages {
+    stage('Nighly beats builds') {
+      steps {
+        build(quietPeriod: 0, job: 'Beats/beats/master', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+        build(quietPeriod: 1000, job: 'Beats/beats/7.x', parameters: [booleanParam(name: 'awsCloudTests', value: true)], wait: false, propagate: false)
+      }
+    }
+  }
+  post {
+    cleanup {
+      notifyBuildResult(prComment: false)
+    }
+  }
+}