Merge remote-tracking branch 'upstream/master' into feature/windows-10

* upstream/master: (29 commits)
  [CI] Enable build stage for arm architecture (elastic#21284)
  [BUILD][CI] fetch dependencies with retry (elastic#21614)
  Add proxy metricset (elastic#21751)
  [Filebeat][httpjson] Fix date_cursor validation (elastic#21756)
  Update endpoint-related terminology in Elastic Agent docs (elastic#21458)
  Move s3_daily_storage and s3_request metricsets to use cloudwatch input (elastic#21703)
  [Agent] Remove heartbeat from bundled beats in agent. (elastic#21602)
  [Elastic Agent] Don't perform install until after enroll (elastic#21772)
  [Elastic Agent] Update path from policy change for Kibana connection. (elastic#21804)
  Change cloud.provider from googlecloud to gcp (elastic#21775)
  Fix stress test package list (elastic#21719)
  [Elastic Agent] Remove the service installer scripts from packaging (elastic#21694)
  [CI] Support Windows-2012 in pipeline 2.0 (elastic#21338)
  Fix non-windows fields on system/filesystem (elastic#21758)
  disable TestReceiveEventsAndMetadata/TestSocketCleanup/TestReceiveNewEventsConcurrently in Windows (elastic#21750)
  Use badger code from upstream repository (elastic#21705)
  Disable writes sync in persistent cache (elastic#21754)
  Make API address and Shard ID required in Cloud Foundry settings (elastic#21759)
  [CI] Support skip-ci label (elastic#21377)
  Increase recommended memory when deploying in Cloud foundry (elastic#21755)
  ...

.ci/packaging.groovy

@@ -246,8 +246,12 @@ def runE2ETestForPackages(){
 
 def release(){
  withBeatsEnv(){
- dir("${env.BEATS_FOLDER}") {
- sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+ withEnv([
+ "DEV=true"
+ ]) {
+ dir("${env.BEATS_FOLDER}") {
+ sh(label: "Release ${env.BEATS_FOLDER} ${env.PLATFORMS}", script: 'mage package')
+ }
  }
  publishPackages("${env.BEATS_FOLDER}")
  }

.ci/scripts/install-go.sh

@@ -5,7 +5,8 @@ MSG="environment variable missing"
 GO_VERSION=${GO_VERSION:?$MSG}
 PROPERTIES_FILE=${PROPERTIES_FILE:-"go_env.properties"}
 HOME=${HOME:?$MSG}
-ARCH=$(uname -s| tr '[:upper:]' '[:lower:]')
+OS=$(uname -s| tr '[:upper:]' '[:lower:]')
+ARCH=$(uname -m| tr '[:upper:]' '[:lower:]')
 GVM_CMD="${HOME}/bin/gvm"
 
 if command -v go
@@ -19,10 +20,20 @@ then
  fi
 fi
 
+if [ "${ARCH}" == "aarch64" ] ; then
+ GVM_ARCH_SUFFIX=arm64
+elif [ "${ARCH}" == "x86_64" ] ; then
+ GVM_ARCH_SUFFIX=amd64
+elif [ "${ARCH}" == "i686" ] ; then
+ GVM_ARCH_SUFFIX=386
+else
+ GVM_ARCH_SUFFIX=arm
+fi
+
 echo "UNMET DEP: Installing Go"
 mkdir -p "${HOME}/bin"
 
-curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${ARCH}-amd64"
+curl -sSLo "${GVM_CMD}" "https://github.com/andrewkroh/gvm/releases/download/v0.2.2/gvm-${OS}-${GVM_ARCH_SUFFIX}"
 chmod +x "${GVM_CMD}"
 
 gvm ${GO_VERSION}|cut -d ' ' -f 2|tr -d '\"' > ${PROPERTIES_FILE}

CHANGELOG.next.asciidoc

@@ -25,6 +25,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Allow embedding of CAs, Certificate of private keys for anything that support TLS in ouputs and inputs https://github.com/elastic/beats/pull/21179
 - Update to Golang 1.12.1. {pull}11330[11330]
 - Disable Alibaba Cloud and Tencent Cloud metadata providers by default. {pull}13812[12812]
+- API address is a required setting in `add_cloudfoundry_metadata`. {pull}21759[21759]
 
 *Auditbeat*
 
@@ -78,6 +79,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Removed experimental modules `citrix`, `kaspersky`, `rapid7` and `tenable`. {pull}20706[20706]
 - Add support for GMT timezone offsets in `decode_cef`. {pull}20993[20993]
 - Fix parsing of Elasticsearch node name by `elasticsearch/slowlog` fileset. {pull}14547[14547]
+- API address and shard ID are required settings in the Cloud Foundry input. {pull}21759[21759]
 
 *Heartbeat*
 
@@ -95,6 +97,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix ECS compliance of user.id field in system/users metricset {pull}19019[19019]
 - Rename googlecloud stackdriver metricset to metrics. {pull}19718[19718]
 - Remove "invalid zero" metrics on Windows and Darwin, don't report linux-only memory and diskio metrics when running under agent. {pull}21457[21457]
+- Change cloud.provider from googlecloud to gcp. {pull}21775[21775]
+- API address and shard ID are required settings in the Cloud Foundry module. {pull}21759[21759]
 
 *Packetbeat*
 
@@ -366,7 +370,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix remote_write flaky test. {pull}21173[21173]
 - Visualization title fixes in aws, azure and googlecloud compute dashboards. {pull}21098[21098]
 - Add a switch to the driver definition on SQL module to use pretty names {pull}17378[17378]
+- Fix retrieving resources by ID for the azure module. {pull}21711[21711] {issue}21707[21707]
 - Use timestamp from CloudWatch API when creating events. {pull}21498[21498]
+- Report the correct windows events for system/filesystem {pull}21758[21758]
 
 *Packetbeat*
 
@@ -458,6 +464,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add istiod metricset. {pull}21519[21519]
 - Release `add_cloudfoundry_metadata` as GA. {pull}21525[21525]
 - Add support for OpenStack SSL metadata APIs in `add_cloud_metadata`. {pull}21590[21590]
+- Add proxy metricset for istio module. {pull}21751[21751]
 
 *Auditbeat*
 
@@ -750,6 +757,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Move Prometheus query & remote_write to GA. {pull}21507[21507]
 - Expand unsupported option from namespace to metrics in the azure module. {pull}21486[21486]
 - Map cloud data filed `cloud.account.id` to azure subscription. {pull}21483[21483] {issue}21381[21381]
+- Move s3_daily_storage and s3_request metricsets to use cloudwatch input. {pull}21703[21703]
 
 *Packetbeat*
 

Jenkinsfile

@@ -17,7 +17,6 @@ pipeline {
  DOCKERELASTIC_SECRET = 'secret/observability-team/ci/docker-registry/prod'
  DOCKER_COMPOSE_VERSION = "1.21.0"
  DOCKER_REGISTRY = 'docker.elastic.co'
- GOX_FLAGS = "-arch amd64"
  JOB_GCS_BUCKET = 'beats-ci-temp'
  JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
  OSS_MODULE_PATTERN = '^[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
@@ -29,7 +28,7 @@ pipeline {
  XPACK_MODULE_PATTERN = '^x-pack\\/[a-z0-9]+beat\\/module\\/([^\\/]+)\\/.*'
  }
  options {
- timeout(time: 2, unit: 'HOURS')
+ timeout(time: 3, unit: 'HOURS')
  buildDiscarder(logRotator(numToKeepStr: '20', artifactNumToKeepStr: '20', daysToKeepStr: '30'))
  timestamps()
  ansiColor('xterm')
@@ -46,6 +45,7 @@ pipeline {
  booleanParam(name: 'awsCloudTests', defaultValue: true, description: 'Run AWS cloud integration tests.')
  string(name: 'awsRegion', defaultValue: 'eu-central-1', description: 'Default AWS region to use for testing.')
  booleanParam(name: 'runAllStages', defaultValue: false, description: 'Allow to run all stages.')
+ booleanParam(name: 'armTest', defaultValue: false, description: 'Allow ARM stages.')
  booleanParam(name: 'macosTest', defaultValue: false, description: 'Allow macOS stages.')
  string(name: 'PYTEST_ADDOPTS', defaultValue: '', description: 'Additional options to pass to pytest. Use PYTEST_ADDOPTS="-k pattern" to only run tests matching the specified pattern. For retries you can use `--reruns 3 --reruns-delay 15`')
  }
@@ -103,13 +103,17 @@ pipeline {
  script {
  def mapParallelTasks = [:]
  def content = readYaml(file: 'Jenkinsfile.yml')
- content['projects'].each { projectName ->
- generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
- mapParallelTasks["${k}"] = v
+ if (content?.disabled?.when?.labels && beatsWhen(project: 'top-level', content: content?.disabled?.when)) {
+ error 'Pull Request has been configured to be disabled when there is a skip-ci label match'
+ } else {
+ content['projects'].each { projectName ->
+ generateStages(project: projectName, changeset: content['changeset']).each { k,v ->
+ mapParallelTasks["${k}"] = v
+ }
  }
+ notifyBuildReason()
+ parallel(mapParallelTasks)
  }
- notifyBuildReason()
- parallel(mapParallelTasks)
  }
  }
  }
@@ -219,23 +223,32 @@ def withBeatsEnv(Map args = [:], Closure body) {
  def withModule = args.get('withModule', false)
  def directory = args.get('directory', '')
 
- def goRoot, path, magefile, pythonEnv, testResults, artifacts
+ def goRoot, path, magefile, pythonEnv, testResults, artifacts, gox_flags
 
  if(isUnix()) {
- goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+ if (isArm() && is64arm()) {
+ // TODO: nodeOS() should support ARM
+ goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.linux.arm64"
+ gox_flags = '-arch arm'
+ } else {
+ goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${nodeOS()}.amd64"
+ gox_flags = '-arch amd64'
+ }
  path = "${env.WORKSPACE}/bin:${goRoot}/bin:${env.PATH}"
  magefile = "${WORKSPACE}/.magefile"
  pythonEnv = "${WORKSPACE}/python-env"
  testResults = '**/build/TEST*.xml'
  artifacts = '**/build/TEST*.out'
  } else {
  def chocoPath = 'C:\\ProgramData\\chocolatey\\bin'
+ def mingw64Path = 'C:\\tools\\mingw64\\bin'
  def chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts'
  goRoot = "${env.USERPROFILE}\\.gvm\\versions\\go${GO_VERSION}.windows.amd64"
- path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH}"
+ path = "${env.WORKSPACE}\\bin;${goRoot}\\bin;${chocoPath};${chocoPython3Path};${env.PATH};${mingw64Path}"
  magefile = "${env.WORKSPACE}\\.magefile"
  testResults = "**\\build\\TEST*.xml"
  artifacts = "**\\build\\TEST*.out"
+ gox_flags = '-arch amd64'
  }
 
  deleteDir()
@@ -253,7 +266,8 @@ def withBeatsEnv(Map args = [:], Closure body) {
  "PYTHON_ENV=${pythonEnv}",
  "RACE_DETECTOR=true",
  "TEST_COVERAGE=true",
- "TEST_TAGS=${env.TEST_TAGS},oracle"
+ "TEST_TAGS=${env.TEST_TAGS},oracle",
+ "GOX_FLAGS=${gox_flags}"
  ]) {
  if(isDockerInstalled()) {
  dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
@@ -270,6 +284,13 @@ def withBeatsEnv(Map args = [:], Closure body) {
  fi''')
  }
  try {
+ // Add more stability when dependencies are not accessible temporarily
+ // See https://github.com/elastic/beats/issues/21609
+ // retry/try/catch approach reports errors, let's avoid it to keep the
+ // notifications cleaner.
+ if (cmd(label: 'Download modules to local cache', script: 'go mod download', returnStatus: true) > 0) {
+ cmd(label: 'Download modules to local cache - retry', script: 'go mod download', returnStatus: true)
+ }
  body()
  } finally {
  if (archive) {
@@ -357,7 +378,9 @@ def archiveTestOutput(Map args = [:]) {
  def folder = cmd(label: 'Find system-tests', returnStdout: true, script: 'python .ci/scripts/search_system_tests.py').trim()
  log(level: 'INFO', text: "system-tests='${folder}'. If no empty then let's create a tarball")
  if (folder.trim()) {
- def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + nodeOS()
+ // TODO: nodeOS() should support ARM
+ def os_suffix = isArm() ? 'linux' : nodeOS()
+ def name = folder.replaceAll('/', '-').replaceAll('\\\\', '-').replaceAll('build', '').replaceAll('^-', '') + '-' + os_suffix
  tar(file: "${name}.tgz", archive: true, dir: folder)
  }
  }

Jenkinsfile.yml

@@ -40,10 +40,9 @@ changeset:
  - "^testing/.*"
  - "^x-pack/libbeat/.*"
 
-## Proposal
-## TBC: This will allow to configure what to do based on the PR configuration
 disabled:
  when:
- labels: ## Skip the GitHub Pull Request builds if there is a GitHub label match
- - "skip-ci"
- draft: true ## Skip the GitHub Pull Request builds with Draft PRs.
+ labels: ## Skip the GitHub Pull Request builds if any of the given GitHub labels match with the assigned labels in the PR.
+ - skip-ci
+ ## TODO: This will allow to configure what to do based on the PR configuration
+ draft: true ## Skip the GitHub Pull Request builds with Draft PRs.

NOTICE.txt

@@ -4473,12 +4473,12 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 
 --------------------------------------------------------------------------------
-Dependency : github.com/elastic/badger/v2
-Version: v2.2007.2-beats
+Dependency : github.com/dgraph-io/badger/v2
+Version: v2.2007.3-0.20201012072640-f5a7e0a1c83b
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/badger/v2@v2.2007.2-beats/LICENSE:
+Contents of probable licence file $GOMODCACHE/github.com/dgraph-io/badger/v2@v2.2007.3-0.20201012072640-f5a7e0a1c83b/LICENSE:
 
  Apache License
  Version 2.0, January 2004

auditbeat/Jenkinsfile.yml

@@ -13,9 +13,22 @@ when:
  tags: true ## for all the tags
 platform: "linux && ubuntu-18" ## default label for all the stages
 stages:
+ arm:
+ mage: "mage build unitTest"
+ platforms: ## override default label in this specific stage.
+ - "arm"
+ when: ## Override the top-level when.
+ comments:
+ - "/test auditbeat for arm"
+ labels:
+ - "arm"
+ parameters:
+ - "armTest"
+ branches: true ## for all the branches
+ tags: true ## for all the tags
  build:
  mage: "mage build test"
- crosscompile: 
+ crosscompile:
  make: "make -C auditbeat crosscompile"
  macos:
  mage: "mage build unitTest"
@@ -45,6 +58,17 @@ stages:
  - "windows-2016"
  branches: true ## for all the branches
  tags: true ## for all the tags
+ windows-2012:
+ mage: "mage build unitTest"
+ platforms: ## override default labels in this specific stage.
+ - "windows-2012-r2"
+ when: ## Override the top-level when.
+ comments:
+ - "/test auditbeat for windows-2012"
+ labels:
+ - "windows-2012"
+ branches: true ## for all the branches
+ tags: true ## for all the tags
  windows-10:
  mage: "mage build unitTest"
  platforms: ## override default labels in this specific stage.

deploy/cloudfoundry/filebeat/filebeat.yml

@@ -11,7 +11,7 @@ filebeat.inputs:
  #doppler_address: ${DOPPLER_ADDRESS}
  #uaa_address: ${UAA_ADDRESS}
  #rlp_address: ${RLP_ADDRESS}
- #shard_id: ${SHARD_ID}
+ shard_id: ${SHARD_ID}
  #version: v1
 
 

deploy/cloudfoundry/filebeat/manifest.yml

@@ -1,6 +1,6 @@
 applications:
 - name: filebeat
- memory: 256M
+ memory: 512M
  instances: 1
  buildpacks:
  - binary_buildpack

deploy/cloudfoundry/metricbeat/manifest.yml

@@ -1,6 +1,6 @@
 applications:
  - name: metricbeat
- memory: 256M
+ memory: 512M
  instances: 1
  buildpacks:
  - binary_buildpack

deploy/cloudfoundry/metricbeat/metricbeat.yml

@@ -11,7 +11,8 @@ metricbeat.modules:
  #doppler_address: ${DOPPLER_ADDRESS}
  #uaa_address: ${UAA_ADDRESS}
  #rlp_address: ${RLP_ADDRESS}
- #shard_id: ${SHARD_ID}
+ shard_id: ${SHARD_ID}
+ #version: v1
 
 
 #================================ Outputs =====================================