Merge remote-tracking branch 'upstream/master' into feature/ci-pipeli…

…ne-2.0 * upstream/master: (39 commits) [ITs] Revert healthcheck for elasticsearchssl service to the previous behaviour (elastic#20558) [Heartbeat] Fix packaging (elastic#20566) [Heartbeat] Add Magefile to X-Pack (elastic#20549) [Packetbeat] Add "network" to event.category (elastic#20392) fix typo in docs (elastic#20541) Add service resource in k8s cluster role (elastic#20546) Update Golang version to 1.14.7 (elastic#20508) Add missing inputs to filebeat spec (elastic#20388) add warning log in aws and googlecloud module for API cost (elastic#20523) Fix fortinet.firewall.mem value to be interpreted as integer (elastic#19335) [CI] add more resilience (elastic#20505) [JJBB] fix credentials with a service account for golang-crossbuild (elastic#20537) [ITs] change healthcheck for elasticsearch (elastic#20514) [JJBB] fix credentials with a service account (elastic#20535) chore(ci): use build step for checking if is PR (elastic#20536) [CI] runbld project name (elastic#20466) Add panw.panos.endreason field (elastic#18705) [Filebeat] Fix PANW field spelling "veredict" to "verdict" (elastic#18808) Fix typo in netflow module docs (elastic#18992) Modified auditd ingest pipeline to handle node=hostname (elastic#19659) ...
v1v · Aug 12, 2020 · 23c69ce · 23c69ce
2 parents 2f21086 + cf959bf
commit 23c69ce
Show file tree

Hide file tree

Showing 218 changed files with 5,565 additions and 1,709 deletions.
diff --git a/.ci/jobs/golang-crossbuild-mbp.yml b/.ci/jobs/golang-crossbuild-mbp.yml
@@ -16,7 +16,7 @@
  notification-context: 'beats-ci'
  repo: golang-crossbuild
  repo-owner: elastic
- credentials-id: github-app-beats-ci
+ credentials-id: 2a9602aa-ab9f-4e52-baf3-b71ca88469c7-UserAndToken
  ssh-checkout:
  credentials: f6c7695a-671e-4f4f-a331-acdce44ff9ba
  build-strategies:

diff --git a/.ci/packaging.groovy b/.ci/packaging.groovy
@@ -79,7 +79,7 @@ pipeline {
  'x-pack/dockerlogbeat',
  'x-pack/filebeat',
  'x-pack/functionbeat',
- // 'x-pack/heartbeat',
+ 'x-pack/heartbeat',
  // 'x-pack/journalbeat',
  'x-pack/metricbeat',
  // 'x-pack/packetbeat',
@@ -161,7 +161,6 @@ def pushCIDockerImages(){
  } else if ("${env.BEATS_FOLDER}" == "filebeat") {
  tagAndPush('filebeat-oss')
  } else if ("${env.BEATS_FOLDER}" == "heartbeat"){
- tagAndPush('heartbeat')
  tagAndPush('heartbeat-oss')
  } else if ("${env.BEATS_FOLDER}" == "journalbeat"){
  tagAndPush('journalbeat')
@@ -177,6 +176,8 @@ def pushCIDockerImages(){
  tagAndPush('elastic-agent')
  } else if ("${env.BEATS_FOLDER}" == "x-pack/filebeat"){
  tagAndPush('filebeat')
+ } else if ("${env.BEATS_FOLDER}" == "x-pack/heartbeat"){
+ tagAndPush('heartbeat')
  } else if ("${env.BEATS_FOLDER}" == "x-pack/metricbeat"){
  tagAndPush('metricbeat')
  }
@@ -190,7 +191,7 @@ def tagAndPush(name){
  }
 
  def tagName = "${libbetaVer}"
- if (env.CHANGE_ID?.trim()) {
+ if (isPR()) {
  tagName = "pr-${env.CHANGE_ID}"
  }
 

diff --git a/.ci/scripts/install-tools.bat b/.ci/scripts/install-tools.bat
@@ -4,7 +4,7 @@ set PATH=%WORKSPACE%\bin;C:\ProgramData\chocolatey\bin;C:\tools\mingw64\bin;%PAT
 
 where /q curl
 IF ERRORLEVEL 1 (
- choco install curl -y --no-progress --skipdownloadcache
+  choco install curl -y --no-progress --skipdownloadcache
 )
 mkdir %WORKSPACE%\bin
 
@@ -28,16 +28,22 @@ go get github.com/magefile/mage
 mage -version
 where mage
 
-if not exist C:\Python38\python.exe (
- REM Install python 3.8.
- choco install python -y -r --no-progress --version 3.8.2 || echo ERROR && exit /b
+IF NOT EXIST C:\Python38\python.exe (
+ REM Install python 3.8
+ choco install python -y -r --no-progress --version 3.8.5
+ IF NOT ERRORLEVEL 0 (
+ exit /b 1
+ )
 )
 python --version
 where python
 
-if not exist C:\tools\mingw64\bin\gcc.exe (
+IF NOT EXIST C:\tools\mingw64\bin\gcc.exe (
  REM Install mingw 5.3.0
- choco install mingw -y -r --no-progress --version 5.3.0 || echo ERROR && exit /b
+ choco install mingw -y -r --no-progress --version 5.3.0
+ IF NOT ERRORLEVEL 0 (
+ exit /b 1
+ )
 )
 gcc --version
 where gcc
diff --git a/.ci/windows.groovy b/.ci/windows.groovy
@@ -527,7 +527,8 @@ def dumpFilteredEnvironment(){
  echo "PROCESSES: ${env.PROCESSES}"
  echo "TIMEOUT: ${env.TIMEOUT}"
  echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}"
- echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}"
+ echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}"
+ echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}"
  echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}"
  echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
  echo "STRESS_TESTS: ${env.STRESS_TESTS}"

diff --git a/.go-version b/.go-version
@@ -1 +1 @@
-1.14.4
+1.14.7
diff --git a/CHANGELOG-developer.next.asciidoc b/CHANGELOG-developer.next.asciidoc
@@ -98,3 +98,4 @@ The list below covers the major changes between 7.0.0-rc2 and master only.
 - Remove vendor folder from repository. {pull}18655[18655]
 - Added SQL helper that can be used from any Metricbeat module {pull}18955[18955]
 - Update Go version to 1.14.4. {pull}19753[19753]
+- Update Go version to 1.14.7. {pull}20508[20508]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -63,6 +63,8 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Adds Gsuite Groups support. {pull}19725[19725]
 - Move file metrics to dataset endpoint {pull}19977[19977]
 - Add `while_pattern` type to multiline reader. {pull}19662[19662]
+- Tracking session end reason in panw module. {pull}18705[18705]
+- Fix PANW field spelling "veredict" to "verdict" on event.action {pull}18808[18808]
 
 *Heartbeat*
 
@@ -155,6 +157,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Output errors when Kibana index pattern setup fails. {pull}20121[20121]
 - Fix issue in autodiscover that kept inputs stopped after config updates. {pull}20305[20305]
 - Log debug message if the Kibana dashboard can not be imported from the archive because of the invalid archive directory structure {issue}12211[12211], {pull}13387[13387]
+- Add service resource in k8s cluster role. {pull}20546[20546]
 
 *Auditbeat*
 
@@ -173,6 +176,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 
 *Filebeat*
 
+- Fix mapping of fortinet.firewall.mem as integer. {pull}19335[19335]
 - Ensure all zeek timestamps include millisecond precision. {issue}14599[14599] {pull}16766[16766]
 - Fix s3 input hanging with GetObjectRequest API call by adding context_timeout config. {issue}15502[15502] {pull}15590[15590]
 - Add shared_credential_file to cloudtrail config {issue}15652[15652] {pull}15656[15656]
@@ -232,6 +236,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix millisecond timestamp normalization issues in CrowdStrike module {issue}20035[20035], {pull}20138[20138]
 - Fix support for message code 106100 in Cisco ASA and FTD. {issue}19350[19350] {pull}20245[20245]
 - Fix `fortinet` setting `event.timezone` to the system one when no `tz` field present {pull}20273[20273]
+- Fix `okta` geoip lookup in pipeline for `destination.ip` {pull}20454[20454]
+- Fix mapping exception in the `googlecloud/audit` dataset pipeline. {issue}18465[18465] {pull}20465[20465]
+- Fix `cisco` asa and ftd parsing of messages 106102 and 106103. {pull}20469[20469]
+- Improve validation checks for Azure configuration {issue}20369[20369] {pull}20389[20389]
+- Fix event.kind for system/syslog pipeline {issue}20365[20365] {pull}20390[20390]
 
 *Heartbeat*
 
@@ -305,11 +314,13 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Modify doc for app_insights metricset to contain example of config. {pull}20185[20185]
 - Add required option for `metrics` in app_insights. {pull}20406[20406]
 - Groups same timestamp metric values to one event in the app_insights metricset. {pull}20403[20403]
+- Updates vm_compute metricset with more info on guest metrics. {pull}20448[20448]
 
 *Packetbeat*
 
 - Enable setting promiscuous mode automatically. {pull}11366[11366]
 - Fix process monitoring when ipv6 is disabled under Linux. {issue}19941[19941] {pull}19945[19945]
+- Add "network" to event.category {issue}20364[20364] {pull}20392[20392]
 
 *Winlogbeat*
 
@@ -366,9 +377,12 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support to trim captured values in the dissect processor. {pull}19464[19464]
 - Added the `max_cached_sessions` option to the script processor. {pull}19562[19562]
 - Add support for DNS over TLS for the dns_processor. {pull}19321[19321]
+- Add minimum cache TTL for successful DNS responses. {pull}18986[18986]
 - Set index.max_docvalue_fields_search in index template to increase value to 200 fields. {issue}20215[20215]
+- Add leader election for Kubernetes autodiscover. {pull}20281[20281]
 - Add capability of enriching process metadata with contianer id also for non-privileged containers in `add_process_metadata` processor. {pull}19767[19767]
 
+
 *Auditbeat*
 
 - Reference kubernetes manifests include configuration for auditd and enrichment with kubernetes metadata. {pull}17431[17431]
@@ -502,10 +516,14 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add experimental dataset squid/log for Squid Proxy Server logs {pull}19713[19713]
 - Add experimental dataset zscaler/zia for Zscaler Internet Access logs {pull}19713[19713]
 - Add initial support for configurable file identity tracking. {pull}18748[18748]
+- Add support for reading auditd logs that are prefixed with `node=`. {pull}19659[19659]
 - Add event.ingested for CrowdStrike module {pull}20138[20138]
 - Add support for additional fields and FirewallMatchEvent type events in CrowdStrike module {pull}20138[20138]
 - Add event.ingested for Suricata module {pull}20220[20220]
+- Add support for custom header and headersecret for filebeat http_endpoint input {pull}20435[20435]
 - Add event.ingested to all Filebeat modules. {pull}20386[20386]
+- Return error when log harvester tries to open a named pipe. {issue}18682[18682] {pull}20450[20450]
+
 
 *Heartbeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -1024,6 +1024,9 @@ def installTools() {
  retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
  retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") }
  } else {
+ // Install python3 with the specific step, even though install-tools.bat will verify if it's there anyway.
+ // TODO: as soon as python3 is installed in the CI Workers we will be able to remove the line below.
+ retryWithSleep(retries: i, seconds: 5, backoff: true){ installTools([ [tool: 'python3', version: '3.8', exclude: 'rc'] ]) }
  retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
  }
 }
@@ -1088,7 +1091,8 @@ def dumpFilteredEnvironment(){
  echo "PROCESSES: ${env.PROCESSES}"
  echo "TIMEOUT: ${env.TIMEOUT}"
  echo "PYTHON_TEST_FILES: ${env.PYTHON_TEST_FILES}"
- echo "NOSETESTS_OPTIONS: ${env.NOSETESTS_OPTIONS}"
+ echo "PYTEST_ADDOPTS: ${env.PYTEST_ADDOPTS}"
+ echo "PYTEST_OPTIONS: ${env.PYTEST_OPTIONS}"
  echo "TEST_ENVIRONMENT: ${env.TEST_ENVIRONMENT}"
  echo "SYSTEM_TESTS: ${env.SYSTEM_TESTS}"
  echo "STRESS_TESTS: ${env.STRESS_TESTS}"
@@ -1422,23 +1426,24 @@ def junitAndStore(Map params = [:]){
 def runbld() {
  catchError(buildResult: 'SUCCESS', message: 'runbld post build action failed.') {
  if (stashedTestReports) {
+ def jobName = isPR() ? 'elastic+beats+pull-request' : 'elastic+beats'
+ deleteDir()
+ unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}")
  dir("${env.BASE_DIR}") {
- sh(label: 'Prepare workspace context',
- script: 'find . -type f -name "TEST*.xml" -path "*/build/*" -delete')
  // Unstash the test reports
  stashedTestReports.each { k, v ->
  dir(k) {
  unstash(v)
  }
  }
- sh(label: 'Process JUnit reports with runbld',
- script: '''\
- cat >./runbld-script <<EOF
- echo "Processing JUnit reports with runbld..."
- EOF
- /usr/local/bin/runbld ./runbld-script
- '''.stripIndent()) // stripIdent() requires '''/
  }
+ sh(label: 'Process JUnit reports with runbld',
+ script: """\
+ cat >./runbld-script <<EOF
+ echo "Processing JUnit reports with runbld..."
+ EOF
+ /usr/local/bin/runbld ./runbld-script --job-name ${jobName}
+ """.stripIndent()) // stripIdent() requires '''/
  }
  }
 }
diff --git a/Makefile b/Makefile
@@ -18,7 +18,7 @@ XPACK_SUFFIX=x-pack/
 # PROJECTS_XPACK_PKG is a list of Beats that have independent packaging support
 # in the x-pack directory (rather than having the OSS build produce both sets
 # of artifacts). This will be removed once we complete the transition.
-PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/metricbeat x-pack/winlogbeat
+PROJECTS_XPACK_PKG=x-pack/auditbeat x-pack/dockerlogbeat x-pack/filebeat x-pack/heartbeat x-pack/metricbeat x-pack/winlogbeat
 # PROJECTS_XPACK_MAGE is a list of Beats whose primary build logic is based in
 # Mage. For compatibility with CI testing these projects support a subset of the
 # makefile targets. After all Beats converge to primarily using Mage we can

diff --git a/auditbeat/Dockerfile b/auditbeat/Dockerfile
@@ -1,4 +1,4 @@
-FROM golang:1.14.4
+FROM golang:1.14.7
 
 RUN \
  apt-get update \

diff --git a/deploy/kubernetes/metricbeat-kubernetes.yaml b/deploy/kubernetes/metricbeat-kubernetes.yaml
@@ -346,6 +346,7 @@ rules:
  - events
  - pods
  - secrets
+ - services
  verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
  resources:
@@ -367,6 +368,12 @@ rules:
  - "/metrics"
  verbs:
  - get
+- apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - '*'
 ---
 apiVersion: v1
 kind: ServiceAccount

diff --git a/deploy/kubernetes/metricbeat/metricbeat-role.yaml b/deploy/kubernetes/metricbeat/metricbeat-role.yaml
@@ -12,6 +12,7 @@ rules:
  - events
  - pods
  - secrets
+ - services
  verbs: ["get", "list", "watch"]
 - apiGroups: ["extensions"]
  resources:
@@ -33,3 +34,9 @@ rules:
  - "/metrics"
  verbs:
  - get
+- apiGroups:
+ - coordination.k8s.io
+ resources:
+ - leases
+ verbs:
+ - '*'
diff --git a/dev-tools/mage/check.go b/dev-tools/mage/check.go
@@ -43,12 +43,12 @@ import (
 // if it finds any modifications. If executed in in verbose mode it will write
 // the results of 'git diff' to stdout to indicate what changes have been made.
 //
-// It checks the file permissions of nosetests test cases and YAML files.
+// It checks the file permissions of python test cases and YAML files.
 // It checks .go source files using 'go vet'.
 func Check() error {
  fmt.Println(">> check: Checking source code for common problems")
 
- mg.Deps(GoVet, CheckNosetestsNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat)
+ mg.Deps(GoVet, CheckPythonTestNotExecutable, CheckYAMLNotExecutable, CheckDashboardsFormat)
 
  changes, err := GitDiffIndex()
  if err != nil {
@@ -124,16 +124,15 @@ func GitDiff() error {
  return err
 }
 
-// CheckNosetestsNotExecutable checks that none of the nosetests files are
-// executable. Nosetests silently skips executable .py files and we don't want
-// this to happen.
-func CheckNosetestsNotExecutable() error {
+// CheckPythonTestNotExecutable checks that none of the python test files are
+// executable. They are silently skipped and we don't want this to happen.
+func CheckPythonTestNotExecutable() error {
  if runtime.GOOS == "windows" {
  // Skip windows because it doesn't have POSIX permissions.
  return nil
  }
 
- tests, err := FindFiles(nosetestsTestFiles...)
+ tests, err := FindFiles(pythonTestFiles...)
  if err != nil {
  return err
  }
@@ -151,7 +150,7 @@ func CheckNosetestsNotExecutable() error {
  }
 
  if len(executableTestFiles) > 0 {
- return errors.Errorf("nosetests files cannot be executable because "+
+ return errors.Errorf("python test files cannot be executable because "+
  "they will be skipped. Fix permissions of %v", executableTestFiles)
  }
  return nil