Merge branch 'master' into huaweicloud-metadata

CHANGELOG.asciidoc

@@ -3,10 +3,27 @@
 :issue: https://github.com/elastic/beats/issues/
 :pull: https://github.com/elastic/beats/pull/
 
+[[release-notes-8.0.0-alpha2]]
+=== Beats version 8.0.0-alpha2
+
+Changes will be described in a later alpha / beta.
+
 [[release-notes-8.0.0-alpha1]]
 === Beats version 8.0.0-alpha1
 
 Changes will be described in a later alpha / beta.
+[[release-notes-7.14.2]]
+=== Beats version 7.14.2
+https://github.com/elastic/beats/compare/v7.14.1...v7.14.2[View commits]
+
+==== Bugfixes
+
+*Filebeat*
+
+- Auditd module: Fix the top exec commands dashboard visualization. {pull}27638[27638]
+- Store offset in `log.offset` field of events from the filestream input. {pull}27688[27688]
+- Fix `httpjson` input rate limit processing and documentation. {pull}27739[27739]
+
 [[release-notes-7.14.1]]
 === Beats version 7.14.1
 https://github.com/elastic/beats/compare/v7.14.0...v7.14.1[View commits]

CHANGELOG.next.asciidoc

@@ -85,6 +85,10 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add option for S3 input to work without SQS notification {issue}18205[18205] {pull}27332[27332]
 - Fix Crowdstrike ingest pipeline that was creating flattened `process` fields. {issue}27622[27622] {pull}27623[27623]
 - Rename `log.path` to `log.file.path` in filestream to be consistent with `log` input and ECS. {pull}27761[27761]
+- Removes old module aliases for `googlecloud` (moved to gcp) and `apache2` (moved to apache). {pull}27919[27919]
+- Removes old module name aliases (gsuite) and removing old cyberark module in favor of the new cyberarkpas{pull}27915[27915]
+- Only filesets that are explicitly configured will be enabled. {issue}17256[17256] {pull}27526[27526]
+- All filesets are disabled in the default configuration. {issue}17256[17256] {pull}27762[27762]
 
 *Heartbeat*
 - Remove long deprecated `watch_poll` functionality. {pull}27166[27166]
@@ -110,6 +114,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add support for kube-state-metrics v2.0.0 {pull}27552[27552]
 - Add User-Agent header to HTTP requests. {issue}18160[18160] {pull}27509[27509]
 - Errors should be thrown as errors. Metricsets inside Metricbeat will now throw errors as the `error` log level. {pull}27804[27804]
+- Remove deprecated fields in Docker module. {issue}11835[11835] {pull}27933[27933]
 
 *Packetbeat*
 
@@ -209,7 +214,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Do not try to load ILM policy if `check_exists` is `false`. {pull}27508[27508] {issue}26322[26322]
 - Fix bug with cgroups hierarchy override path in cgroups {pull}27620[27620]
 - Beat `setup kibana` command may use the elasticsearch API key defined in `output.elasticsearch.api_key`. {issue}24015[24015] {pull}27540[27540]
+- Fix `decode_xml` handling of array merging when using `to_lower: true`. {pull}27922[27922]
 - Seperate namespaces for V1 and V2 controller paths {pull}27676[27676]
+- Beats dashboards use custom index when `setup.dashboards.index` is set. {issue}21232[21232] {pull}27901[27901]
 
 *Auditbeat*
 
@@ -305,15 +312,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix Zeek dashboard reference to `zeek.ssl.server.name` field. {pull}21696[21696]
 - Fix for `field [source] not present as part of path [source.ip]` error in azure pipelines. {pull}22377[22377]
 - Drop aws.vpcflow.pkt_srcaddr and aws.vpcflow.pkt_dstaddr when equal to "-". {pull}22721[22721] {issue}22716[22716]
-- Convert the o365 module's `client.port` and `source.port` to numbers (from strings) in events. {pull}22939[22939]
-- Fix gcp/vpcflow module error where input type was defaulting to file. {pull}24719[24719]
-- Fix s3 input when there is a blank line in the log file. {pull}25357[25357]
 - Fixes the Snyk module to work with the new API changes. {pull}27358[27358]
 - Fixes a bug in `http_endpoint` that caused numbers encoded as strings. {issue}27382[27382] {pull}27480[27480]
 - Update indentation for azure filebeat configuration. {pull}26604[26604]
-- Auditd: Fix Top Exec Commands dashboard visualization. {pull}27638[27638]
-- Store offset in `log.offset` field of events from the filestream input. {pull}27688[27688]
-- Fix `httpjson` input rate limit processing and documentation. {pull}[]
 - Update Filebeat compatibility function to remove processor description field on ES < 7.9.0 {pull}27774[27774]
 - Make filestream events ECS compliant. {issue}27776[27776]
 
@@ -528,6 +529,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - The Kafka support library Sarama has been updated to 1.29.1. {pull}27717[27717]
 - Kafka is now supported up to version 2.8.0. {pull}27720[27720]
 - Add Huawei Cloud provider to add_cloud_metadata. {pull}27607[27607]
+- Add default seccomp policy for linux arm64. {pull}27955[27955]
 
 *Auditbeat*
 
@@ -754,7 +756,9 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `join` and `sprintf` functions to `httpjson` input. {pull}27735[27735]
 - Improve memory usage of line reader of `log` and `filestream` input. {pull}27782[27782]
 - Add `ignore_empty_value` flag to `httpjson` `split` processor. {pull}27880[27880]
-
+- Update Cisco ASA/FTD ingest pipeline grok/dissect patterns for multiple message IDs. {issue}26869[26869] {pull}26879[26879]
+- Add write access to `url.value` from `request.transforms` in `httpjson` input. {pull}27937[27937]
+- Add Base64 encoded HMAC and UUID template functions to `httpjson` input {pull}27873[27873]
 
 *Heartbeat*
 

NOTICE.txt

@@ -7351,11 +7351,11 @@ OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/go-seccomp-bpf
-Version: v1.1.0
+Version: v1.2.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@v1.1.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/go-seccomp-bpf@v1.2.0/LICENSE.txt:
 
 
  Apache License

auditbeat/docs/fields.asciidoc

@@ -17137,18 +17137,6 @@ type: keyword
 
 --
 
-*`kubernetes.container.image`*::
-+
---
-Kubernetes container image
-
-
-type: alias
-
-alias to: container.image.name
-
---
-
 [[exported-fields-process]]
 == Process fields
 

dev-tools/ecs-migration.yml

@@ -54,7 +54,7 @@
  alias6: true
  alias: true
 
-- from: docker.container.labels  # TODO: How to map these?
+- from: docker.container.labels # TODO: How to map these?
  to: container.labels
  alias6: false
  alias: true
@@ -115,8 +115,8 @@
 
 - from: source
  to:
- - log.file.path
- - log.source.address
+  - log.file.path
+  - log.source.address
  alias: false
  beat: filebeat
 
@@ -428,7 +428,7 @@
  beat: filebeat
 
 - from: suricata.eve.timestamp
- to: '@timestamp'
+ to: "@timestamp"
  alias: true
  beat: filebeat
 
@@ -476,7 +476,7 @@
  beat: filebeat
 
 - from: system.auth.timestamp
- to: '@timestamp'
+ to: "@timestamp"
  alias: true
  beat: filebeat
 
@@ -560,155 +560,6 @@
  alias: true
  beat: filebeat
 
-## Apache module
-
-- from: apache2.access.remote_ip
- to: source.address
- alias: true
- beat: filebeat
-
-- from: apache2.access.user_name
- to: user.name
- alias: true
- beat: filebeat
-
-- from: apache2.access.method
- to: http.request.method
- alias: true
- beat: filebeat
-
-- from: apache2.access.url
- to: url.original
- alias: true
- beat: filebeat
-
-- from: apache2.access.http_version
- to: http.version
- alias: true
- beat: filebeat
-
-- from: apache2.access.response_code
- to: http.response.status_code
- alias: true
- beat: filebeat
-
-- from: apache2.access.referrer
- to: http.request.referrer
- alias: true
- beat: filebeat
-
-- from: apache2.access.agent
- to: user_agent.original
- alias: true
- beat: filebeat
-
-- from: apache2.access.body_sent.bytes
- to: http.response.body.bytes
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.continent_name
- to: source.geo.continent_name
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.country_iso_code
- to: source.geo.country_iso_code
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.location
- to: source.geo.location
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.region_name
- to: source.geo.region_name
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.city_name
- to: source.geo.city_name
- alias: true
- beat: filebeat
-
-- from: apache2.access.geoip.region_iso_code
- to: source.geo.region_iso_code
- alias: true
- beat: filebeat
-
-- from: apache2.access.user_agent.original
- to: user_agent.original
- alias: true
- beat: filebeat
-- from: apache2.access.user_agent.device
- to: user_agent.device.name
- alias: true
- beat: filebeat
-- from: apache2.access.user_agent.name
- to: user_agent.name
- alias: true
- beat: filebeat
-- from: apache2.access.user_agent.os
- to: user_agent.os.full_name
- alias: true
- beat: filebeat
-- from: apache2.access.user_agent.os_name
- to: user_agent.os.name
- alias: true
- beat: filebeat
-
-- from: apache2.access.user_agent.major
- to: user_agent.version
- alias: false
- beat: filebeat
-- from: apache2.access.user_agent.minor
- to: user_agent.version
- alias: false
- beat: filebeat
-- from: apache2.access.user_agent.patch
- to: user_agent.version
- alias: false
- beat: filebeat
-- from: apache2.access.user_agent.os_major
- to: user_agent.os.version
- alias: false
- beat: filebeat
-- from: apache2.access.user_agent.os_minor
- to: user_agent.os.version
- alias: false
- beat: filebeat
-- from: apache2.access.user_agent.os_patch
- to: user_agent.os.version
- alias: false
- beat: filebeat
-
-### Error fileset
-- from: apache2.error.message
- to: message
- alias: true
- beat: filebeat
-
-- from: apache2.error.level
- to: log.level
- alias: true
- beat: filebeat
-
-- from: apache2.error.client
- to: source.address
- alias: true
- beat: filebeat
-
-- from: apache2.error.pid
- to: process.pid
- alias: true
- beat: filebeat
-
-- from: apache2.error.tid
- to: process.thread.id
- alias: true
- beat: filebeat
-
 ## Elasticsearch module
 
 - from: elasticsearch.audit.origin_address
@@ -1748,7 +1599,6 @@
  alias: true
  beat: metricbeat
 
-
 ### Redis
 
 - from: php_fpm.status.pid

dev-tools/mage/check.go

@@ -36,6 +36,7 @@ import (
  "github.com/pkg/errors"
 
  "github.com/elastic/beats/v7/dev-tools/mage/gotool"
+ "github.com/elastic/beats/v7/libbeat/dashboards"
  "github.com/elastic/beats/v7/libbeat/processors/dissect"
 )
 
@@ -260,6 +261,14 @@ func checkDashboardForErrors(file string, d []byte) bool {
  fmt.Println(" ", err)
  }
 
+ replaced := dashboards.ReplaceIndexInDashboardObject("my-test-index-*", d)
+ if bytes.Contains(replaced, []byte(BeatName+"-*")) {
+ hasErrors = true
+ fmt.Printf(">> Cannot modify all index pattern references in dashboard - %s\n", file)
+ fmt.Println("Please edit the dashboard override function named ReplaceIndexInDashboardObject in libbeat.")
+ fmt.Println(string(replaced))
+ }
+
  return hasErrors
 }