Merge branch 'master' into recorded_future

elastic · Jun 27, 2021 · 3e2d0e0 · 3e2d0e0
2 parents eb3c2fb + 076e0a6
commit 3e2d0e0
Show file tree

Hide file tree

Showing 65 changed files with 2,160 additions and 302 deletions.
diff --git a/CHANGELOG-developer.asciidoc b/CHANGELOG-developer.asciidoc
@@ -208,3 +208,4 @@ The list below covers the major changes between 6.3.0 and 7.0.0-alpha2 only.
 - Allow/Merge fields.yml overrides {pull}9188[9188]
 - Filesets can now define multiple ingest pipelines, with the first one considered as the entry point pipeline. {pull}8914[8914]
 - Add `group_measurements_by_instance` option to windows perfmon metricset. {pull}8688[8688]
+- Bump ECS version to 1.10.0. {issue}25734[25734]
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -277,6 +277,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix bug in aws-s3 input where the end of gzipped log files might have been discarded. {pull}26260[26260]
 - Fix bug in `httpjson` that prevented `first_event` getting updated. {pull}26407[26407]
 - Fix bug in the Syslog input that misparsed rfc5424 days starting with 0. {pull}26419[26419]
+- Do not close filestream harvester if an unexpected error is returned when close.on_state_change.* is enabled. {pull}26411[26411]
 
 *Filebeat*
 
@@ -385,6 +386,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Fix incorrect field name appending to `related.hash` in `threatintel.abusechmalware` ingest pipeline. {issue}25151[25151] {pull}25674[25674]
 - Add improvements to the azure activitylogs and platformlogs ingest pipelines. {pull}26148[26148]
 - Fix `kibana.log` pipeline when `event.duration` calculation becomes a Long. {issue}24556[24556] {pull}25675[25675]
+- Removed incorrect `http.request.referrer` field from `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 
 *Heartbeat*
 
@@ -493,6 +495,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Update config in `windows.yml` file. {issue}23027[23027]{pull}23327[23327]
 - Fix metric grouping for windows/perfmon module {issue}23489[23489] {pull}23505[23505]
 - Major refactor of system/cpu and system/core metrics. {pull}25771[25771]
+- Fix GCP Project ID being ingested as `cloud.account.id` in `gcp.billing` module {issue}26357[26357] {pull}26412[26412]
 
 *Packetbeat*
 
@@ -826,7 +829,11 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Add `include_s3_metadata` config option to the `aws-s3` input for including object metadata in events. {pull}26267[26267]
 - RFC 5424 and UNIX socket support in the Syslog input are now GA {pull}26293[26293]
 - Update grok patterns for HA Proxy module {issue}25827[25827] {pull}25835[25835]
+- Update PanOS module's date processor formats to parse `strict_date_optional_time_nanos`. {issue}26033[26033] {pull}26158[26158]
+- Update Okta module to parse additional fields to `okta.debug_context.debug_data`. {issue}25689[25689] {pull}25818[25818]
 - Added dataset `anomalithreatstream` to the `threatintel` module to ingest indicators from Anomali ThreatStream {pull}26350[26350]
+- Add support for `copytruncate` method when rotating input logs with an external tool in `filestream` input. {pull}23457[23457]
+- Add `uri_parts` and `user_agent` ingest processors to `aws.elb` module. {issue}26435[26435] {pull}26441[26441]
 - Added dataset `recordedfuture` to the `threatintel` module to ingest indicators from Recorded Future Connect API {pull}26481[26481]
 
 *Heartbeat*

diff --git a/NOTICE.txt b/NOTICE.txt
@@ -6133,11 +6133,11 @@ This Agreement is governed by the laws of the State of New York and the intellec
 
 --------------------------------------------------------------------------------
 Dependency : github.com/elastic/ecs
-Version: v1.8.0
+Version: v1.10.0
 Licence type (autodetected): Apache-2.0
 --------------------------------------------------------------------------------
 
-Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.8.0/LICENSE.txt:
+Contents of probable licence file $GOMODCACHE/github.com/elastic/ecs@v1.10.0/LICENSE.txt:
 
 
  Apache License

diff --git a/auditbeat/include/fields.go b/auditbeat/include/fields.go
diff --git a/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl b/filebeat/_meta/config/filebeat.inputs.reference.yml.tmpl
@@ -292,6 +292,16 @@ filebeat.inputs:
  # original for harvesting but will report the symlink name as source.
  #prospector.scanner.symlinks: false
 
+ ### Log rotation
+
+ # When an external tool rotates the input files with copytruncate strategy
+ # use this section to help the input find the rotated files.
+ #rotation.external.strategy.copytruncate:
+ # Regex that matches the rotated files.
+ # suffix_regex: \.\d$
+ # If the rotated filename suffix is a datetime, set it here. 
+ # dateformat: -20060102
+
  ### State options
 
  # Files for the modification data is older then clean_inactive the state from the registry is removed

diff --git a/filebeat/docs/fields.asciidoc b/filebeat/docs/fields.asciidoc
@@ -113109,6 +113109,133 @@ type: keyword
 
 --
 
+[float]
+=== suspicious_activity
+
+The suspicious activity fields from the debug data.
+
+
+
+*`okta.debug_context.debug_data.suspicious_activity.browser`*::
++
+--
+The browser used.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_city`*::
++
+--
+The city where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_country`*::
++
+--
+The country where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_id`*::
++
+--
+The event ID.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_ip`*::
++
+--
+The IP of the suspicious event.
+
+
+type: ip
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_latitude`*::
++
+--
+The latitude where the suspicious activity took place.
+
+
+type: float
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_longitude`*::
++
+--
+The longitude where the suspicious activity took place.
+
+
+type: float
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_state`*::
++
+--
+The state where the suspicious activity took place.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_transaction_id`*::
++
+--
+The event transaction ID.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.event_type`*::
++
+--
+The event type.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.os`*::
++
+--
+The OS of the system from where the suspicious activity occured.
+
+
+type: keyword
+
+--
+
+*`okta.debug_context.debug_data.suspicious_activity.timestamp`*::
++
+--
+The timestamp of when the activity occurred.
+
+
+type: date
+
+--
+
 [float]
 === authentication_context
 

diff --git a/filebeat/docs/inputs/input-filestream-file-options.asciidoc b/filebeat/docs/inputs/input-filestream-file-options.asciidoc
@@ -482,3 +482,56 @@ Set the location of the marker file the following way:
 ----
 file_identity.inode_marker.path: /logs/.filebeat-marker
 ----
+
+=== Log rotation
+
+As log files are constantly written, they must be rotated and purged to prevent
+the logger application from filling up the disk. Rotation is done by an external
+application, thus, {beatname_uc} needs information how to cooperate with it.
+
+When reading from rotating files make sure the paths configuration includes
+both the active file and all rotated files.
+
+By default, {beatname_uc} is able to track files correctly in the following strategies:
+* create: new active file with a unique name is created on rotation
+* rename: rotated files are renamed
+
+However, in case of copytruncate strategy, you should provide additional configuration
+to {beatname_uc}.
+
+[float]
+==== rotation.external.strategy.copytruncate
+
+experimental[]
+
+If the log rotating application copies the contents of the active file and then
+truncates the original file, use these options to help {beatname_uc} to read files
+correctly.
+
+Set the option `suffix_regex` so {beatname_uc} can tell active and rotated files apart. There are
+two supported suffix types in the input: numberic and date.
+
+==== Numeric suffix
+
+If your rotated files have an incrementing index appended to the end of the filename, e.g.
+active file `apache.log` and the rotated files are named `apache.log.1`, `apache.log.2`, etc,
+use the following configuration.
+
+[source,yaml]
+---
+rotation.external.strategy.copytruncate:
+ suffix_regex: \.\d$
+---
+
+==== Date suffix
+
+If the rotation date is appended to the end of the filename, e.g. active file `apache.log` and the
+rotated files are named `apache.log-20210526`, `apache.log-20210527`, etc. use the following configuration:
+
+[source,yaml]
+---
+rotation.external.strategy.copytruncate:
+ suffix_regex: \-\d{6}$
+ dateformat: -20060102
+---
+
diff --git a/filebeat/filebeat.reference.yml b/filebeat/filebeat.reference.yml
@@ -699,6 +699,16 @@ filebeat.inputs:
  # original for harvesting but will report the symlink name as source.
  #prospector.scanner.symlinks: false
 
+ ### Log rotation
+
+ # When an external tool rotates the input files with copytruncate strategy
+ # use this section to help the input find the rotated files.
+ #rotation.external.strategy.copytruncate:
+ # Regex that matches the rotated files.
+ # suffix_regex: \.\d$
+ # If the rotated filename suffix is a datetime, set it here. 
+ # dateformat: -20060102
+
  ### State options
 
  # Files for the modification data is older then clean_inactive the state from the registry is removed

diff --git a/filebeat/include/fields.go b/filebeat/include/fields.go
diff --git a/filebeat/input/filestream/config.go b/filebeat/input/filestream/config.go
@@ -41,6 +41,7 @@ type config struct {
  HarvesterLimit uint32 `config:"harvester_limit" validate:"min=0"`
  IgnoreOlder time.Duration `config:"ignore_older"`
  IgnoreInactive ignoreInactiveType `config:"ignore_inactive"`
+ Rotation *common.ConfigNamespace `config:"rotation"`
 }
 
 type closerConfig struct {
@@ -78,6 +79,17 @@ type backoffConfig struct {
  Max time.Duration `config:"max" validate:"nonzero"`
 }
 
+type rotationConfig struct {
+ Strategy *common.ConfigNamespace `config:"strategy" validate:"required"`
+}
+
+type commonRotationConfig struct {
+ SuffixRegex string `config:"suffix_regex" validate:"required"`
+ DateFormat string `config:"dateformat"`
+}
+
+type copyTruncateConfig commonRotationConfig
+
 func defaultConfig() config {
  return config{
  Reader: defaultReaderConfig(),