From f32ba51038ab68061435973e9efd59dce7e378fd Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Thu, 22 Oct 2020 15:10:31 +0200 Subject: [PATCH 1/2] Make cursor optional for okta and update docs --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/modules/okta.asciidoc | 26 ++++++++++++++----- .../filebeat/module/okta/_meta/docs.asciidoc | 26 ++++++++++++++----- .../module/okta/system/config/input.yml | 4 +++ .../filebeat/module/okta/system/manifest.yml | 2 ++ 5 files changed, 47 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index ae48f268977..3e4252f5239 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -640,6 +640,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adding support for FIPS in s3 input {pull}21446[21446] - Add SSL option to checkpoint module {pull}19560[19560] - Add max_number_of_messages config into s3 input. {pull}21993[21993] +- Add optional settings to Okta to preserve state between restarts {pull}22091[22091] *Heartbeat* diff --git a/filebeat/docs/modules/okta.asciidoc b/filebeat/docs/modules/okta.asciidoc index 038f6d088dd..985c06022cc 100644 --- a/filebeat/docs/modules/okta.asciidoc +++ b/filebeat/docs/modules/okta.asciidoc @@ -32,12 +32,6 @@ the logs while honoring any https://developer.okta.com/docs/reference/rate-limits/[rate-limiting] headers sent by Okta. -NOTE: This module does not persist the timestamp of the last read event in -order to facilitate resuming on restart. This feature will be coming in a future -version. When you restart the module will read events from the beginning of the -log. To minimize duplicates documents the module uses the event's Okta UUID -value as the Elasticsearch `_id`. - This is an example configuration for the module. [source,yaml] @@ -99,6 +93,26 @@ information. supported_protocols: [TLSv1.2] ---- +*`var.keep_state`*:: + +By default, this module does not persist the timestamp of the last read event. When you restart the module will read events from the beginning of the +log. To minimize duplicates documents the module uses the event's Okta UUID +value as the Elasticsearch `_id`. To use the last read event timestamp between restarts, `var.keep_state` can be set to `true`. It defaults to `false`. ++ +[source,yaml] +---- + var.keep_state: true +---- + +*`var.initial_interval`*:: + +When `var.keep_state` is set to `true`, an initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. It defaults to `24h`. ++ +[source,yaml] +---- + var.initial_interval: 24h # will fetch events starting 24h ago. +---- + [float] === Example dashboard diff --git a/x-pack/filebeat/module/okta/_meta/docs.asciidoc b/x-pack/filebeat/module/okta/_meta/docs.asciidoc index 1ea5cc6a66d..f1cfaf3f56e 100644 --- a/x-pack/filebeat/module/okta/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/okta/_meta/docs.asciidoc @@ -27,12 +27,6 @@ the logs while honoring any https://developer.okta.com/docs/reference/rate-limits/[rate-limiting] headers sent by Okta. -NOTE: This module does not persist the timestamp of the last read event in -order to facilitate resuming on restart. This feature will be coming in a future -version. When you restart the module will read events from the beginning of the -log. To minimize duplicates documents the module uses the event's Okta UUID -value as the Elasticsearch `_id`. - This is an example configuration for the module. [source,yaml] @@ -94,6 +88,26 @@ information. supported_protocols: [TLSv1.2] ---- +*`var.keep_state`*:: + +By default, this module does not persist the timestamp of the last read event. When you restart the module will read events from the beginning of the +log. To minimize duplicates documents the module uses the event's Okta UUID +value as the Elasticsearch `_id`. To use the last read event timestamp between restarts, `var.keep_state` can be set to `true`. It defaults to `false`. ++ +[source,yaml] +---- + var.keep_state: true +---- + +*`var.initial_interval`*:: + +When `var.keep_state` is set to `true`, an initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. It defaults to `24h`. ++ +[source,yaml] +---- + var.initial_interval: 24h # will fetch events starting 24h ago. +---- + [float] === Example dashboard diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index 990d1a5c921..f0d43939625 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -44,9 +44,13 @@ ssl: {{ .ssl | tojson }} url: {{ .url }} {{ end }} +{{ if .keep_state }} date_cursor.field: published date_cursor.url_field: since + {{ if .initial_interval }} date_cursor.initial_interval: {{ .initial_interval }} + {{ end }} +{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/okta/system/manifest.yml b/x-pack/filebeat/module/okta/system/manifest.yml index f8f83fd9aee..2a4d9abe681 100644 --- a/x-pack/filebeat/module/okta/system/manifest.yml +++ b/x-pack/filebeat/module/okta/system/manifest.yml @@ -32,6 +32,8 @@ var: - name: tags default: [forwarded] - name: url + - name: keep_state + default: false - name: initial_interval default: 24h From 07db56f94907e6a40ede32b2bf1ae6a7796c5a73 Mon Sep 17 00:00:00 2001 From: Marc Guasch Date: Tue, 27 Oct 2020 10:11:55 +0100 Subject: [PATCH 2/2] Remove keep_state flag --- CHANGELOG.next.asciidoc | 2 +- filebeat/docs/modules/okta.asciidoc | 13 +------------ x-pack/filebeat/module/okta/_meta/docs.asciidoc | 13 +------------ x-pack/filebeat/module/okta/system/config/input.yml | 4 ---- x-pack/filebeat/module/okta/system/manifest.yml | 2 -- 5 files changed, 3 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 3e4252f5239..99008bf7181 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -640,7 +640,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Adding support for FIPS in s3 input {pull}21446[21446] - Add SSL option to checkpoint module {pull}19560[19560] - Add max_number_of_messages config into s3 input. {pull}21993[21993] -- Add optional settings to Okta to preserve state between restarts {pull}22091[22091] +- Update Okta documentation for new stateful restarts. {pull}22091[22091] *Heartbeat* diff --git a/filebeat/docs/modules/okta.asciidoc b/filebeat/docs/modules/okta.asciidoc index 985c06022cc..d1f8e6ea2ec 100644 --- a/filebeat/docs/modules/okta.asciidoc +++ b/filebeat/docs/modules/okta.asciidoc @@ -93,20 +93,9 @@ information. supported_protocols: [TLSv1.2] ---- -*`var.keep_state`*:: - -By default, this module does not persist the timestamp of the last read event. When you restart the module will read events from the beginning of the -log. To minimize duplicates documents the module uses the event's Okta UUID -value as the Elasticsearch `_id`. To use the last read event timestamp between restarts, `var.keep_state` can be set to `true`. It defaults to `false`. -+ -[source,yaml] ----- - var.keep_state: true ----- - *`var.initial_interval`*:: -When `var.keep_state` is set to `true`, an initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. It defaults to `24h`. +An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `24h`. + [source,yaml] ---- diff --git a/x-pack/filebeat/module/okta/_meta/docs.asciidoc b/x-pack/filebeat/module/okta/_meta/docs.asciidoc index f1cfaf3f56e..297a8644987 100644 --- a/x-pack/filebeat/module/okta/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/okta/_meta/docs.asciidoc @@ -88,20 +88,9 @@ information. supported_protocols: [TLSv1.2] ---- -*`var.keep_state`*:: - -By default, this module does not persist the timestamp of the last read event. When you restart the module will read events from the beginning of the -log. To minimize duplicates documents the module uses the event's Okta UUID -value as the Elasticsearch `_id`. To use the last read event timestamp between restarts, `var.keep_state` can be set to `true`. It defaults to `false`. -+ -[source,yaml] ----- - var.keep_state: true ----- - *`var.initial_interval`*:: -When `var.keep_state` is set to `true`, an initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. It defaults to `24h`. +An initial interval can be defined. The first time the module starts, will fetch events from the current moment minus the initial interval value. Following restarts will fetch events starting from the last event read. It defaults to `24h`. + [source,yaml] ---- diff --git a/x-pack/filebeat/module/okta/system/config/input.yml b/x-pack/filebeat/module/okta/system/config/input.yml index f0d43939625..990d1a5c921 100644 --- a/x-pack/filebeat/module/okta/system/config/input.yml +++ b/x-pack/filebeat/module/okta/system/config/input.yml @@ -44,13 +44,9 @@ ssl: {{ .ssl | tojson }} url: {{ .url }} {{ end }} -{{ if .keep_state }} date_cursor.field: published date_cursor.url_field: since - {{ if .initial_interval }} date_cursor.initial_interval: {{ .initial_interval }} - {{ end }} -{{ end }} {{ else if eq .input "file" }} diff --git a/x-pack/filebeat/module/okta/system/manifest.yml b/x-pack/filebeat/module/okta/system/manifest.yml index 2a4d9abe681..f8f83fd9aee 100644 --- a/x-pack/filebeat/module/okta/system/manifest.yml +++ b/x-pack/filebeat/module/okta/system/manifest.yml @@ -32,8 +32,6 @@ var: - name: tags default: [forwarded] - name: url - - name: keep_state - default: false - name: initial_interval default: 24h