From 4e1d4b9127f0de0c8edf89adb19dbcd182bfd851 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 21 Apr 2020 11:56:12 +0200 Subject: [PATCH] Add config option to select a different azure cloud env in the azure-eventhub input and azure module (#17659) * add option to select a different azure env * changelog * rename * add test * mage fmt update --- CHANGELOG.next.asciidoc | 1 + filebeat/docs/modules/azure.asciidoc | 13 ++++++++++ .../docs/inputs/input-azure-eventhub.asciidoc | 17 +++++++++--- x-pack/filebeat/filebeat.reference.yml | 8 +++--- x-pack/filebeat/input/azureeventhub/config.go | 2 ++ x-pack/filebeat/input/azureeventhub/eph.go | 26 ++++++++++++++++++- .../filebeat/input/azureeventhub/eph_test.go | 16 ++++++++++++ x-pack/filebeat/module/azure/_meta/config.yml | 8 +++--- .../filebeat/module/azure/_meta/docs.asciidoc | 13 ++++++++++ .../activitylogs/config/azure-eventhub.yml | 1 + .../module/azure/activitylogs/manifest.yml | 1 + .../azure/auditlogs/config/azure-eventhub.yml | 1 + .../module/azure/auditlogs/manifest.yml | 1 + .../signinlogs/config/azure-eventhub.yml | 1 + .../module/azure/signinlogs/manifest.yml | 1 + x-pack/filebeat/modules.d/azure.yml.disabled | 8 +++--- 16 files changed, 102 insertions(+), 16 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 5831a2c44ce..fbcf1da05a8 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -257,6 +257,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Add dashboard for Google Cloud Audit and AWS CloudTrail. {pull}17379[17379] - Improve ECS categorization field mappings for mysql module. {issue}16172[16172] {pull}17491[17491] - Release Google Cloud module as GA. {pull}17511[17511] +- Add config option to select a different azure cloud env in the azure-eventhub input and azure module. {issue}17649[17649] {pull}17659[17659] - Added new Checkpoint Syslog filebeat module. {pull}17682[17682] - Improve ECS categorization field mappings for nats module. {issue}16173[16173] {pull}17550[17550] - Enhance `elasticsearch/server` fileset to handle ECS-compatible logs emitted by Elasticsearch. {issue}17715[17715] {pull}17714[17714] diff --git a/filebeat/docs/modules/azure.asciidoc b/filebeat/docs/modules/azure.asciidoc index 5d52e33beac..b194b7c320c 100644 --- a/filebeat/docs/modules/azure.asciidoc +++ b/filebeat/docs/modules/azure.asciidoc @@ -43,6 +43,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -52,6 +53,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -61,6 +63,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" ``` @@ -90,6 +93,16 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc index 15b628169ce..ac91fb476d6 100644 --- a/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc +++ b/x-pack/filebeat/docs/inputs/input-azure-eventhub.asciidoc @@ -28,6 +28,8 @@ Example configuration: storage_account: "azureeph" storage_account_key: "....." storage_account_container: "" + resource_manager_endpoint: "" + ---- ==== Configuration options @@ -36,7 +38,7 @@ The `azure-eventhub` input supports the following configuration: ==== `eventhub` -The name of the eventhub users would like to read from. +The name of the eventhub users would like to read from, field required. ==== `consumer_group` @@ -50,14 +52,23 @@ A Blob Storage account is required in order to store/retrieve/update the offset ==== `storage_account` -The name of the storage account. +The name of the storage account. Required. ==== `storage_account_key` -The storage account key, this key will be used to authorize access to data in your storage account. +The storage account key, this key will be used to authorize access to data in your storage account, option is required. ==== `storage_account_container` Optional, the name of the storage account container you would like to store the offset information in. +==== `resource_manager_endpoint` + +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. diff --git a/x-pack/filebeat/filebeat.reference.yml b/x-pack/filebeat/filebeat.reference.yml index 068e332a5a7..3f88fa42976 100644 --- a/x-pack/filebeat/filebeat.reference.yml +++ b/x-pack/filebeat/filebeat.reference.yml @@ -306,15 +306,15 @@ filebeat.modules: activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" auditlogs: diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index b567b25e35b..0521d3a76e6 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -17,6 +17,8 @@ type azureInputConfig struct { SAName string `config:"storage_account"` SAKey string `config:"storage_account_key"` SAContainer string `config:"storage_account_container"` + // by default the azure public environment is used, to override, users can provide a specific resource manager endpoint + OverrideEnvironment string `config:"resource_manager_endpoint"` } const ephContainerName = "filebeat" diff --git a/x-pack/filebeat/input/azureeventhub/eph.go b/x-pack/filebeat/input/azureeventhub/eph.go index 8848483c8be..bab54a45223 100644 --- a/x-pack/filebeat/input/azureeventhub/eph.go +++ b/x-pack/filebeat/input/azureeventhub/eph.go @@ -16,6 +16,14 @@ import ( "github.com/Azure/go-autorest/autorest/azure" ) +// users can select from one of the already defined azure cloud envs +var environments = map[string]azure.Environment{ + azure.ChinaCloud.ResourceManagerEndpoint: azure.ChinaCloud, + azure.GermanCloud.ResourceManagerEndpoint: azure.GermanCloud, + azure.PublicCloud.ResourceManagerEndpoint: azure.PublicCloud, + azure.USGovernmentCloud.ResourceManagerEndpoint: azure.USGovernmentCloud, +} + // runWithEPH will consume ingested events using the Event Processor Host (EPH) https://github.com/Azure/azure-event-hubs-go#event-processor-host, https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-event-processor-host func (a *azureInput) runWithEPH() error { // create a new Azure Storage Leaser / Checkpointer @@ -23,7 +31,11 @@ func (a *azureInput) runWithEPH() error { if err != nil { return err } - leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, azure.PublicCloud) + env, err := getAzureEnvironment(a.config.OverrideEnvironment) + if err != nil { + return err + } + leaserCheckpointer, err := storage.NewStorageLeaserCheckpointer(cred, a.config.SAName, a.config.SAContainer, env) if err != nil { return err } @@ -74,3 +86,15 @@ func (a *azureInput) runWithEPH() error { } return nil } + +func getAzureEnvironment(overrideResManager string) (azure.Environment, error) { + // if no overrride is set then the azure public cloud is used + if overrideResManager == "" { + return azure.PublicCloud, nil + } + if env, ok := environments[overrideResManager]; ok { + return env, nil + } + // can retrieve hybrid env from the resource manager endpoint + return azure.EnvironmentFromURL(overrideResManager) +} diff --git a/x-pack/filebeat/input/azureeventhub/eph_test.go b/x-pack/filebeat/input/azureeventhub/eph_test.go index 3a0ac99db7d..b48499eb7c4 100644 --- a/x-pack/filebeat/input/azureeventhub/eph_test.go +++ b/x-pack/filebeat/input/azureeventhub/eph_test.go @@ -7,6 +7,8 @@ package azureeventhub import ( "testing" + "github.com/Azure/go-autorest/autorest/azure" + "github.com/stretchr/testify/assert" ) @@ -26,3 +28,17 @@ func TestRunWithEPH(t *testing.T) { err := input.runWithEPH() assert.Error(t, err, '7') } + +func TestGetAzureEnvironment(t *testing.T) { + resMan := "" + env, err := getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.PublicCloud) + resMan = "https://management.microsoftazure.de/" + env, err = getAzureEnvironment(resMan) + assert.NoError(t, err) + assert.Equal(t, env, azure.GermanCloud) + resMan = "http://management.invalidhybrid.com/" + env, err = getAzureEnvironment(resMan) + assert.Errorf(t, err, "invalid character 'F' looking for beginning of value") +} diff --git a/x-pack/filebeat/module/azure/_meta/config.yml b/x-pack/filebeat/module/azure/_meta/config.yml index 7509037c28e..ab7f477b8bb 100644 --- a/x-pack/filebeat/module/azure/_meta/config.yml +++ b/x-pack/filebeat/module/azure/_meta/config.yml @@ -3,15 +3,15 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" auditlogs: diff --git a/x-pack/filebeat/module/azure/_meta/docs.asciidoc b/x-pack/filebeat/module/azure/_meta/docs.asciidoc index 5bf7bb576d0..eea82995532 100644 --- a/x-pack/filebeat/module/azure/_meta/docs.asciidoc +++ b/x-pack/filebeat/module/azure/_meta/docs.asciidoc @@ -38,6 +38,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" auditlogs: enabled: false @@ -47,6 +48,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" signinlogs: enabled: false @@ -56,6 +58,7 @@ Will retrieve azure Active Directory audit logs. The audit logs provide traceabi connection_string: "" storage_account: "" storage_account_key: "" + resource_manager_endpoint: "" ``` @@ -85,6 +88,16 @@ The name of the storage account the state/offsets will be stored and updated. _string_ The storage account key, this key will be used to authorize access to data in your storage account. +`resource_manager_endpoint` :: +_string_ +Optional, by default we are using the azure public environment, to override, users can provide a specific resource manager endpoint in order to use a different azure environment. +Ex: +https://management.chinacloudapi.cn/ for azure ChinaCloud +https://management.microsoftazure.de/ for azure GermanCloud +https://management.azure.com/ for azure PublicCloud +https://management.usgovcloudapi.net/ for azure USGovernmentCloud +Users can also use this in case of a Hybrid Cloud model, where one may define their own endpoints. + include::../include/what-happens.asciidoc[] include::../include/gs-link.asciidoc[] diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index b89bebb30f9..9b747e1092d 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/activitylogs/manifest.yml b/x-pack/filebeat/module/azure/activitylogs/manifest.yml index 7375b6e42a4..4d5c20a7271 100644 --- a/x-pack/filebeat/module/azure/activitylogs/manifest.yml +++ b/x-pack/filebeat/module/azure/activitylogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 01796611504..3c2ea50cf8b 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -4,4 +4,5 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/auditlogs/manifest.yml b/x-pack/filebeat/module/azure/auditlogs/manifest.yml index d6cd469718b..095371bff16 100644 --- a/x-pack/filebeat/module/azure/auditlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/auditlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index b89bebb30f9..9b747e1092d 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -4,3 +4,4 @@ eventhub: {{ .eventhub }} consumer_group: {{ .consumer_group }} storage_account: {{ .storage_account }} storage_account_key: {{ .storage_account_key }} +resource_manager_endpoint: {{ .resource_manager_endpoint }} diff --git a/x-pack/filebeat/module/azure/signinlogs/manifest.yml b/x-pack/filebeat/module/azure/signinlogs/manifest.yml index f68109af4a1..97fddae51e9 100644 --- a/x-pack/filebeat/module/azure/signinlogs/manifest.yml +++ b/x-pack/filebeat/module/azure/signinlogs/manifest.yml @@ -10,6 +10,7 @@ var: - name: connection_string - name: storage_account - name: storage_account_key + - name: resource_manager_endpoint ingest_pipeline: - ingest/pipeline.json diff --git a/x-pack/filebeat/modules.d/azure.yml.disabled b/x-pack/filebeat/modules.d/azure.yml.disabled index c8003fbcf96..0c7eb3d6e01 100644 --- a/x-pack/filebeat/modules.d/azure.yml.disabled +++ b/x-pack/filebeat/modules.d/azure.yml.disabled @@ -6,15 +6,15 @@ activitylogs: enabled: true var: - # Eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub + # eventhub name containing the activity logs, overwrite he default value if the logs are exported in a different eventhub eventhub: "insights-operational-logs" - # Consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module + # consumer group name that has access to the event hub, we advise creating a dedicated consumer group for the azure module consumer_group: "$Default" # the connection string required to communicate with Event Hubs, steps to generate one here https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-get-connection-string connection_string: "" - # the name of the storage account the state/offsets will be stored and updated. + # the name of the storage account the state/offsets will be stored and updated storage_account: "" - #The storage account key, this key will be used to authorize access to data in your storage account. + # the storage account key, this key will be used to authorize access to data in your storage account storage_account_key: "" auditlogs: