From 916d4e0e559b3c7b29b3b779b3b1c48cb8309a53 Mon Sep 17 00:00:00 2001 From: Mariana Dima Date: Tue, 23 Feb 2021 11:50:05 +0000 Subject: [PATCH 1/3] Empty configuration options generate `` string for azure-eventhub input (#24156) * work on no value * changelog --- CHANGELOG.next.asciidoc | 1 + x-pack/filebeat/input/azureeventhub/config.go | 2 +- .../activitylogs/config/azure-eventhub.yml | 22 +++++++++++++++++-- .../azure/auditlogs/config/azure-eventhub.yml | 22 +++++++++++++++++-- .../platformlogs/config/azure-eventhub.yml | 22 +++++++++++++++++-- .../signinlogs/config/azure-eventhub.yml | 22 +++++++++++++++++-- 6 files changed, 82 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc index 21c1d90a394..6af78023970 100644 --- a/CHANGELOG.next.asciidoc +++ b/CHANGELOG.next.asciidoc @@ -281,6 +281,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d - Change the `event.created` in Netflow events to be the time the event was created by Filebeat to be consistent with ECS. {pull}23094[23094] - Update `filestream` reader offset when a line is skipped. {pull}23417[23417] +- Add check for empty values in azure module. {pull}24156[24156] *Filebeat* diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go index 68ad8d109e0..e24f3f2b0f6 100644 --- a/x-pack/filebeat/input/azureeventhub/config.go +++ b/x-pack/filebeat/input/azureeventhub/config.go @@ -33,7 +33,7 @@ func (conf *azureInputConfig) Validate() error { return errors.New("no event hub name configured") } if conf.SAName == "" || conf.SAKey == "" { - return errors.New("missing storage account information") + return errors.New("no storage account or storage account key configured") } if conf.SAContainer == "" { conf.SAContainer = fmt.Sprintf("%s-%s", ephContainerName, conf.EventHubName) diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml index 8701cae46fb..a949730a58f 100644 --- a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml @@ -1,11 +1,29 @@ type: azure-eventhub -connection_string: {{ .connection_string }} +{{ if .eventhub }} eventhub: {{ .eventhub }} +storage_account_container: filebeat-activitylogs-{{ .eventhub }} +{{ end }} + +{{ if .connection_string }} +connection_string: {{ .connection_string }} +{{ end }} + +{{ if .consumer_group }} consumer_group: {{ .consumer_group }} +{{ end }} + +{{ if .storage_account }} storage_account: {{ .storage_account }} +{{ end }} + +{{ if .storage_account_key }} storage_account_key: {{ .storage_account_key }} +{{ end }} + +{{ if .resource_manager_endpoint }} resource_manager_endpoint: {{ .resource_manager_endpoint }} -storage_account_container: filebeat-activitylogs-{{ .eventhub }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml index 7f5eb091550..a5460ed456e 100644 --- a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml @@ -1,11 +1,29 @@ type: azure-eventhub -connection_string: {{ .connection_string }} +{{ if .eventhub }} eventhub: {{ .eventhub }} +storage_account_container: filebeat-auditlogs-{{ .eventhub }} +{{ end }} + +{{ if .connection_string }} +connection_string: {{ .connection_string }} +{{ end }} + +{{ if .consumer_group }} consumer_group: {{ .consumer_group }} +{{ end }} + +{{ if .storage_account }} storage_account: {{ .storage_account }} +{{ end }} + +{{ if .storage_account_key }} storage_account_key: {{ .storage_account_key }} +{{ end }} + +{{ if .resource_manager_endpoint }} resource_manager_endpoint: {{ .resource_manager_endpoint }} -storage_account_container: filebeat-auditlogs-{{ .eventhub }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml index 80a73bc9905..49cfcef3a84 100644 --- a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml @@ -1,11 +1,29 @@ type: azure-eventhub -connection_string: {{ .connection_string }} +{{ if .eventhub }} eventhub: {{ .eventhub }} +storage_account_container: filebeat-platformlogs-{{ .eventhub }} +{{ end }} + +{{ if .connection_string }} +connection_string: {{ .connection_string }} +{{ end }} + +{{ if .consumer_group }} consumer_group: {{ .consumer_group }} +{{ end }} + +{{ if .storage_account }} storage_account: {{ .storage_account }} +{{ end }} + +{{ if .storage_account_key }} storage_account_key: {{ .storage_account_key }} +{{ end }} + +{{ if .resource_manager_endpoint }} resource_manager_endpoint: {{ .resource_manager_endpoint }} -storage_account_container: filebeat-platformlogs-{{ .eventhub }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml index e37c7c61a4d..9a6a86e08fa 100644 --- a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml +++ b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml @@ -1,11 +1,29 @@ type: azure-eventhub -connection_string: {{ .connection_string }} +{{ if .eventhub }} eventhub: {{ .eventhub }} +storage_account_container: filebeat-signinlogs-{{ .eventhub }} +{{ end }} + +{{ if .connection_string }} +connection_string: {{ .connection_string }} +{{ end }} + +{{ if .consumer_group }} consumer_group: {{ .consumer_group }} +{{ end }} + +{{ if .storage_account }} storage_account: {{ .storage_account }} +{{ end }} + +{{ if .storage_account_key }} storage_account_key: {{ .storage_account_key }} +{{ end }} + +{{ if .resource_manager_endpoint }} resource_manager_endpoint: {{ .resource_manager_endpoint }} -storage_account_container: filebeat-signinlogs-{{ .eventhub }} +{{ end }} + tags: {{.tags | tojson}} publisher_pipeline.disable_host: {{ inList .tags "forwarded" }} processors: From 05b523a10d154001a6f38bc445c3564c95d45c7c Mon Sep 17 00:00:00 2001 From: Chris Mark Date: Tue, 23 Feb 2021 14:40:33 +0200 Subject: [PATCH 2/3] Add example input autodsicover config (#24157) --- .../elastic-agent-standalone-kubernetes.yml | 25 +++++++++++++++++++ ...-agent-standalone-daemonset-configmap.yaml | 25 +++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml index 46b625fa0ce..a97a4c6bed7 100644 --- a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml +++ b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml @@ -313,6 +313,31 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none + # Add extra input blocks here, based on conditions + # so as to automatically identify targeted Pods and start monitoring them + # using a predefined integration. For instance: + #- name: redis + # type: redis/metrics + # use_output: default + # meta: + # package: + # name: redis + # version: 0.3.6 + # data_stream: + # namespace: default + # streams: + # - data_stream: + # dataset: redis.info + # type: metrics + # metricsets: + # - info + # hosts: + # - '${kubernetes.pod.ip}:6379' + # idle_timeout: 20s + # maxconn: 10 + # network: tcp + # period: 10s + # condition: ${kubernetes.pod.labels.app} == 'redis' --- apiVersion: apps/v1 kind: Deployment diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml index 11c2992c93c..3e339ef8860 100644 --- a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml +++ b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml @@ -229,3 +229,28 @@ data: - 'https://${env.NODE_NAME}:10250' period: 10s ssl.verification_mode: none + # Add extra input blocks here, based on conditions + # so as to automatically identify targeted Pods and start monitoring them + # using a predefined integration. For instance: + #- name: redis + # type: redis/metrics + # use_output: default + # meta: + # package: + # name: redis + # version: 0.3.6 + # data_stream: + # namespace: default + # streams: + # - data_stream: + # dataset: redis.info + # type: metrics + # metricsets: + # - info + # hosts: + # - '${kubernetes.pod.ip}:6379' + # idle_timeout: 20s + # maxconn: 10 + # network: tcp + # period: 10s + # condition: ${kubernetes.pod.labels.app} == 'redis' From 9e6b67600fdd3be13695b9794a2e78d87e2d2f58 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 23 Feb 2021 15:59:01 +0000 Subject: [PATCH 3/3] [CI] Add ARM packaging (#24041) --- Jenkinsfile | 27 +++++++++++++++++++++++++-- auditbeat/Jenkinsfile.yml | 6 ++++++ filebeat/Jenkinsfile.yml | 6 ++++++ heartbeat/Jenkinsfile.yml | 6 ++++++ journalbeat/Jenkinsfile.yml | 6 ++++++ metricbeat/Jenkinsfile.yml | 6 ++++++ packetbeat/Jenkinsfile.yml | 6 ++++++ x-pack/auditbeat/Jenkinsfile.yml | 6 ++++++ x-pack/dockerlogbeat/Jenkinsfile.yml | 6 ++++++ x-pack/elastic-agent/Jenkinsfile.yml | 6 ++++++ x-pack/filebeat/Jenkinsfile.yml | 8 +++++++- x-pack/heartbeat/Jenkinsfile.yml | 6 ++++++ x-pack/metricbeat/Jenkinsfile.yml | 6 ++++++ x-pack/packetbeat/Jenkinsfile.yml | 6 ++++++ 14 files changed, 104 insertions(+), 3 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index b317251ce53..41a839fff3e 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -262,7 +262,20 @@ def k8sTest(Map args = [:]) { } /** -* This method runs the packaging +* This method runs the packaging for ARM +*/ +def packagingArm(Map args = [:]) { + def PLATFORMS = [ 'linux/arm64' ].join(' ') + withEnv([ + "PLATFORMS=${PLATFORMS}", + "PACKAGES=docker" + ]) { + target(args) + } +} + +/** +* This method runs the packaging for Linux */ def packagingLinux(Map args = [:]) { def PLATFORMS = [ '+all', @@ -286,7 +299,6 @@ def packagingLinux(Map args = [:]) { } } - /** * Upload the packages to their snapshot or pull request buckets * @param beatsFolder beats folder @@ -922,6 +934,17 @@ class RunCommand extends co.elastic.beats.BeatsFunction { if(args?.content?.containsKey('mage')) { steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id) } + if(args?.content?.containsKey('packaging-arm')) { + steps.packagingArm(context: args.context, + command: args.content.get('packaging-arm'), + directory: args.project, + label: args.label, + isMage: true, + id: args.id, + e2e: args.content.get('e2e'), + package: true, + dockerArch: 'arm64') + } if(args?.content?.containsKey('packaging-linux')) { steps.packagingLinux(context: args.context, command: args.content.get('packaging-linux'), diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml index 346145606b5..aadb8b52ce8 100644 --- a/auditbeat/Jenkinsfile.yml +++ b/auditbeat/Jenkinsfile.yml @@ -87,3 +87,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml index 10947425828..b5083f0b068 100644 --- a/filebeat/Jenkinsfile.yml +++ b/filebeat/Jenkinsfile.yml @@ -85,3 +85,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml index 8c1bbeaa574..0d4be939c18 100644 --- a/heartbeat/Jenkinsfile.yml +++ b/heartbeat/Jenkinsfile.yml @@ -84,3 +84,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml index 0336f113162..a91d99dd115 100644 --- a/journalbeat/Jenkinsfile.yml +++ b/journalbeat/Jenkinsfile.yml @@ -37,3 +37,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml index 70874ab1277..1e219ea95b1 100644 --- a/metricbeat/Jenkinsfile.yml +++ b/metricbeat/Jenkinsfile.yml @@ -79,3 +79,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false ## e2e is enabled only for x-pack beats + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml index 3205b791b79..be0dd1d9522 100644 --- a/packetbeat/Jenkinsfile.yml +++ b/packetbeat/Jenkinsfile.yml @@ -85,3 +85,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml index 7a1edec5dd2..fe8ef3658c7 100644 --- a/x-pack/auditbeat/Jenkinsfile.yml +++ b/x-pack/auditbeat/Jenkinsfile.yml @@ -85,3 +85,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml index ee9a4b06aca..d31c4e32b5f 100644 --- a/x-pack/dockerlogbeat/Jenkinsfile.yml +++ b/x-pack/dockerlogbeat/Jenkinsfile.yml @@ -25,3 +25,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml index 9a30d836391..dc642ce1ca3 100644 --- a/x-pack/elastic-agent/Jenkinsfile.yml +++ b/x-pack/elastic-agent/Jenkinsfile.yml @@ -89,3 +89,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml index 5985900fcda..d6961a6a4b1 100644 --- a/x-pack/filebeat/Jenkinsfile.yml +++ b/x-pack/filebeat/Jenkinsfile.yml @@ -91,4 +91,10 @@ stages: packaging-linux: packaging-linux: "mage package" e2e: - enabled: false \ No newline at end of file + enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" \ No newline at end of file diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml index 3cd24f2c2b7..cc6cb0525ab 100644 --- a/x-pack/heartbeat/Jenkinsfile.yml +++ b/x-pack/heartbeat/Jenkinsfile.yml @@ -74,3 +74,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml index 08aaa1d6fc0..f38240dbf4a 100644 --- a/x-pack/metricbeat/Jenkinsfile.yml +++ b/x-pack/metricbeat/Jenkinsfile.yml @@ -84,3 +84,9 @@ stages: e2e: enabled: false entrypoint: 'metricbeat-test.sh' + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm" diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml index dcefc94c669..be199fa928e 100644 --- a/x-pack/packetbeat/Jenkinsfile.yml +++ b/x-pack/packetbeat/Jenkinsfile.yml @@ -86,3 +86,9 @@ stages: packaging-linux: "mage package" e2e: enabled: false + packaging-arm: + packaging-arm: "mage package" + e2e: + enabled: false + platforms: ## override default label in this specific stage. + - "arm"