Merge remote-tracking branch 'upstream/master' into feature/retry-win…

…dows-7 * upstream/master: [CI] Add ARM packaging (elastic#24041) Add example input autodsicover config (elastic#24157) Empty configuration options generate `<no value>` string for azure-eventhub input (elastic#24156)
v1v · Feb 23, 2021 · 965d5e9 · 965d5e9
2 parents 3ea53ee + 9e6b676
commit 965d5e9
Show file tree

Hide file tree

Showing 22 changed files with 236 additions and 12 deletions.
diff --git a/CHANGELOG.next.asciidoc b/CHANGELOG.next.asciidoc
@@ -281,6 +281,7 @@ https://github.com/elastic/beats/compare/v7.0.0-alpha2...master[Check the HEAD d
 - Change the `event.created` in Netflow events to be the time the event was created by Filebeat
  to be consistent with ECS. {pull}23094[23094]
 - Update `filestream` reader offset when a line is skipped. {pull}23417[23417]
+- Add check for empty values in azure module. {pull}24156[24156]
 
 *Filebeat*
 

diff --git a/Jenkinsfile b/Jenkinsfile
@@ -269,7 +269,20 @@ def k8sTest(Map args = [:]) {
 }
 
 /**
-* This method runs the packaging
+* This method runs the packaging for ARM
+*/
+def packagingArm(Map args = [:]) {
+ def PLATFORMS = [ 'linux/arm64' ].join(' ')
+ withEnv([
+ "PLATFORMS=${PLATFORMS}",
+ "PACKAGES=docker"
+ ]) {
+ target(args)
+ }
+}
+
+/**
+* This method runs the packaging for Linux
 */
 def packagingLinux(Map args = [:]) {
  def PLATFORMS = [ '+all',
@@ -293,7 +306,6 @@ def packagingLinux(Map args = [:]) {
  }
 }
 
-
 /**
 * Upload the packages to their snapshot or pull request buckets
 * @param beatsFolder beats folder
@@ -965,6 +977,17 @@ class RunCommand extends co.elastic.beats.BeatsFunction {
  if(args?.content?.containsKey('mage')) {
  steps.target(context: args.context, command: args.content.mage, directory: args.project, label: args.label, withModule: withModule, isMage: true, id: args.id)
  }
+ if(args?.content?.containsKey('packaging-arm')) {
+ steps.packagingArm(context: args.context,
+ command: args.content.get('packaging-arm'),
+ directory: args.project,
+ label: args.label,
+ isMage: true,
+ id: args.id,
+ e2e: args.content.get('e2e'),
+ package: true,
+ dockerArch: 'arm64')
+ }
  if(args?.content?.containsKey('packaging-linux')) {
  steps.packagingLinux(context: args.context,
  command: args.content.get('packaging-linux'),

diff --git a/auditbeat/Jenkinsfile.yml b/auditbeat/Jenkinsfile.yml
@@ -87,3 +87,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml b/deploy/kubernetes/elastic-agent-standalone-kubernetes.yml
@@ -313,6 +313,31 @@ data:
  - 'https://${env.NODE_NAME}:10250'
  period: 10s
  ssl.verification_mode: none
+ # Add extra input blocks here, based on conditions
+ # so as to automatically identify targeted Pods and start monitoring them
+ # using a predefined integration. For instance:
+ #- name: redis
+ # type: redis/metrics
+ # use_output: default
+ # meta:
+ # package:
+ # name: redis
+ # version: 0.3.6
+ # data_stream:
+ # namespace: default
+ # streams:
+ # - data_stream:
+ # dataset: redis.info
+ # type: metrics
+ # metricsets:
+ # - info
+ # hosts:
+ # - '${kubernetes.pod.ip}:6379'
+ # idle_timeout: 20s
+ # maxconn: 10
+ # network: tcp
+ # period: 10s
+ # condition: ${kubernetes.pod.labels.app} == 'redis'
 ---
 apiVersion: apps/v1
 kind: Deployment

diff --git a/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml b/deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml
@@ -229,3 +229,28 @@ data:
  - 'https://${env.NODE_NAME}:10250'
  period: 10s
  ssl.verification_mode: none
+ # Add extra input blocks here, based on conditions
+ # so as to automatically identify targeted Pods and start monitoring them
+ # using a predefined integration. For instance:
+ #- name: redis
+ # type: redis/metrics
+ # use_output: default
+ # meta:
+ # package:
+ # name: redis
+ # version: 0.3.6
+ # data_stream:
+ # namespace: default
+ # streams:
+ # - data_stream:
+ # dataset: redis.info
+ # type: metrics
+ # metricsets:
+ # - info
+ # hosts:
+ # - '${kubernetes.pod.ip}:6379'
+ # idle_timeout: 20s
+ # maxconn: 10
+ # network: tcp
+ # period: 10s
+ # condition: ${kubernetes.pod.labels.app} == 'redis'
diff --git a/filebeat/Jenkinsfile.yml b/filebeat/Jenkinsfile.yml
@@ -85,3 +85,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/heartbeat/Jenkinsfile.yml b/heartbeat/Jenkinsfile.yml
@@ -84,3 +84,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/journalbeat/Jenkinsfile.yml b/journalbeat/Jenkinsfile.yml
@@ -37,3 +37,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/metricbeat/Jenkinsfile.yml b/metricbeat/Jenkinsfile.yml
@@ -79,3 +79,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false ## e2e is enabled only for x-pack beats
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/packetbeat/Jenkinsfile.yml b/packetbeat/Jenkinsfile.yml
@@ -85,3 +85,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/auditbeat/Jenkinsfile.yml b/x-pack/auditbeat/Jenkinsfile.yml
@@ -85,3 +85,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/dockerlogbeat/Jenkinsfile.yml b/x-pack/dockerlogbeat/Jenkinsfile.yml
@@ -25,3 +25,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/elastic-agent/Jenkinsfile.yml b/x-pack/elastic-agent/Jenkinsfile.yml
@@ -89,3 +89,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/filebeat/Jenkinsfile.yml b/x-pack/filebeat/Jenkinsfile.yml
@@ -91,4 +91,10 @@ stages:
  packaging-linux:
  packaging-linux: "mage package"
  e2e:
- enabled: false
+ enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/filebeat/input/azureeventhub/config.go b/x-pack/filebeat/input/azureeventhub/config.go
@@ -33,7 +33,7 @@ func (conf *azureInputConfig) Validate() error {
  return errors.New("no event hub name configured")
  }
  if conf.SAName == "" || conf.SAKey == "" {
- return errors.New("missing storage account information")
+ return errors.New("no storage account or storage account key configured")
  }
  if conf.SAContainer == "" {
  conf.SAContainer = fmt.Sprintf("%s-%s", ephContainerName, conf.EventHubName)

diff --git a/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/activitylogs/config/azure-eventhub.yml
@@ -1,11 +1,29 @@
 type: azure-eventhub
-connection_string: {{ .connection_string }}
+{{ if .eventhub }}
 eventhub: {{ .eventhub }}
+storage_account_container: filebeat-activitylogs-{{ .eventhub }}
+{{ end }}
+
+{{ if .connection_string }}
+connection_string: {{ .connection_string }}
+{{ end }}
+
+{{ if .consumer_group }}
 consumer_group: {{ .consumer_group }}
+{{ end }}
+
+{{ if .storage_account }}
 storage_account: {{ .storage_account }}
+{{ end }}
+
+{{ if .storage_account_key }}
 storage_account_key: {{ .storage_account_key }}
+{{ end }}
+
+{{ if .resource_manager_endpoint }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
-storage_account_container: filebeat-activitylogs-{{ .eventhub }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/auditlogs/config/azure-eventhub.yml
@@ -1,11 +1,29 @@
 type: azure-eventhub
-connection_string: {{ .connection_string }}
+{{ if .eventhub }}
 eventhub: {{ .eventhub }}
+storage_account_container: filebeat-auditlogs-{{ .eventhub }}
+{{ end }}
+
+{{ if .connection_string }}
+connection_string: {{ .connection_string }}
+{{ end }}
+
+{{ if .consumer_group }}
 consumer_group: {{ .consumer_group }}
+{{ end }}
+
+{{ if .storage_account }}
 storage_account: {{ .storage_account }}
+{{ end }}
+
+{{ if .storage_account_key }}
 storage_account_key: {{ .storage_account_key }}
+{{ end }}
+
+{{ if .resource_manager_endpoint }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
-storage_account_container: filebeat-auditlogs-{{ .eventhub }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 processors:

diff --git a/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/platformlogs/config/azure-eventhub.yml
@@ -1,11 +1,29 @@
 type: azure-eventhub
-connection_string: {{ .connection_string }}
+{{ if .eventhub }}
 eventhub: {{ .eventhub }}
+storage_account_container: filebeat-platformlogs-{{ .eventhub }}
+{{ end }}
+
+{{ if .connection_string }}
+connection_string: {{ .connection_string }}
+{{ end }}
+
+{{ if .consumer_group }}
 consumer_group: {{ .consumer_group }}
+{{ end }}
+
+{{ if .storage_account }}
 storage_account: {{ .storage_account }}
+{{ end }}
+
+{{ if .storage_account_key }}
 storage_account_key: {{ .storage_account_key }}
+{{ end }}
+
+{{ if .resource_manager_endpoint }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
-storage_account_container: filebeat-platformlogs-{{ .eventhub }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 

diff --git a/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml b/x-pack/filebeat/module/azure/signinlogs/config/azure-eventhub.yml
@@ -1,11 +1,29 @@
 type: azure-eventhub
-connection_string: {{ .connection_string }}
+{{ if .eventhub }}
 eventhub: {{ .eventhub }}
+storage_account_container: filebeat-signinlogs-{{ .eventhub }}
+{{ end }}
+
+{{ if .connection_string }}
+connection_string: {{ .connection_string }}
+{{ end }}
+
+{{ if .consumer_group }}
 consumer_group: {{ .consumer_group }}
+{{ end }}
+
+{{ if .storage_account }}
 storage_account: {{ .storage_account }}
+{{ end }}
+
+{{ if .storage_account_key }}
 storage_account_key: {{ .storage_account_key }}
+{{ end }}
+
+{{ if .resource_manager_endpoint }}
 resource_manager_endpoint: {{ .resource_manager_endpoint }}
-storage_account_container: filebeat-signinlogs-{{ .eventhub }}
+{{ end }}
+
 tags: {{.tags | tojson}}
 publisher_pipeline.disable_host: {{ inList .tags "forwarded" }}
 processors:

diff --git a/x-pack/heartbeat/Jenkinsfile.yml b/x-pack/heartbeat/Jenkinsfile.yml
@@ -74,3 +74,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/metricbeat/Jenkinsfile.yml b/x-pack/metricbeat/Jenkinsfile.yml
@@ -84,3 +84,9 @@ stages:
  e2e:
  enabled: false
  entrypoint: 'metricbeat-test.sh'
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"
diff --git a/x-pack/packetbeat/Jenkinsfile.yml b/x-pack/packetbeat/Jenkinsfile.yml
@@ -86,3 +86,9 @@ stages:
  packaging-linux: "mage package"
  e2e:
  enabled: false
+ packaging-arm:
+ packaging-arm: "mage package"
+ e2e:
+ enabled: false
+ platforms: ## override default label in this specific stage.
+ - "arm"