Only add Kibana config for Beats that support it

config/crds/all-crds.yaml

@@ -557,8 +557,8 @@ spec:
               type: string
             kibanaRef:
               description: KibanaRef is a reference to a Kibana instance running in
-                the same Kubernetes cluster. It allows APM agent central configuration
-                management in Kibana.
+                the same Kubernetes cluster. It allows automatic setup of dashboards
+                and visualizations.
               properties:
                 name:
                   description: Name of the Kubernetes object.

config/crds/bases/beat.k8s.elastic.co_beats.yaml

@@ -12162,8 +12162,8 @@ spec:
               type: string
             kibanaRef:
               description: KibanaRef is a reference to a Kibana instance running in
-                the same Kubernetes cluster. It allows APM agent central configuration
-                management in Kibana.
+                the same Kubernetes cluster. It allows automatic setup of dashboards
+                and visualizations.
               properties:
                 name:
                   description: Name of the Kubernetes object.

pkg/controller/beat/common/config.go

@@ -20,6 +20,13 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 )
 
+type DefaultConfigs struct {
+	// Managed config is handled by ECK and will not be nullified by user provided config.
+	Managed *settings.CanonicalConfig
+	// Unmanaged config is default config that will be overridden by any user provided config.
+	Unmanaged *settings.CanonicalConfig
+}
+
 // setOutput will set the output section in Beat config according to the association configuration.
 func buildOutputConfig(client k8s.Client, associated beatv1beta1.BeatESAssociation) (*settings.CanonicalConfig, error) {
 	if !associated.AssociationConf().IsConfigured() {
@@ -46,7 +53,8 @@ func buildOutputConfig(client k8s.Client, associated beatv1beta1.BeatESAssociati
 	return settings.MustCanonicalConfig(esOutput), nil
 }
 
-func buildKibanaConfig(client k8s.Client, associated beatv1beta1.BeatKibanaAssociation) (*settings.CanonicalConfig, error) {
+// BuildKibanaConfig builds on optional Kibana configuration for dashboard setup and visualizations.
+func BuildKibanaConfig(client k8s.Client, associated beatv1beta1.BeatKibanaAssociation) (*settings.CanonicalConfig, error) {
 	if !associated.AssociationConf().IsConfigured() {
 		return settings.NewCanonicalConfig(), nil
 	}
@@ -75,24 +83,22 @@ func buildBeatConfig(
 	log logr.Logger,
 	client k8s.Client,
 	beat beatv1beta1.Beat,
-	defaultConfig *settings.CanonicalConfig,
+	defaultConfigs DefaultConfigs,
 ) ([]byte, error) {
 	cfg := settings.NewCanonicalConfig()
 
 	outputCfg, err := buildOutputConfig(client, beatv1beta1.BeatESAssociation{Beat: &beat})
 	if err != nil {
 		return nil, err
 	}
-	kibanaCfg, err := buildKibanaConfig(client, beatv1beta1.BeatKibanaAssociation{Beat: &beat})
-
-	err = cfg.MergeWith(outputCfg, kibanaCfg)
+	err = cfg.MergeWith(outputCfg, defaultConfigs.Managed)
 	if err != nil {
 		return nil, err
 	}
 	// use only the default config or only the provided config - no overriding, no merging
 	userConfig := beat.Spec.Config
 	if userConfig == nil {
-		if err := cfg.MergeWith(defaultConfig); err != nil {
+		if err := cfg.MergeWith(defaultConfigs.Unmanaged); err != nil {
 			return nil, err
 		}
 	} else {
@@ -112,10 +118,10 @@ func buildBeatConfig(
 
 func reconcileConfig(
 	params DriverParams,
-	defaultConfig *settings.CanonicalConfig,
+	defaultConfigs DefaultConfigs,
 	configHash hash.Hash,
 ) error {
-	cfgBytes, err := buildBeatConfig(params.Logger, params.Client, params.Beat, defaultConfig)
+	cfgBytes, err := buildBeatConfig(params.Logger, params.Client, params.Beat, defaultConfigs)
 	if err != nil {
 		return err
 	}

pkg/controller/beat/common/config_test.go

@@ -36,7 +36,8 @@ func Test_buildBeatConfig(t *testing.T) {
 		},
 	)
 
-	defaultConfig := settings.MustParseConfig([]byte("default: true"))
+	unmanagedDefaultCfg := settings.MustParseConfig([]byte("default: true"))
+	managedDefaultCfg := settings.MustParseConfig([]byte("setup.kibana: true"))
 	userConfig := &commonv1.Config{Data: map[string]interface{}{"user": "true"}}
 	userCanonicalConfig := settings.MustCanonicalConfig(userConfig.Data)
 	outputCAYaml := settings.MustParseConfig([]byte(`output.elasticsearch.ssl.certificate_authorities: 
@@ -83,7 +84,7 @@ func Test_buildBeatConfig(t *testing.T) {
 		name          string
 		client        k8s.Client
 		beat          beatv1beta1.Beat
-		defaultConfig *settings.CanonicalConfig
+		defaultConfig DefaultConfigs
 		want          *settings.CanonicalConfig
 		wantErr       bool
 	}{
@@ -94,8 +95,8 @@ func Test_buildBeatConfig(t *testing.T) {
 		{
 			name:          "no association, only default config",
 			beat:          beatv1beta1.Beat{},
-			defaultConfig: defaultConfig,
-			want:          defaultConfig,
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
+			want:          unmanagedDefaultCfg,
 		},
 		{
 			name: "no association, only user config",
@@ -109,15 +110,27 @@ func Test_buildBeatConfig(t *testing.T) {
 			beat: beatv1beta1.Beat{Spec: beatv1beta1.BeatSpec{
 				Config: userConfig,
 			}},
-			defaultConfig: defaultConfig,
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
 			want:          userCanonicalConfig,
 		},
+		{
+			name: "no association, both default configs and user config",
+			beat: beatv1beta1.Beat{Spec: beatv1beta1.BeatSpec{
+				Config: userConfig,
+			}},
+			defaultConfig: DefaultConfigs{
+				Unmanaged: unmanagedDefaultCfg,
+				Managed:   managedDefaultCfg,
+			},
+			want: merge(userCanonicalConfig, managedDefaultCfg),
+		},
+
 		{
 			name:          "association without ca, only default config",
 			client:        clientWithSecret,
 			beat:          withAssociation,
-			defaultConfig: defaultConfig,
-			want:          merge(defaultConfig, outputYaml),
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
+			want:          merge(unmanagedDefaultCfg, outputYaml),
 		},
 		{
 			name:   "association without ca, only user config",
@@ -129,15 +142,15 @@ func Test_buildBeatConfig(t *testing.T) {
 			name:          "association without ca, default and user config",
 			client:        clientWithSecret,
 			beat:          withAssociationWithConfig,
-			defaultConfig: defaultConfig,
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
 			want:          merge(userCanonicalConfig, outputYaml),
 		},
 		{
 			name:          "association with ca, only default config",
 			client:        clientWithSecret,
 			beat:          withAssociationWithCA,
-			defaultConfig: defaultConfig,
-			want:          merge(defaultConfig, outputYaml, outputCAYaml),
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
+			want:          merge(unmanagedDefaultCfg, outputYaml, outputCAYaml),
 		},
 		{
 			name:   "association with ca, only user config",
@@ -149,9 +162,19 @@ func Test_buildBeatConfig(t *testing.T) {
 			name:          "association with ca, default and user config",
 			client:        clientWithSecret,
 			beat:          withAssociationWithCAAndConfig,
-			defaultConfig: defaultConfig,
+			defaultConfig: DefaultConfigs{Unmanaged: unmanagedDefaultCfg},
 			want:          merge(userCanonicalConfig, outputYaml, outputCAYaml),
 		},
+		{
+			name:   "association with ca, both default configs and user config",
+			client: clientWithSecret,
+			beat:   withAssociationWithCAAndConfig,
+			defaultConfig: DefaultConfigs{
+				Unmanaged: unmanagedDefaultCfg,
+				Managed:   managedDefaultCfg,
+			},
+			want: merge(userCanonicalConfig, managedDefaultCfg, outputYaml, outputCAYaml),
+		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
 			gotYaml, gotErr := buildBeatConfig(logrtesting.NullLogger{}, tt.client, tt.beat, tt.defaultConfig)

pkg/controller/beat/common/driver.go

@@ -17,7 +17,6 @@ import (
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/container"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/defaults"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/reconciler"
-	"github.com/elastic/cloud-on-k8s/pkg/controller/common/settings"
 	"github.com/elastic/cloud-on-k8s/pkg/utils/k8s"
 )
 
@@ -56,7 +55,7 @@ func ValidateBeatSpec(spec beatv1beta1.BeatSpec) error {
 
 func Reconcile(
 	params DriverParams,
-	defaultConfig *settings.CanonicalConfig,
+	defaultConfigs DefaultConfigs,
 	defaultImage container.Image,
 	modifyPodFunc func(builder *defaults.PodTemplateBuilder),
 ) *reconciler.Results {
@@ -71,7 +70,7 @@ func Reconcile(
 	}
 
 	configHash := sha256.New224()
-	if err := reconcileConfig(params, defaultConfig, configHash); err != nil {
+	if err := reconcileConfig(params, defaultConfigs, configHash); err != nil {
 		return results.WithError(err)
 	}
 

pkg/controller/beat/filebeat/config.go

@@ -4,7 +4,11 @@
 
 package filebeat
 
-import "github.com/elastic/cloud-on-k8s/pkg/controller/common/settings"
+import (
+	"github.com/elastic/cloud-on-k8s/pkg/apis/beat/v1beta1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/beat/common"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/common/settings"
+)
 
 var (
 	defaultConfig = settings.MustParseConfig([]byte(
@@ -24,3 +28,14 @@ processors:
 - add_host_metadata: {}
 `))
 )
+
+func (d *Driver) configParams() (common.DefaultConfigs, error) {
+	kibanaConfig, err := common.BuildKibanaConfig(d.Client, v1beta1.BeatKibanaAssociation{Beat: &d.Beat})
+	if err != nil {
+		return common.DefaultConfigs{}, err
+	}
+	return common.DefaultConfigs{
+		Managed:   kibanaConfig,
+		Unmanaged: defaultConfig,
+	}, nil
+}

pkg/controller/beat/filebeat/filebeat.go

@@ -59,9 +59,14 @@ func (d *Driver) Reconcile() *reconciler.Results {
 		}
 	}
 
+	defaultConfigs, err := d.configParams()
+	if err != nil {
+		return reconciler.NewResult(d.DriverParams.Context).WithError(err)
+	}
+
 	return beatcommon.Reconcile(
 		d.DriverParams,
-		defaultConfig,
+		defaultConfigs,
 		container.FilebeatImage,
 		f)
 }

pkg/controller/beat/metricbeat/config.go

@@ -5,6 +5,8 @@
 package metricbeat
 
 import (
+	"github.com/elastic/cloud-on-k8s/pkg/apis/beat/v1beta1"
+	"github.com/elastic/cloud-on-k8s/pkg/controller/beat/common"
 	"github.com/elastic/cloud-on-k8s/pkg/controller/common/settings"
 )
 
@@ -42,3 +44,14 @@ processors:
 - add_cloud_metadata: {}
 `))
 )
+
+func (d *Driver) configParams() (common.DefaultConfigs, error) {
+	kibanaConfig, err := common.BuildKibanaConfig(d.Client, v1beta1.BeatKibanaAssociation{Beat: &d.Beat})
+	if err != nil {
+		return common.DefaultConfigs{}, err
+	}
+	return common.DefaultConfigs{
+		Managed:   kibanaConfig,
+		Unmanaged: defaultConfig,
+	}, nil
+}

pkg/controller/beat/metricbeat/metricbeat.go

@@ -62,9 +62,14 @@ func (d *Driver) Reconcile() *reconciler.Results {
 		builder.WithArgs("-e", "-c", beatcommon.ConfigMountPath, "-system.hostfs=/hostfs")
 	}
 
+	defaultConfigs, err := d.configParams()
+	if err != nil {
+		return reconciler.NewResult(d.DriverParams.Context).WithError(err)
+	}
+
 	return beatcommon.Reconcile(
 		d.DriverParams,
-		defaultConfig,
+		defaultConfigs,
 		container.MetricbeatImage,
 		f)
 }

pkg/controller/beat/otherbeat/otherbeat.go

@@ -28,5 +28,5 @@ func NewDriver(params beatcommon.DriverParams) beatcommon.Driver {
 }
 
 func (d *Driver) Reconcile() *reconciler.Results {
-	return beatcommon.Reconcile(d.DriverParams, nil, "", nil)
+	return beatcommon.Reconcile(d.DriverParams, beatcommon.DefaultConfigs{}, "", nil)
 }