Add missing metricsets (#8048)

* add missing metricsets, enable ingest pipeline metricset only if version is 8.7.0 and higher

* Update config/recipes/beats/stack_monitoring.yaml

---------

Co-authored-by: Thibault Richard <thbkrkr@users.noreply.github.com>

config/recipes/beats/stack_monitoring.yaml

@@ -29,8 +29,11 @@ spec:
  - index
  - index_recovery
  - index_summary
+ - ingest_pipeline
  - ml_job
+ - node
  - node_stats
+ - pending_tasks
  - shard
  period: 10s
  hosts: "https://${data.host}:${data.ports.https}"

pkg/controller/common/stackmon/config.go

@@ -13,6 +13,7 @@ import (
  "path/filepath"
  "text/template"
 
+ "github.com/blang/semver/v4"
  "github.com/pkg/errors"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -173,6 +174,7 @@ type inputConfigData struct {
  IsSSL bool
  HasCA bool
  CAPath string
+ Version semver.Version
 }
 
 // buildMetricbeatBaseConfig builds the base configuration for Metricbeat with the Elasticsearch or Kibana modules used
@@ -187,6 +189,7 @@ func buildMetricbeatBaseConfig(
  password string,
  isTLS bool,
  configTemplate string,
+ version semver.Version,
 ) (string, volume.VolumeLike, error) {
  hasCA := false
  if isTLS {
@@ -200,9 +203,10 @@ func buildMetricbeatBaseConfig(
  configData := inputConfigData{
  Username: username,
  Password: password,
- URL: url, // Metricbeat in the sidecar connects to the monitored resource using `localhost`
- IsSSL: isTLS, // enable SSL configuration based on whether the monitored resource has TLS enabled
- HasCA: hasCA, // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
+ URL: url, // Metricbeat in the sidecar connects to the monitored resource using `localhost`
+ IsSSL: isTLS, // enable SSL configuration based on whether the monitored resource has TLS enabled
+ HasCA: hasCA, // the CA is optional to support custom certificate issued by a well-known CA, so without provided CA to configure
+ Version: version, // Version of the monitored resource
  }
 
  var caVolume volume.VolumeLike
@@ -216,9 +220,18 @@ func buildMetricbeatBaseConfig(
  configData.CAPath = filepath.Join(caVolume.VolumeMount().MountPath, certificates.CAFileName)
  }
 
+ templateFuncMap := template.FuncMap{
+ "isVersionGTE": func(minAllowedVersion string) (bool, error) {
+ minAllowedSemver, err := semver.Parse(minAllowedVersion)
+ if err != nil {
+ return false, err
+ }
+ return version.GTE(minAllowedSemver), nil
+ },
+ }
  // render the config template with the config data
  var metricbeatConfig bytes.Buffer
- err := template.Must(template.New("").Parse(configTemplate)).Execute(&metricbeatConfig, configData)
+ err := template.Must(template.New("").Funcs(templateFuncMap).Parse(configTemplate)).Execute(&metricbeatConfig, configData)
  if err != nil {
  return "", nil, err
  }

pkg/controller/common/stackmon/config_test.go

@@ -8,6 +8,7 @@ import (
  "context"
  "testing"
 
+ "github.com/blang/semver/v4"
  "github.com/stretchr/testify/assert"
  corev1 "k8s.io/api/core/v1"
  metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -131,6 +132,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
  certsSecret *corev1.Secret
  hasCA bool
  baseConfig string
+ version semver.Version
  }{
  {
  name: "with TLS and a CA",
@@ -148,7 +150,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
  password: 1234567890
  ssl.enabled: true
  ssl.verification_mode: "certificate"
+ ingest_pipeline: "enabled"
  ssl.certificate_authorities: ["/mnt/elastic-internal/xx-monitoring/namespace/name/certs/ca.crt"]`,
+ version: semver.MustParse("8.7.0"),
  },
  {
  name: "with TLS and no CA",
@@ -164,17 +168,32 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
  username: elastic-internal-monitoring
  password: 1234567890
  ssl.enabled: true
- ssl.verification_mode: "certificate"`,
+ ssl.verification_mode: "certificate"
+ ingest_pipeline: "enabled"`,
+ version: semver.MustParse("8.7.0"),
  },
  {
  name: "without TLS",
  isTLS: false,
+ baseConfig: `
+ hosts: ["scheme://localhost:1234"]
+ username: elastic-internal-monitoring
+ password: 1234567890
+ ssl.enabled: false
+ ssl.verification_mode: "certificate"
+ ingest_pipeline: "enabled"`,
+ version: semver.MustParse("8.7.0"),
+ },
+ {
+ name: "with version less than 8.7.0",
+ isTLS: false,
  baseConfig: `
  hosts: ["scheme://localhost:1234"]
  username: elastic-internal-monitoring
  password: 1234567890
  ssl.enabled: false
  ssl.verification_mode: "certificate"`,
+ version: semver.MustParse("8.6.0"),
  },
  }
  baseConfigTemplate := `
@@ -183,6 +202,9 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
  password: {{ .Password }}
  ssl.enabled: {{ .IsSSL }}
  ssl.verification_mode: "certificate"
+ {{- if isVersionGTE "8.7.0" }}
+ ingest_pipeline: "enabled"
+ {{- end }}
  {{- if .HasCA }}
  ssl.certificate_authorities: ["{{ .CAPath }}"]
  {{- end }}`
@@ -210,6 +232,7 @@ func TestBuildMetricbeatBaseConfig(t *testing.T) {
  "1234567890",
  tc.isTLS,
  baseConfigTemplate,
+ tc.version,
  )
  assert.NoError(t, err)
  assert.Equal(t, tc.baseConfig, baseConfig)

pkg/controller/common/stackmon/sidecar.go

@@ -33,6 +33,11 @@ func NewMetricBeatSidecar(
  password string,
  isTLS bool,
 ) (BeatSidecar, error) {
+ v, err := version.Parse(imageVersion)
+ if err != nil {
+ return BeatSidecar{}, err // error unlikely and should have been caught during validation
+ }
+
  baseConfig, sourceCaVolume, err := buildMetricbeatBaseConfig(
  client,
  associationType,
@@ -43,15 +48,12 @@ func NewMetricBeatSidecar(
  password,
  isTLS,
  baseConfigTemplate,
+ v,
  )
  if err != nil {
  return BeatSidecar{}, err
  }
 
- v, err := version.Parse(imageVersion)
- if err != nil {
- return BeatSidecar{}, err // error unlikely and should have been caught during validation
- }
  image := container.ImageRepository(container.MetricbeatImage, v)
 
  // EmptyDir volume so that MetricBeat does not write in the container image, which allows ReadOnlyRootFilesystem: true

pkg/controller/elasticsearch/stackmon/metricbeat.tpl.yml

@@ -8,10 +8,15 @@ metricbeat.modules:
  - index
  - index_recovery
  - index_summary
+ {{- if isVersionGTE "8.7.0" }}
+ - ingest_pipeline
+ {{- end }}
  - ml_job
+ - node
  - node_stats
  - pending_tasks
  - shard
+
  period: 10s
  xpack.enabled: true
  hosts: ["{{ .URL }}"]