Fix log collection with hints when missing container ports or annotations

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -355,6 +355,28 @@ data:
               #     match: after
             paths:
               - /var/log/containers/*${kubernetes.container.id}.log
+      - name: filestream-generic
+        id: hints-container-logs-${kubernetes.hints.container_id}
+        type: filestream
+        use_output: default
+        streams:
+          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+            data_stream:
+              dataset: kubernetes.hints.container_logs
+              type: logs
+            exclude_files: [ ]
+            exclude_lines: [ ]
+            parsers:
+              - container:
+                  format: auto
+                  stream: ${kubernetes.hints.generic_logs.stream|'all'}
+            paths:
+              - /var/log/containers/*${kubernetes.hints.container_id}.log
+            prospector:
+              scanner:
+                symlinks: true
+            tags: [ ]
+        data_stream.namespace: default
       - id: audit-log
         type: filestream
         use_output: default

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml

@@ -355,6 +355,28 @@ data:
               #     match: after
             paths:
               - /var/log/containers/*${kubernetes.container.id}.log
+      - name: filestream-generic
+        id: hints-container-logs-${kubernetes.hints.container_id}
+        type: filestream
+        use_output: default
+        streams:
+          - condition: ${kubernetes.hints.generic_logs.container_logs.enabled} == true
+            data_stream:
+              dataset: kubernetes.hints.container_logs
+              type: logs
+            exclude_files: [ ]
+            exclude_lines: [ ]
+            parsers:
+              - container:
+                  format: auto
+                  stream: ${kubernetes.hints.generic_logs.stream|'all'}
+            paths:
+              - /var/log/containers/*${kubernetes.hints.container_id}.log
+            prospector:
+              scanner:
+                symlinks: true
+            tags: [ ]
+        data_stream.namespace: default
       - id: audit-log
         type: filestream
         use_output: default

internal/pkg/composable/providers/kubernetes/config.go

@@ -7,8 +7,6 @@ package kubernetes
 import (
 	"time"
 
-	"github.com/elastic/elastic-agent-libs/config"
-
 	"github.com/elastic/elastic-agent-autodiscover/kubernetes"
 	"github.com/elastic/elastic-agent-autodiscover/kubernetes/metadata"
 	"github.com/elastic/elastic-agent-libs/logp"
@@ -37,8 +35,8 @@ type Config struct {
 	LabelsDedot      bool `config:"labels.dedot"`
 	AnnotationsDedot bool `config:"annotations.dedot"`
 
-	Hints  *config.C `config:"hints"`
-	Prefix string    `config:"prefix"`
+	Hints  Hints  `config:"hints"`
+	Prefix string `config:"prefix"`
 }
 
 // Resources config section for resources' config blocks
@@ -48,6 +46,12 @@ type Resources struct {
 	Service Enabled `config:"service"`
 }
 
+// Hints config section for hints' config blocks
+type Hints struct {
+	Enabled              bool `config:"enabled"`
+	DefaultContainerLogs bool `config:"default_container_logs"`
+}
+
 // Enabled config section for resources' config blocks
 type Enabled struct {
 	Enabled bool `config:"enabled"`
@@ -62,6 +66,7 @@ func (c *Config) InitDefaults() {
 	c.AnnotationsDedot = true
 	c.AddResourceMetadata = metadata.GetDefaultResourceMetadataConfig()
 	c.Prefix = "co.elastic"
+	c.Hints.DefaultContainerLogs = true
 }
 
 // Validate ensures correctness of config

internal/pkg/composable/providers/kubernetes/kubernetes.go

@@ -57,7 +57,7 @@ func DynamicProviderBuilder(logger *logger.Logger, c *config.Config, managed boo
 
 // Run runs the kubernetes context provider.
 func (p *dynamicProvider) Run(comm composable.DynamicProviderComm) error {
-	if p.config.Hints.Enabled() {
+	if p.config.Hints.Enabled {
 		betalogger := logp.NewLogger("cfgwarn")
 		betalogger.Warnf("BETA: Hints' feature is beta.")
 	}

internal/pkg/composable/providers/kubernetes/pod.go

@@ -155,7 +155,7 @@ func (p *pod) emitRunning(pod *kubernetes.Pod) {
 	data := generatePodData(pod, p.metagen, namespaceAnnotations)
 	data.mapping["scope"] = p.scope
 
-	if p.config.Hints.Enabled() { // This is "hints based autodiscovery flow"
+	if p.config.Hints.Enabled { // This is "hints based autodiscovery flow"
 		if !p.managed {
 			if ann, ok := data.mapping["annotations"]; ok {
 				annotations, _ := ann.(mapstr.M)
@@ -388,22 +388,26 @@ func generateContainerData(
 				_, _ = containerMeta.Put("port_name", port.Name)
 				k8sMapping["container"] = containerMeta
 
-				if config.Hints.Enabled() { // This is "hints based autodiscovery flow"
+				if config.Hints.Enabled { // This is "hints based autodiscovery flow"
 					if !managed {
-						if ann, ok := k8sMapping["annotations"]; ok {
-							annotations, _ := ann.(mapstr.M)
-							hints := utils.GenerateHints(annotations, "", config.Prefix)
-							if len(hints) > 0 {
-								logger.Debugf("Extracted hints are :%v", hints)
-								hintsMapping := GenerateHintsMapping(hints, k8sMapping, logger, c.ID)
-								logger.Debugf("Generated hints mappings are :%v", hintsMapping)
-								_ = comm.AddOrUpdate(
-									eventID,
-									PodPriority,
-									map[string]interface{}{"hints": hintsMapping},
-									processors,
-								)
-							}
+						hintsMapping := getHintsMapping(k8sMapping, logger, config.Prefix, c.ID)
+						if len(hintsMapping) > 0 {
+							_ = comm.AddOrUpdate(
+								eventID,
+								PodPriority,
+								map[string]interface{}{"hints": hintsMapping},
+								processors,
+							)
+						} else if config.Hints.DefaultContainerLogs {
+							// in case of no package detected in the hints fallback to the generic log collection
+							_, _ = hintsMapping.Put("generic_logs.container_logs.enabled", true)
+							_, _ = hintsMapping.Put("container_id", c.ID)
+							_ = comm.AddOrUpdate(
+								eventID,
+								PodPriority,
+								map[string]interface{}{"hints": hintsMapping},
+								processors,
+							)
 						}
 					}
 				} else { // This is the "template-based autodiscovery" flow
@@ -412,7 +416,45 @@ func generateContainerData(
 			}
 		} else {
 			k8sMapping["container"] = containerMeta
-			_ = comm.AddOrUpdate(eventID, ContainerPriority, k8sMapping, processors)
+			if config.Hints.Enabled { // This is "hints based autodiscovery flow"
+				if !managed {
+					hintsMapping := getHintsMapping(k8sMapping, logger, config.Prefix, c.ID)
+					if len(hintsMapping) > 0 {
+						_ = comm.AddOrUpdate(
+							eventID,
+							PodPriority,
+							map[string]interface{}{"hints": hintsMapping},
+							processors,
+						)
+					} else if config.Hints.DefaultContainerLogs {
+						// in case of no package detected in the hints fallback to the generic log collection
+						_, _ = hintsMapping.Put("generic_logs.container_logs.enabled", true)
+						_, _ = hintsMapping.Put("container_id", c.ID)
+						_ = comm.AddOrUpdate(
+							eventID,
+							PodPriority,
+							map[string]interface{}{"hints": hintsMapping},
+							processors,
+						)
+					}
+				}
+			} else { // This is the "template-based autodiscovery" flow
+				_ = comm.AddOrUpdate(eventID, ContainerPriority, k8sMapping, processors)
+			}
+		}
+	}
+}
+
+func getHintsMapping(k8sMapping map[string]interface{}, logger *logp.Logger, prefix string, cID string) mapstr.M {
+	hintsMapping := mapstr.M{}
+	if ann, ok := k8sMapping["annotations"]; ok {
+		annotations, _ := ann.(mapstr.M)
+		hints := utils.GenerateHints(annotations, "", prefix)
+		if len(hints) > 0 {
+			logger.Debugf("Extracted hints are :%v", hints)
+			hintsMapping = GenerateHintsMapping(hints, k8sMapping, logger, cID)
+			logger.Debugf("Generated hints mappings are :%v", hintsMapping)
 		}
 	}
+	return hintsMapping
 }