Update kubernetes namespace field. (#1792)

* Use pipeline to rename fileds in cases of agents 7.x and stack ^8.x
elastic · Oct 25, 2021 · b1ee043 · b1ee043
1 parent 559bc03
commit b1ee043
Show file tree

Hide file tree

Showing 54 changed files with 1,201 additions and 95 deletions.
diff --git a/packages/kubernetes/changelog.yml b/packages/kubernetes/changelog.yml
@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "1.2.0"
+  changes:
+    - description: Update Kubernetes namespace field
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1792
 - version: "1.1.1"
   changes:
     - description: Update Kubernetes integration Readme

diff --git a/packages/kubernetes/data_stream/container/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/container/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes container metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/container/fields/base-fields.yml b/packages/kubernetes/data_stream/container/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/container_logs/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/container_logs/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes container logs.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml b/packages/kubernetes/data_stream/container_logs/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/controllermanager/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/controllermanager/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes controllermanager metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/controllermanager/fields/base-fields.yml b/packages/kubernetes/data_stream/controllermanager/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/event/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/event/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes event metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/event/fields/base-fields.yml b/packages/kubernetes/data_stream/event/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/event/fields/fields.yml b/packages/kubernetes/data_stream/event/fields/fields.yml
@@ -96,4 +96,4 @@
         - name: uid
           type: keyword
           description: |
-            UUID version of the object
+            uid version of the object
diff --git a/packages/kubernetes/data_stream/node/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/node/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes node metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/node/fields/base-fields.yml b/packages/kubernetes/data_stream/node/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/pod/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/pod/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes pod metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'
diff --git a/packages/kubernetes/data_stream/pod/fields/base-fields.yml b/packages/kubernetes/data_stream/pod/fields/base-fields.yml
@@ -29,9 +29,31 @@
         Kubernetes pod IP
 
     - name: namespace
-      type: keyword
-      description: >
-        Kubernetes namespace
+      type: group
+      fields:
+        - name: name
+          type: keyword
+          description: >
+            Kubernetes namespace name
+
+        - name: uid
+          type: keyword
+          description: >
+            Kubernetes namespace uid
+
+        - name: labels.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace labels map
+
+        - name: annotations.*
+          type: object
+          object_type: keyword
+          object_type_mapping_type: "*"
+          description: >
+            Kubernetes namespace annotations map
 
     - name: node.name
       type: keyword

diff --git a/packages/kubernetes/data_stream/scheduler/elasticsearch/ingest_pipeline/default.yml b/packages/kubernetes/data_stream/scheduler/elasticsearch/ingest_pipeline/default.yml
@@ -0,0 +1,23 @@
+---
+description: Pipeline for parsing kubernetes scheduler metrics.
+processors:
+  - rename:
+      field: kubernetes.namespace
+      target_field: kubernetes.namespace.name
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_labels
+      target_field: kubernetes.namespace.labels
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_annotations
+      target_field: kubernetes.namespace.annotations
+      ignore_missing: true
+  - rename:
+      field: kubernetes.namespace_uid
+      target_field: kubernetes.namespace.uid
+      ignore_missing: true
+on_failure:
+  - set:
+      field: error.message
+      value: '{{ _ingest.on_failure_message }}'