TSDB Enablement for InfluxDB

packages/influxdb/changelog.yml

@@ -1,4 +1,9 @@
 # newer versions go on top
+- version: "0.3.0"
+  changes:
+    - description: TSDB Enablment changes included, Minor bug fixes.
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/5312
 - version: "0.2.1"
   changes:
     - description: Added categories and/or subcategories.

packages/influxdb/data_stream/advstatus/elasticsearch/ingest_pipeline/default.yml

@@ -6,10 +6,6 @@ processors:
         target_field: influxdb.advstatus
         ignore_missing: true
         ignore_failure: true
-    - remove:
-        field: [influxdb.advstatus.labels.compiler_type, influxdb.advstatus.labels.job, influxdb.advstatus.labels.id, influxdb.advstatus.labels.level, influxdb.advstatus.labels.engine, influxdb.advstatus.labels.handler, influxdb.advstatus.labels.path, influxdb.advstatus.labels.taskID, influxdb.advstatus.labels.task_type, influxdb.advstatus.labels.quantile]
-        ignore_missing: true
-        ignore_failure: true
     - rename:
         field: influxdb.advstatus.labels.instance
         target_field: influxdb.advstatus.instance
@@ -20,6 +16,11 @@ processors:
         target_field: influxdb.advstatus.org
         ignore_failure: true
         ignore_missing: true
+    - rename:
+        field: influxdb.advstatus.labels.taskID
+        target_field: influxdb.advstatus.labels.taskid
+        ignore_failure: true
+        ignore_missing: true
     - rename:
         field: influxdb.advstatus.qc_all_duration_seconds.histogram
         target_field: influxdb.advstatus.query_controller.all_duration_seconds.histogram

packages/influxdb/data_stream/advstatus/fields/ecs.yml

@@ -4,5 +4,24 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
+- external: ecs
+  name: cloud.project.id
+  dimension: true
+- external: ecs
+  name: cloud.instance.id
+  dimension: true
+- external: ecs
+  name: cloud.provider
+  dimension: true
+- external: ecs
+  name: container.id
+  dimension: true
+- external: ecs
+  name: host.name
+  dimension: true

packages/influxdb/data_stream/advstatus/fields/fields.yml

@@ -6,10 +6,87 @@
       type: keyword
       description: |
         InfluxDB instance.
+      # Reason to add as a dimension field: To support multiple influxdb instances
+      dimension: true
     - name: org
       type: keyword
       description: |
         Organization id of the Organization created in InfluxDB.
+      # Reason to add as a dimension field: To support multiple orgs
+      dimension: true
+    - name: labels
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          # Reason to add as a dimension field: To support multiple orgs
+          dimension: true
+        - name: job
+          type: keyword
+          description: Type of the job
+          # Reason to add as a dimension field: To support multiple job type
+          dimension: true
+        - name: engine
+          type: keyword
+          # Reason to add as a dimension field: To support multiple TSDB storage engine type
+          dimension: true
+          description: TSDB storage engine
+        - name: walPath
+          type: keyword
+          # Reason to add as a dimension field: To support multiple walPath locations
+          dimension: true
+          description: Path to the WAL file
+        - name: handler
+          type: keyword
+          # Reason to add as a dimension field: To support different request handler type
+          dimension: true
+          description: Request handler.
+        - name: user_agent 
+          type: keyword
+          # Reason to add as a dimension field: To support multiple user agent type
+          dimension: true
+          description: Type of user agent
+        - name: compiler_type
+          type: keyword
+          # Reason to add as a dimension field: To support multiple compiler type
+          dimension: true
+          description: Type of the compiler
+        - name: task_type
+          type: keyword
+          description: Type of the task
+          # Reason to add as a dimension field: To support multiple task types
+          dimension: true
+        - name: bucket
+          type: keyword
+          description: Bucket ID
+          # Reason to add as a dimension field: To support more than one bucket ids
+          dimension: true 
+        - name: taskid
+          type: keyword
+          description: Task ID of the influxdb tasks
+          # Reason to add as a dimension field: To support multiple task id
+          dimension: true
+        - name: quantile
+          type: keyword
+          description: Number that indicates the histogram quantile value. 
+          # Reason to add as a dimension field: To support different quantile limit values
+          dimension: true
+        - name: method
+          type: keyword
+          description: Type of service operation
+          # Reason to add as a dimension field: To support multiple service operation
+          dimension: true  
+        - name: op
+          type: keyword
+          description: Extended information related to various operations
+          # Reason to add as a dimension field: To support multiple operation types
+          dimension: true
+        - name: path
+          type: keyword
+          # Reason to add as a dimension field: To support multiple http api paths
+          dimension: true
+          description: |
+            HTTP request endpoint.
     - name: query_controller
       type: group
       fields:

packages/influxdb/data_stream/advstatus/manifest.yml

@@ -23,3 +23,8 @@ streams:
         type: text
         title: Period
         default: 60s
+
+elasticsearch:
+  index_template:
+    settings:
+      index.mapping.dimension_fields.limit: 24

packages/influxdb/data_stream/status/elasticsearch/ingest_pipeline/default.yml

@@ -1,15 +1,10 @@
 ---
     description: Pipeline for Influxdb status metrics
     processors:
-        - rename:
+        - rename:   
             field: prometheus.metrics
             target_field: influxdb.status
             ignore_missing: true
-            ignore_failure: true
-        - remove:
-            field: [prometheus.labels.compiler_type, prometheus.labels.job, prometheus.labels.id, prometheus.labels.level, prometheus.labels.engine, prometheus.labels.user_agent, prometheus.labels.handler, prometheus.labels.task_type]
-            ignore_missing: true
-            ignore_failure: true
         - rename:
             field: influxdb.status.go_memstats_alloc_bytes
             target_field: influxdb.status.go_runtime.memstats_alloc_bytes
@@ -222,6 +217,11 @@
             field: prometheus.labels.org
             target_field: influxdb.status.org
             ignore_missing: true
+        - rename:
+            field: prometheus.labels
+            target_field: influxdb.status.label
+            ignore_missing: true
+            ignore_failure: true
     on_failure:
     - set:
         field: error.message

packages/influxdb/data_stream/status/fields/base-fields.yml

@@ -11,3 +11,4 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
+

packages/influxdb/data_stream/status/fields/ecs.yml

@@ -4,5 +4,24 @@
   name: ecs.version
 - external: ecs
   name: service.address
+  dimension: true
 - external: ecs
   name: service.type
+- external: ecs
+  name: agent.id
+  dimension: true
+- external: ecs
+  name: cloud.project.id
+  dimension: true
+- external: ecs
+  name: cloud.instance.id
+  dimension: true
+- external: ecs
+  name: cloud.provider
+  dimension: true
+- external: ecs
+  name: container.id
+  dimension: true
+- external: ecs
+  name: host.name
+  dimension: true

packages/influxdb/data_stream/status/fields/fields.yml

@@ -4,12 +4,83 @@
   fields:
     - name: instance
       type: keyword
+      # Reason to add as a dimension field: To support multiple influxdb instances 
+      dimension: true
       description: |
         InfluxDB instance.
     - name: org
       type: keyword
+      # Reason to add as a dimension field: To support multiple orgs
+      dimension: true
       description: |
         Organization id of the Organization created in InfluxDB.
+    - name: label
+      type: group
+      fields:
+        - name: id
+          type: keyword
+          # Reason to add as a dimension field: To support multiple orgs
+          dimension: true
+        - name: job
+          type: keyword
+          description: Type of the job
+          # Reason to add as a dimension field: To support multiple job type
+          dimension: true
+        - name: engine
+          type: keyword
+          # Reason to add as a dimension field: To support multiple TSDB storage engine type
+          dimension: true
+          description: TSDB storage engine
+        - name: walPath
+          type: keyword
+          # Reason to add as a dimension field: To support multiple walPath locations
+          dimension: true
+          description: Path to the WAL file
+        - name: handler
+          type: keyword
+          # Reason to add as a dimension field: To support different request handler type
+          dimension: true
+          description: Request handler.
+        - name: user_agent 
+          type: keyword
+          # Reason to add as a dimension field: To support multiple user agent type
+          dimension: true
+          description: Type of user agent
+        - name: compiler_type
+          type: keyword
+          # Reason to add as a dimension field: To support multiple compiler type
+          dimension: true
+          description: Type of the compiler
+        - name: task_type
+          type: keyword
+          description: Type of the task
+          # Reason to add as a dimension field: To support multiple task types
+          dimension: true
+        - name: bucket
+          type: keyword
+          description: Bucket ID
+          # Reason to add as a dimension field: To support more than one bucket ids
+          dimension: true 
+        - name: taskid
+          type: keyword
+          # Reason to add as a dimension field: To support multiple task id
+          description: Task ID of the influxdb tasks
+          dimension: true
+        - name: quantile
+          type: keyword
+          description: Number that indicates the histogram quantile value. 
+          # Reason to add as a dimension field: To support different quantile limit values
+          dimension: true
+        - name: method
+          type: keyword
+          description: Type of service operation
+          # Reason to add as a dimension field: To support multiple service operation
+          dimension: true  
+        - name: op
+          type: keyword
+          # Reason to add as a dimension field: To support multiple operation types
+          description: Extended information related to various operations
+          dimension: true 
     - name: http_api_requests_total
       metric_type: counter
       type: double
@@ -225,25 +296,25 @@
       fields:
         - name: method
           type: keyword
+          # Reason to add as a dimension field: To support multiple http API method
+          dimension: true
           description: |
             HTTP request method.
         - name: path
           type: keyword
+          # Reason to add as a dimension field: To support multiple http api paths
+          dimension: true
           description: |
             HTTP request endpoint.
         - name: response_code
           type: keyword
+          # Reason to add as a dimension field: To support multiple HTTP response code
+          dimension: true
           description: |
             Response code of HTTP API request.
         - name: http_status
           type: keyword
+          dimension: true
+          # Reason to add as a dimension field: To support multiple HTTP status code
           description: |
             HTTP API request call status.
-    - name: walPath
-      type: keyword
-      description: |
-        Directory path where InfluxDB stores Write Ahead Log.
-    - name: bucket
-      type: keyword
-      description: |
-        Bucket id of the bucket where time series data is stored.

packages/influxdb/data_stream/status/manifest.yml

@@ -23,3 +23,8 @@ streams:
         type: text
         title: Period
         default: 60s
+
+elasticsearch:
+  index_template:
+    settings:
+      index.mapping.dimension_fields.limit: 32