Introduce datasources in package to configure inputs and streams

dev/package-examples/nginx-1.2.0/dataset/error/agent/config/input.yml → dev/package-examples/nginx-1.2.0/dataset/access/agent/stream/stream.yml

@@ -1,8 +1,10 @@
-type: log
+
+input: log
+paths:
 {{#each paths}}
-paths: "{{this}}"
+  - "{{this}}"
 {{/each}}
-ingest_pipeline: {{pipeline}}
+
 exclude_files: [".gz$"]
 
 processors:

dev/package-examples/nginx-1.2.0/dataset/access/manifest.yml

@@ -8,18 +8,26 @@ type: logs
 # The ingest pipeline which should be used.
 ingest_pipeline: default
 
-vars:
-  - name: paths
-    # Should we define this as array? How will the UI best make sense of it?
-    type: textarea
-    default:
-      - /var/log/nginx/access.log*
-    # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
-    # This would need to be based on a predefined definition on what can be filtered on
-    os.darwin:
-      - /usr/local/var/log/nginx/access.log*
-    os.windows:
-      - c:/programdata/nginx/logs/*access.log*
+# List of supported inputs
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        # Should we define this as array? How will the UI best make sense of it?
+        description: Paths to the nginx access log file.
+        type: text
+        multi: true
+        default:
+          - /var/log/nginx/access.log*
+        # I suggest to use ECS fields for this config options here: https://github.com/elastic/ecs/blob/master/schemas/os.yml
+        # This would need to be based on a predefined definition on what can be filtered on
+        os.darwin:
+          default:
+          - /usr/local/var/log/nginx/access.log*
+        os.windows:
+          default:
+          - c:/programdata/nginx/logs/*access.log*
 
 requirements:
   elasticsearch.processors:

dev/package-examples/nginx-1.2.0/dataset/error/agent/stream/stream.yml

@@ -0,0 +1,23 @@
+
+# The selected input has to be passed to the stream config when processed.
+
+{{#if input == log}}
+input: log
+
+{{#each paths}}
+paths: "{{this}}"
+{{/each}}
+exclude_files: [".gz$"]
+
+processors:
+  - add_locale: ~
+{{/if}}
+
+
+# This is an example stream config on how multiple inputs could be supported
+
+{{#if input == syslog}}
+input: syslog
+
+# TODO: would need some more config options
+{{/if}}

dev/package-examples/nginx-1.2.0/dataset/error/manifest.yml

@@ -1,11 +1,29 @@
 title: Nginx Error Logs
 type: logs
 ingest_pipeline: pipeline
-vars:
-  - name: paths
-    default:
-      - /var/log/nginx/error.log*
-    os.darwin:
-      - /usr/local/var/log/nginx/error.log*
-    os.windows:
-      - c:/programdata/nginx/logs/error.log*
+
+
+# This is an example that multiple inputs are supported by one dataset
+inputs:
+  - type: log
+    vars:
+      - name: paths
+        required: true
+        default:
+          - /var/log/nginx/error.log*
+
+        # TODO: The exact definition of os specific paths still needs to be defined
+        os.darwin:
+          - /usr/local/var/log/nginx/error.log*
+        os.windows:
+          - c:/programdata/nginx/logs/error.log*
+
+  - type: syslog
+    vars:
+      # Are udp and tcp syslog input two different inputs?
+      - name: protocol.udp.host
+        required: true
+        default:
+          - "localhost:9000"
+
+

dev/package-examples/nginx-1.2.0/dataset/stubstatus/agent/input/input.yml → dev/package-examples/nginx-1.2.0/dataset/stubstatus/agent/stream/stream.yml

@@ -1,4 +1,5 @@
-type: metric/nginx
+# Defines which input to use, should be stripped out when creating the config
+input: metrics/nginx
 metricsets: ["stubstatus"]
 period: {{period}}
 enabled: true

dev/package-examples/nginx-1.2.0/dataset/stubstatus/manifest.yml

@@ -10,17 +10,10 @@ compatibility: linux, freebsd
 # Each input can be in its own release status
 release: beta
 
-vars:
-  - name: hosts
-    description: Nginx hosts
-    default:
-      ["http://127.0.0.1"]
-    required: true
-  - name: period
-    description: "Collection period. Valid values: 10s, 5m, 2h"
-    default: "10s"
-  - name: username
-    type: text
-  - name: password
-    # This is the html input type?
-    type: password
+# List of inputs this dataset supports
+inputs:
+  - type: "metric/nginx"
+
+    # Only the variables have to be repeated that are not specified as part of the input
+    vars:
+      # All variables are specified in the input already

dev/package-examples/nginx-1.2.0/manifest.yml

@@ -12,3 +12,50 @@ requirement:
     versions: ">7.1.0 <7.6.0"
   elasticsearch:
     versions: ">7.0.1"
+
+datasources:
+  -
+    # Do we need a name for the data source?
+    name: nginx
+
+    # List of inputs this datasource supports
+    inputs:
+      -
+        # An id can be given, in case the type used here is not unique
+        # This is for selection in the stream
+        # id: nginx
+        type: nginx/metrics
+        descrition: Collecting metrics for nginx.
+
+        # Common configuration options for this input
+        vars:
+          - name: hosts
+            description: Nginx hosts
+            default:
+              ["http://127.0.0.1"]
+            # All the config options that are required should be shown in the UI
+            required: true
+            multi: true
+            type: text
+          - name: period
+            description: "Collection period. Valid values: 10s, 5m, 2h"
+            default: "10s"
+            type: duration
+          - name: username
+            type: text
+          - name: password
+            # This is the html input type?
+            type: password
+
+      -
+        type: logs
+        description: Collect nginx logs.
+
+        # Common configuration options for this input
+        vars:
+
+      -
+        type: syslog
+
+        # Common configuration options for this input
+        vars:

dev/package-examples/system-0.9.0/dataset/cpu/agent/stream/stream.yml

@@ -0,0 +1,6 @@
+input: metrics/system
+enabled: false # default true
+metricset: cpu
+period: 10s
+dataset: system.cpu
+metrics: ["percentages", "normalized_percentages"]

dev/package-examples/system-0.9.0/dataset/cpu/manifest.yml

@@ -5,4 +5,5 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
-
+inputs:
+  - type: metrics/system

dev/package-examples/system-0.9.0/dataset/load/agent/stream/stream.yml

@@ -0,0 +1,4 @@
+inputy: metric/system
+enabled: true
+metricsets:
+  - load

dev/package-examples/system-0.9.0/dataset/load/manifest.yml

@@ -5,4 +5,5 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
-
+inputs:
+  - type: system/metrics

dev/package-examples/system-0.9.0/dataset/memory/agent/stream.yml/stream.yml

@@ -0,0 +1,4 @@
+input: metric/system
+enabled: true
+metricsets:
+  - memory

dev/package-examples/system-0.9.0/dataset/memory/manifest.yml

@@ -5,4 +5,6 @@ release: beta
 # Needs to describe the type of this input
 type: metrics
 
+inputs:
+  - type: system/metrics
 

dev/package-examples/system-0.9.0/manifest.yml

@@ -10,3 +10,10 @@ release: ga
 requirement:
   kibana:
     versions: "<8.0.0"
+
+
+datasources:
+  - inputs:
+      type: metrics/system
+  - inputs:
+      type: log