Add datastream fields to all datasets (#213)

It is planned to move from dataset.* to datastream.*. To make the path easier to 7.9 Elasticsearch (elastic/elasticsearch#60592) and the Elastic Agent already ship with the datastream.* fields. Because of this, these should also be added to the mappings of the packages.

The agent will ship datastream.* fields. Because of this, the import scripts can be adjusted to only use these fields.

This PR doese not rename any Golang variables. This should be done in a follow up and in sync with potential changes to the registry.

dev/import-beats/fields_base_fields.go

@@ -8,6 +8,23 @@ var baseFields = createBaseFields()
 
 func createBaseFields() []fieldDefinition {
 	return []fieldDefinition{
+		{
+			Name:        "datastream.type",
+			Type:        "constant_keyword",
+			Description: "Datastream type.",
+		},
+		{
+			Name:        "datastream.dataset",
+			Type:        "constant_keyword",
+			Description: "Datastream dataset name.",
+		},
+		{
+			Name:        "datastream.namespace",
+			Type:        "constant_keyword",
+			Description: "Datastream namespace.",
+		},
+		// TODO: This should be removed as soon as it is not a requirement anymore by the validation
+		// PR to change this can be found here: https://github.com/elastic/package-registry/pull/618
 		{
 			Name:        "dataset.type",
 			Type:        "constant_keyword",

dev/import-beats/kibana.go

@@ -356,7 +356,7 @@ func stripReferencesToEventModuleInFilter(object mapStr, filterKey, moduleName s
 				return nil, errors.Wrapf(err, "setting meta.type failed")
 			}
 
-			_, err = filterObject.put("meta.value", fmt.Sprintf("{\"prefix\":{\"dataset.name\":\"%s.\"}}", moduleName))
+			_, err = filterObject.put("meta.value", fmt.Sprintf("{\"prefix\":{\"datastream.dataset\":\"%s.\"}}", moduleName))
 			if err != nil {
 				return nil, errors.Wrapf(err, "setting meta.value failed")
 			}
@@ -368,7 +368,7 @@ func stripReferencesToEventModuleInFilter(object mapStr, filterKey, moduleName s
 
 			q := map[string]interface{}{
 				"prefix": map[string]interface{}{
-					"dataset.name": moduleName + ".",
+					"datastream.dataset": moduleName + ".",
 				},
 			}
 			_, err = filterObject.put("query", q)
@@ -415,8 +415,8 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
 	query = strings.ReplaceAll(query, `"`, "")
 	if strings.Contains(query, "event.module:"+moduleName) && (strings.Contains(query, "metricset.name:") || strings.Contains(query, "fileset.name:")) {
 		query = strings.ReplaceAll(query, "event.module:"+moduleName, "")
-		query = strings.ReplaceAll(query, "metricset.name:", fmt.Sprintf("dataset.name:%s.", moduleName))
-		query = strings.ReplaceAll(query, "fileset.name:", fmt.Sprintf("dataset.name:%s.", moduleName))
+		query = strings.ReplaceAll(query, "metricset.name:", fmt.Sprintf("datastream.dataset:%s.", moduleName))
+		query = strings.ReplaceAll(query, "fileset.name:", fmt.Sprintf("datastream.dataset:%s.", moduleName))
 		query = strings.TrimSpace(query)
 		if strings.HasPrefix(query, "AND ") {
 			query = query[4:]
@@ -429,7 +429,7 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
 	} else if strings.Contains(query, "event.module:"+moduleName) {
 		var eventDatasets []string
 		for _, datasetName := range datasetNames {
-			eventDatasets = append(eventDatasets, fmt.Sprintf("dataset.name:%s.%s", moduleName, datasetName))
+			eventDatasets = append(eventDatasets, fmt.Sprintf("datastream.dataset:%s.%s", moduleName, datasetName))
 		}
 
 		value := " (" + strings.Join(eventDatasets, " OR ") + ") "
@@ -450,7 +450,7 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
 }
 
 func replaceFieldEventDatasetWithStreamDataset(data []byte) []byte {
-	return bytes.ReplaceAll(data, []byte("event.dataset"), []byte("dataset.name"))
+	return bytes.ReplaceAll(data, []byte("event.dataset"), []byte("datastream.dataset"))
 }
 
 func replaceBlacklistedWords(data []byte) []byte {

dev/import-beats/packages.go

@@ -40,9 +40,9 @@ func newPackageContent(name string) packageContent {
 				Name:    name,
 				Version: "0.0.1", // TODO
 				Type:    "integration",
+				Release: "experimental",
 			},
 			License: "basic",
-			Release: "experimental",
 			Owner: &util.Owner{
 				Github: "elastic/integrations",
 			},

go.mod

@@ -5,7 +5,7 @@ go 1.12
 require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca
-	github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e
+	github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce
 	github.com/magefile/mage v1.10.0
 	github.com/pkg/errors v0.9.1
 	gopkg.in/yaml.v2 v2.3.0

go.sum

@@ -30,8 +30,8 @@ github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca h1:ikRqi/Z
 github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca/go.mod h1:6PbJXE4kwZc49bi/ckY2IXzAFwHuR+OyYUy2iJ386os=
 github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEypR8zePr0XRbMhO4PJgcHC9f8fDbgAg=
 github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
-github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e h1:B0i7PeWOSzKCX+Xba1SSTq7jAJKZK1IMwGfMOTOO/5I=
-github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e/go.mod h1:ERTTIxAsQOCVZJDqR4LJbDDAtxV+pz4wdPPrKheiAUc=
+github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce h1:8z0Zhk4an7XsDJSvOLWQ4hxhpUMl/4o4hCZwoL5AQ3Q=
+github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -94,8 +94,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
 github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
-github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
-github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=

packages/apache/dataset/access/fields/base-fields.yml

@@ -1,3 +1,13 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
   description: Dataset type.
@@ -10,3 +20,4 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
+

packages/apache/dataset/error/fields/base-fields.yml

@@ -1,3 +1,13 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
   description: Dataset type.
@@ -10,3 +20,4 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
+

packages/apache/dataset/status/fields/base-fields.yml

@@ -1,3 +1,13 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
   description: Dataset type.
@@ -10,3 +20,4 @@
 - name: '@timestamp'
   type: date
   description: Event timestamp.
+

packages/apache/docs/README.md

@@ -24,6 +24,9 @@ Access logs collects the Apache access logs.
 | dataset.name | Dataset name. | constant_keyword |
 | dataset.namespace | Dataset namespace. | constant_keyword |
 | dataset.type | Dataset type. | constant_keyword |
+| datastream.dataset | Datastream dataset. | constant_keyword |
+| datastream.namespace | Datastream namespace. | constant_keyword |
+| datastream.type | Datastream type. | constant_keyword |
 | http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection.  Original case will be mandated in ECS 2.0.0 | keyword |
 | http.request.referrer | Referrer for this HTTP request. | keyword |
 | http.response.body.bytes | Size in bytes of the response body. | long |
@@ -63,6 +66,9 @@ Error logs collects the Apache error logs.
 | dataset.name | Dataset name. | constant_keyword |
 | dataset.namespace | Dataset namespace. | constant_keyword |
 | dataset.type | Dataset type. | constant_keyword |
+| datastream.dataset | Datastream dataset. | constant_keyword |
+| datastream.namespace | Datastream namespace. | constant_keyword |
+| datastream.type | Datastream type. | constant_keyword |
 | http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection.  Original case will be mandated in ECS 2.0.0 | keyword |
 | http.request.referrer | Referrer for this HTTP request. | keyword |
 | http.response.body.bytes | Size in bytes of the response body. | long |
@@ -235,4 +241,7 @@ An example event for `status` looks as following:
 | dataset.name | Dataset name. | constant_keyword |
 | dataset.namespace | Dataset namespace. | constant_keyword |
 | dataset.type | Dataset type. | constant_keyword |
+| datastream.dataset | Datastream dataset. | constant_keyword |
+| datastream.namespace | Datastream namespace. | constant_keyword |
+| datastream.type | Datastream type. | constant_keyword |
 

packages/apache/manifest.yml

@@ -1,7 +1,7 @@
 format_version: 1.0.0
 name: apache
 title: Apache
-version: 0.1.2
+version: 0.1.3
 license: basic
 description: Apache Integration
 type: integration

packages/aws/dataset/billing/fields/base-fields.yml

@@ -1,16 +1,23 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.
+

packages/aws/dataset/cloudtrail/fields/base-fields.yml

@@ -1,16 +1,22 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.

packages/aws/dataset/cloudwatch_logs/fields/base-fields.yml

@@ -1,16 +1,22 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.

packages/aws/dataset/cloudwatch_metrics/fields/base-fields.yml

@@ -1,16 +1,22 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.

packages/aws/dataset/dynamodb/fields/base-fields.yml

@@ -1,16 +1,22 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.

packages/aws/dataset/ebs/fields/base-fields.yml

@@ -1,16 +1,22 @@
+- name: datastream.type
+  type: constant_keyword
+  description: Datastream type.
+- name: datastream.dataset
+  type: constant_keyword
+  description: Datastream dataset.
+- name: datastream.namespace
+  type: constant_keyword
+  description: Datastream namespace.
+
 - name: dataset.type
   type: constant_keyword
-  description: >
-    Dataset type.
+  description: Dataset type.
 - name: dataset.name
   type: constant_keyword
-  description: >
-    Dataset name.
+  description: Dataset name.
 - name: dataset.namespace
   type: constant_keyword
-  description: >
-    Dataset namespace.
-- name: "@timestamp"
+  description: Dataset namespace.
+- name: '@timestamp'
   type: date
-  description: >
-    Event timestamp.
+  description: Event timestamp.