Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add datastream fields to all datasets #213

Merged
merged 9 commits into from
Aug 4, 2020
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions dev/import-beats/fields_base_fields.go
Original file line number Diff line number Diff line change
@@ -8,6 +8,23 @@ var baseFields = createBaseFields()

func createBaseFields() []fieldDefinition {
return []fieldDefinition{
{
Name: "datastream.type",
Type: "constant_keyword",
Description: "Datastream type.",
},
{
Name: "datastream.dataset",
Type: "constant_keyword",
Description: "Datastream dataset name.",
},
{
Name: "datastream.namespace",
Type: "constant_keyword",
Description: "Datastream namespace.",
},
// TODO: This should be removed as soon as it is not a requirement anymore by the validation
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hmm.. shouldn't be validation in the package-registry already adjusted?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If we adjust it now, all old packages are invalid.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

PR for validation: elastic/package-registry#618

once this one is pushed, you can update the reference to the commit

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You can update the reference in docs.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

done

// PR to change this can be found here: https://github.com/elastic/package-registry/pull/618
{
Name: "dataset.type",
Type: "constant_keyword",
12 changes: 6 additions & 6 deletions dev/import-beats/kibana.go
Original file line number Diff line number Diff line change
@@ -356,7 +356,7 @@ func stripReferencesToEventModuleInFilter(object mapStr, filterKey, moduleName s
return nil, errors.Wrapf(err, "setting meta.type failed")
}

_, err = filterObject.put("meta.value", fmt.Sprintf("{\"prefix\":{\"dataset.name\":\"%s.\"}}", moduleName))
_, err = filterObject.put("meta.value", fmt.Sprintf("{\"prefix\":{\"datastream.dataset\":\"%s.\"}}", moduleName))
if err != nil {
return nil, errors.Wrapf(err, "setting meta.value failed")
}
@@ -368,7 +368,7 @@ func stripReferencesToEventModuleInFilter(object mapStr, filterKey, moduleName s

q := map[string]interface{}{
"prefix": map[string]interface{}{
"dataset.name": moduleName + ".",
"datastream.dataset": moduleName + ".",
},
}
_, err = filterObject.put("query", q)
@@ -415,8 +415,8 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
query = strings.ReplaceAll(query, `"`, "")
if strings.Contains(query, "event.module:"+moduleName) && (strings.Contains(query, "metricset.name:") || strings.Contains(query, "fileset.name:")) {
query = strings.ReplaceAll(query, "event.module:"+moduleName, "")
query = strings.ReplaceAll(query, "metricset.name:", fmt.Sprintf("dataset.name:%s.", moduleName))
query = strings.ReplaceAll(query, "fileset.name:", fmt.Sprintf("dataset.name:%s.", moduleName))
query = strings.ReplaceAll(query, "metricset.name:", fmt.Sprintf("datastream.dataset:%s.", moduleName))
query = strings.ReplaceAll(query, "fileset.name:", fmt.Sprintf("datastream.dataset:%s.", moduleName))
query = strings.TrimSpace(query)
if strings.HasPrefix(query, "AND ") {
query = query[4:]
@@ -429,7 +429,7 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
} else if strings.Contains(query, "event.module:"+moduleName) {
var eventDatasets []string
for _, datasetName := range datasetNames {
eventDatasets = append(eventDatasets, fmt.Sprintf("dataset.name:%s.%s", moduleName, datasetName))
eventDatasets = append(eventDatasets, fmt.Sprintf("datastream.dataset:%s.%s", moduleName, datasetName))
}

value := " (" + strings.Join(eventDatasets, " OR ") + ") "
@@ -450,7 +450,7 @@ func stripReferencesToEventModuleInQuery(object mapStr, objectKey, moduleName st
}

func replaceFieldEventDatasetWithStreamDataset(data []byte) []byte {
return bytes.ReplaceAll(data, []byte("event.dataset"), []byte("dataset.name"))
return bytes.ReplaceAll(data, []byte("event.dataset"), []byte("datastream.dataset"))
}

func replaceBlacklistedWords(data []byte) []byte {
2 changes: 1 addition & 1 deletion dev/import-beats/packages.go
Original file line number Diff line number Diff line change
@@ -40,9 +40,9 @@ func newPackageContent(name string) packageContent {
Name: name,
Version: "0.0.1", // TODO
Type: "integration",
Release: "experimental",
},
License: "basic",
Release: "experimental",
Owner: &util.Owner{
Github: "elastic/integrations",
},
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ go 1.12
require (
github.com/blang/semver v3.5.1+incompatible
github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca
github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e
github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce
github.com/magefile/mage v1.10.0
github.com/pkg/errors v0.9.1
gopkg.in/yaml.v2 v2.3.0
6 changes: 2 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
@@ -30,8 +30,8 @@ github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca h1:ikRqi/Z
github.com/elastic/elastic-package v0.0.0-20200731114746-b0437f8f05ca/go.mod h1:6PbJXE4kwZc49bi/ckY2IXzAFwHuR+OyYUy2iJ386os=
github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6 h1:Ehbr7du4rSSEypR8zePr0XRbMhO4PJgcHC9f8fDbgAg=
github.com/elastic/go-ucfg v0.8.4-0.20200415140258-1232bd4774a6/go.mod h1:iaiY0NBIYeasNgycLyTvhJftQlQEUO2hpF+FX0JKxzo=
github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e h1:B0i7PeWOSzKCX+Xba1SSTq7jAJKZK1IMwGfMOTOO/5I=
github.com/elastic/package-registry v0.4.1-0.20200702132954-41c150c8020e/go.mod h1:ERTTIxAsQOCVZJDqR4LJbDDAtxV+pz4wdPPrKheiAUc=
github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce h1:8z0Zhk4an7XsDJSvOLWQ4hxhpUMl/4o4hCZwoL5AQ3Q=
github.com/elastic/package-registry v0.8.1-0.20200804105354-737fb54752ce/go.mod h1:oQx3Tg9ynuC6APd0o0OHud9kyPX6S6IzdJp/R4Hj1HY=
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
@@ -94,8 +94,6 @@ github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y8
github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/radovskyb/watcher v1.0.7 h1:AYePLih6dpmS32vlHfhCeli8127LzkIgwJGcwwe8tUE=
github.com/radovskyb/watcher v1.0.7/go.mod h1:78okwvY5wPdzcb1UYnip1pvrZNIVEIh/Cm+ZuvsUYIg=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
11 changes: 11 additions & 0 deletions packages/apache/dataset/access/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
- name: datastream.type
mtojek marked this conversation as resolved.
Show resolved Hide resolved
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: Dataset type.
@@ -10,3 +20,4 @@
- name: '@timestamp'
type: date
description: Event timestamp.

11 changes: 11 additions & 0 deletions packages/apache/dataset/error/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: Dataset type.
@@ -10,3 +20,4 @@
- name: '@timestamp'
type: date
description: Event timestamp.

11 changes: 11 additions & 0 deletions packages/apache/dataset/status/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,3 +1,13 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: Dataset type.
@@ -10,3 +20,4 @@
- name: '@timestamp'
type: date
description: Event timestamp.

9 changes: 9 additions & 0 deletions packages/apache/docs/README.md
Original file line number Diff line number Diff line change
@@ -24,6 +24,9 @@ Access logs collects the Apache access logs.
| dataset.name | Dataset name. | constant_keyword |
| dataset.namespace | Dataset namespace. | constant_keyword |
| dataset.type | Dataset type. | constant_keyword |
| datastream.dataset | Datastream dataset. | constant_keyword |
| datastream.namespace | Datastream namespace. | constant_keyword |
| datastream.type | Datastream type. | constant_keyword |
| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword |
| http.request.referrer | Referrer for this HTTP request. | keyword |
| http.response.body.bytes | Size in bytes of the response body. | long |
@@ -63,6 +66,9 @@ Error logs collects the Apache error logs.
| dataset.name | Dataset name. | constant_keyword |
| dataset.namespace | Dataset namespace. | constant_keyword |
| dataset.type | Dataset type. | constant_keyword |
| datastream.dataset | Datastream dataset. | constant_keyword |
| datastream.namespace | Datastream namespace. | constant_keyword |
| datastream.type | Datastream type. | constant_keyword |
| http.request.method | HTTP request method. Prior to ECS 1.6.0 the following guidance was provided: "The field value must be normalized to lowercase for querying." As of ECS 1.6.0, the guidance is deprecated because the original case of the method may be useful in anomaly detection. Original case will be mandated in ECS 2.0.0 | keyword |
| http.request.referrer | Referrer for this HTTP request. | keyword |
| http.response.body.bytes | Size in bytes of the response body. | long |
@@ -235,4 +241,7 @@ An example event for `status` looks as following:
| dataset.name | Dataset name. | constant_keyword |
| dataset.namespace | Dataset namespace. | constant_keyword |
| dataset.type | Dataset type. | constant_keyword |
| datastream.dataset | Datastream dataset. | constant_keyword |
| datastream.namespace | Datastream namespace. | constant_keyword |
| datastream.type | Datastream type. | constant_keyword |

2 changes: 1 addition & 1 deletion packages/apache/manifest.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
format_version: 1.0.0
name: apache
title: Apache
version: 0.1.2
version: 0.1.3
license: basic
description: Apache Integration
type: integration
25 changes: 16 additions & 9 deletions packages/aws/dataset/billing/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,23 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.

24 changes: 15 additions & 9 deletions packages/aws/dataset/cloudtrail/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.
24 changes: 15 additions & 9 deletions packages/aws/dataset/cloudwatch_logs/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.
24 changes: 15 additions & 9 deletions packages/aws/dataset/cloudwatch_metrics/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.
24 changes: 15 additions & 9 deletions packages/aws/dataset/dynamodb/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.
24 changes: 15 additions & 9 deletions packages/aws/dataset/ebs/fields/base-fields.yml
Original file line number Diff line number Diff line change
@@ -1,16 +1,22 @@
- name: datastream.type
type: constant_keyword
description: Datastream type.
- name: datastream.dataset
type: constant_keyword
description: Datastream dataset.
- name: datastream.namespace
type: constant_keyword
description: Datastream namespace.

- name: dataset.type
type: constant_keyword
description: >
Dataset type.
description: Dataset type.
- name: dataset.name
type: constant_keyword
description: >
Dataset name.
description: Dataset name.
- name: dataset.namespace
type: constant_keyword
description: >
Dataset namespace.
- name: "@timestamp"
description: Dataset namespace.
- name: '@timestamp'
type: date
description: >
Event timestamp.
description: Event timestamp.
Loading