Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Do not create base-fields.yml files #494

Merged
merged 3 commits into from
Jun 4, 2020
Merged

Do not create base-fields.yml files #494

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
1bc7e53
Do not create base-fields.yml
Jun 4, 2020
e7f3e0f
Add missing base-fields.yml files
Jun 4, 2020
0120851
Add missing base-fields.yml in dev/packages
Jun 4, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 2 additions & 51 deletions dev/generator/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,9 @@ import (
"os"
"path/filepath"

"gopkg.in/yaml.v2"

"github.com/magefile/mage/sh"
"github.com/pkg/errors"
"gopkg.in/yaml.v2"

"github.com/elastic/package-registry/util"
)
Expand All @@ -26,27 +25,7 @@ var (
copy bool
)

const (
packageDirName = "package"
streamFields = `
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
`
)
const packageDirName = "package"

type fieldEntry struct {
name string
Expand Down Expand Up @@ -178,16 +157,6 @@ func buildPackage(packagesBasePath string, p util.Package) error {
return err
}

// Add base-fields.yml if missing to all dataset with the basic stream fields and @timestamp
for _, dataset := range datasets {
datasetPath := filepath.Join(p.BasePath, "dataset", dataset)

err = createBaseFieldsFile(datasetPath)
if err != nil {
return errors.Wrapf(err, "creating file with basic fields failed (datasetPath: %s)", datasetPath)
}
}

// Validate if basic stream fields and @timestamp are present
for _, dataset := range datasets {
datasetPath := filepath.Join(p.BasePath, "dataset", dataset)
Expand Down Expand Up @@ -239,24 +208,6 @@ func buildPackage(packagesBasePath string, p util.Package) error {
return nil
}

// createBaseFieldsFile method creates the base-fields.yml file if missing.
func createBaseFieldsFile(datasetPath string) error {
fieldsDirPath := filepath.Join(datasetPath, "fields")
baseFieldsPath := filepath.Join(fieldsDirPath, "base-fields.yml")
_, err := os.Stat(baseFieldsPath)
if err == nil {
return nil
} else if !os.IsNotExist(err) {
return errors.Wrapf(err, "stat file failed (fieldsPath: %s)", baseFieldsPath)
}

err = os.MkdirAll(fieldsDirPath, 0755)
if err != nil {
return errors.Wrapf(err, "creating directories failed (fieldsDirPath: %s)", fieldsDirPath)
}
return ioutil.WriteFile(baseFieldsPath, []byte(streamFields), 0644)
}

// validateRequiredFields method loads fields from all files and checks if required fields are present.
func validateRequiredFields(datasetPath string) error {
fieldsDirPath := filepath.Join(datasetPath, "fields")
Expand Down
16 changes: 16 additions & 0 deletions testdata/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/example/1.0.0/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
16 changes: 16 additions & 0 deletions testdata/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
- name: stream.type
type: constant_keyword
description: >
Stream type
- name: stream.dataset
type: constant_keyword
description: >
Stream dataset.
- name: stream.namespace
type: constant_keyword
description: >
Stream namespace.
- name: "@timestamp"
type: date
description: >
Event timestamp.
1 change: 0 additions & 1 deletion testdata/public/package/datasources/1.0.0/dataset/examplelog1/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/datasources/1.0.0/dataset/examplelog2/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/datasources/1.0.0/dataset/examplemetric/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/default-pipeline/0.0.2/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/example/1.0.0/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/multiple-false/0.0.1/dataset/foo/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down
1 change: 0 additions & 1 deletion testdata/public/package/no_stream_configs/1.0.0/dataset/log/fields/base-fields.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@

- name: stream.type
type: constant_keyword
description: >
Expand Down