Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Migrate json parser to new style #11226

Merged
merged 5 commits into from
Jun 22, 2022
Merged

feat: Migrate json parser to new style #11226

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
beab40a
Migrate json parser to new parser-style.
srebhan May 31, 2022
e11b5b3
Remove TOML masks for json parser.
srebhan May 31, 2022
539e8ca
Fix json parser usage in httpjson.
srebhan May 31, 2022
ae6f5a1
Fix json parser usage in test cases.
srebhan May 31, 2022
1053c9b
Move json_flattener into own file.
srebhan May 31, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions config/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1835,8 +1835,7 @@ func (c *Config) missingTomlField(_ reflect.Type, key string) error {
"grace", "graphite_separator", "graphite_tag_sanitize_mode", "graphite_tag_support",
"grok_custom_pattern_files", "grok_custom_patterns", "grok_named_patterns", "grok_patterns",
"grok_timezone", "grok_unique_timestamp", "influx_max_line_bytes", "influx_parser_type", "influx_sort_fields",
"influx_uint_support", "interval", "json_name_key", "json_query", "json_strict",
"json_string_fields", "json_time_format", "json_time_key", "json_timestamp_format", "json_timestamp_units", "json_timezone", "json_v2",
"influx_uint_support", "interval", "json_timestamp_units", "json_v2",
"lvm", "metric_batch_size", "metric_buffer_limit", "name_override", "name_prefix",
"name_suffix", "namedrop", "namepass", "order", "pass", "period", "precision",
"prefix", "prometheus_export_timestamp", "prometheus_ignore_timestamp", "prometheus_sort_metrics", "prometheus_string_as_label",
Expand Down
19 changes: 7 additions & 12 deletions plugins/inputs/directory_monitor/directory_monitor_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/plugins/parsers/csv"
"github.com/influxdata/telegraf/plugins/parsers/json"
"github.com/influxdata/telegraf/testutil"
)

Expand Down Expand Up @@ -116,13 +117,10 @@ func TestMultipleJSONFileImports(t *testing.T) {
err := r.Init()
require.NoError(t, err)

parserConfig := parsers.Config{
DataFormat: "json",
JSONNameKey: "Name",
}

r.SetParserFunc(func() (parsers.Parser, error) {
return parsers.NewParser(&parserConfig)
p := &json.Parser{NameKey: "Name"}
err := p.Init()
return p, err
})

// Let's drop a 5-line LINE-DELIMITED json.
Expand Down Expand Up @@ -166,13 +164,10 @@ func TestFileTag(t *testing.T) {
err := r.Init()
require.NoError(t, err)

parserConfig := parsers.Config{
DataFormat: "json",
JSONNameKey: "Name",
}

r.SetParserFunc(func() (parsers.Parser, error) {
return parsers.NewParser(&parserConfig)
p := &json.Parser{NameKey: "Name"}
err := p.Init()
return p, err
})

// Let's drop a 1-line LINE-DELIMITED json.
Expand Down
44 changes: 20 additions & 24 deletions plugins/inputs/exec/exec_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (
"github.com/stretchr/testify/require"

"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/plugins/parsers/json"
"github.com/influxdata/telegraf/testutil"
)

Expand Down Expand Up @@ -80,10 +81,8 @@ func (r runnerMock) Run(_ string, _ []string, _ time.Duration) ([]byte, []byte,
}

func TestExec(t *testing.T) {
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
parser := &json.Parser{MetricName: "exec"}
require.NoError(t, parser.Init())
e := &Exec{
Log: testutil.Logger{},
runner: newRunnerMock([]byte(validJSON), nil, nil),
Expand All @@ -110,10 +109,8 @@ func TestExec(t *testing.T) {
}

func TestExecMalformed(t *testing.T) {
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
parser := &json.Parser{MetricName: "exec"}
require.NoError(t, parser.Init())
e := &Exec{
Log: testutil.Logger{},
runner: newRunnerMock([]byte(malformedJSON), nil, nil),
Expand All @@ -127,10 +124,8 @@ func TestExecMalformed(t *testing.T) {
}

func TestCommandError(t *testing.T) {
parser, _ := parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "exec",
})
parser := &json.Parser{MetricName: "exec"}
require.NoError(t, parser.Init())
e := &Exec{
Log: testutil.Logger{},
runner: newRunnerMock(nil, nil, fmt.Errorf("exit status code 1")),
Expand All @@ -144,14 +139,14 @@ func TestCommandError(t *testing.T) {
}

func TestExecCommandWithGlob(t *testing.T) {
parser, _ := parsers.NewValueParser("metric", "string", "", nil)
parser, err := parsers.NewValueParser("metric", "string", "", nil)
require.NoError(t, err)
e := NewExec()
e.Commands = []string{"/bin/ech* metric_value"}
e.SetParser(parser)

var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.NoError(t, err)
require.NoError(t, acc.GatherError(e.Gather))

fields := map[string]interface{}{
"value": "metric_value",
Expand All @@ -160,14 +155,15 @@ func TestExecCommandWithGlob(t *testing.T) {
}

func TestExecCommandWithoutGlob(t *testing.T) {
parser, _ := parsers.NewValueParser("metric", "string", "", nil)
parser, err := parsers.NewValueParser("metric", "string", "", nil)
require.NoError(t, err)

e := NewExec()
e.Commands = []string{"/bin/echo metric_value"}
e.SetParser(parser)

var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.NoError(t, err)
require.NoError(t, acc.GatherError(e.Gather))

fields := map[string]interface{}{
"value": "metric_value",
Expand All @@ -176,14 +172,14 @@ func TestExecCommandWithoutGlob(t *testing.T) {
}

func TestExecCommandWithoutGlobAndPath(t *testing.T) {
parser, _ := parsers.NewValueParser("metric", "string", "", nil)
parser, err := parsers.NewValueParser("metric", "string", "", nil)
require.NoError(t, err)
e := NewExec()
e.Commands = []string{"echo metric_value"}
e.SetParser(parser)

var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.NoError(t, err)
require.NoError(t, acc.GatherError(e.Gather))

fields := map[string]interface{}{
"value": "metric_value",
Expand All @@ -192,15 +188,15 @@ func TestExecCommandWithoutGlobAndPath(t *testing.T) {
}

func TestExecCommandWithEnv(t *testing.T) {
parser, _ := parsers.NewValueParser("metric", "string", "", nil)
parser, err := parsers.NewValueParser("metric", "string", "", nil)
require.NoError(t, err)
e := NewExec()
e.Commands = []string{"/bin/sh -c 'echo ${METRIC_NAME}'"}
e.Environment = []string{"METRIC_NAME=metric_value"}
e.SetParser(parser)

var acc testutil.Accumulator
err := acc.GatherError(e.Gather)
require.NoError(t, err)
require.NoError(t, acc.GatherError(e.Gather))

fields := map[string]interface{}{
"value": "metric_value",
Expand Down
30 changes: 15 additions & 15 deletions plugins/inputs/file/file_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ import (
"github.com/influxdata/telegraf"
"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/plugins/parsers/csv"
"github.com/influxdata/telegraf/plugins/parsers/json"
"github.com/influxdata/telegraf/testutil"
)

Expand All @@ -43,16 +44,15 @@ func TestFileTag(t *testing.T) {
Files: []string{filepath.Join(wd, "dev/testfiles/json_a.log")},
FileTag: "filename",
}
err = r.Init()
require.NoError(t, err)
require.NoError(t, r.Init())

parserConfig := parsers.Config{
DataFormat: "json",
}
r.SetParserFunc(func() (telegraf.Parser, error) { return parsers.NewParser(&parserConfig) })
r.SetParserFunc(func() (telegraf.Parser, error) {
p := &json.Parser{}
err := p.Init()
return p, err
})

err = r.Gather(&acc)
require.NoError(t, err)
require.NoError(t, r.Gather(&acc))

for _, m := range acc.Metrics {
for key, value := range m.Tags {
Expand All @@ -68,13 +68,13 @@ func TestJSONParserCompile(t *testing.T) {
r := File{
Files: []string{filepath.Join(wd, "dev/testfiles/json_a.log")},
}
err := r.Init()
require.NoError(t, err)
parserConfig := parsers.Config{
DataFormat: "json",
TagKeys: []string{"parent_ignored_child"},
}
r.SetParserFunc(func() (telegraf.Parser, error) { return parsers.NewParser(&parserConfig) })
require.NoError(t, r.Init())

r.SetParserFunc(func() (telegraf.Parser, error) {
p := &json.Parser{TagKeys: []string{"parent_ignored_child"}}
err := p.Init()
return p, err
})

require.NoError(t, r.Gather(&acc))
require.Equal(t, map[string]string{"parent_ignored_child": "hi"}, acc.Metrics[0].Tags)
Expand Down
43 changes: 19 additions & 24 deletions plugins/inputs/http/http_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
httpplugin "github.com/influxdata/telegraf/plugins/inputs/http"
"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/plugins/parsers/csv"
"github.com/influxdata/telegraf/plugins/parsers/json"
"github.com/influxdata/telegraf/testutil"
)

Expand All @@ -39,10 +40,9 @@ func TestHTTPWithJSONFormat(t *testing.T) {
metricName := "metricName"

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand Down Expand Up @@ -83,10 +83,9 @@ func TestHTTPHeaders(t *testing.T) {
}

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand Down Expand Up @@ -117,10 +116,9 @@ func TestHTTPContentLengthHeader(t *testing.T) {
}

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand All @@ -141,10 +139,9 @@ func TestInvalidStatusCode(t *testing.T) {
}

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand All @@ -166,10 +163,9 @@ func TestSuccessStatusCodes(t *testing.T) {
}

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand All @@ -194,10 +190,9 @@ func TestMethod(t *testing.T) {
}

plugin.SetParserFunc(func() (telegraf.Parser, error) {
return parsers.NewParser(&parsers.Config{
DataFormat: "json",
MetricName: "metricName",
})
p := &json.Parser{MetricName: "metricName"}
err := p.Init()
return p, err
})

var acc testutil.Accumulator
Expand Down
9 changes: 4 additions & 5 deletions plugins/inputs/httpjson/httpjson.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import (
"github.com/influxdata/telegraf/config"
"github.com/influxdata/telegraf/plugins/common/tls"
"github.com/influxdata/telegraf/plugins/inputs"
"github.com/influxdata/telegraf/plugins/parsers"
"github.com/influxdata/telegraf/plugins/parsers/json"
)

// DO NOT REMOVE THE NEXT TWO LINES! This is required to embed the sampleConfig data.
Expand Down Expand Up @@ -136,13 +136,12 @@ func (h *HTTPJSON) gatherServer(
"server": serverURL,
}

parser, err := parsers.NewParser(&parsers.Config{
DataFormat: "json",
parser := &json.Parser{
MetricName: msrmntName,
TagKeys: h.TagKeys,
DefaultTags: tags,
})
if err != nil {
}
if err := parser.Init(); err != nil {
return err
}

Expand Down
Loading