From b33d43acfe5336315d1e6cd41657cc9aea8589f8 Mon Sep 17 00:00:00 2001 From: Cameron Sparr Date: Fri, 8 Apr 2016 16:04:45 -0600 Subject: [PATCH] Create a template system for the graphite serializer closes #925 closes #879 --- internal/config/config.go | 9 ++ plugins/outputs/graphite/graphite.go | 14 ++- plugins/outputs/librato/librato.go | 11 +-- plugins/serializers/graphite/graphite.go | 96 ++++++++++++------- plugins/serializers/graphite/graphite_test.go | 20 ++-- plugins/serializers/registry.go | 11 ++- 6 files changed, 100 insertions(+), 61 deletions(-) diff --git a/internal/config/config.go b/internal/config/config.go index 1e07234e83121..cfd6c959349f4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -850,8 +850,17 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error } } + if node, ok := tbl.Fields["template"]; ok { + if kv, ok := node.(*ast.KeyValue); ok { + if str, ok := kv.Value.(*ast.String); ok { + c.Template = str.Value + } + } + } + delete(tbl.Fields, "data_format") delete(tbl.Fields, "prefix") + delete(tbl.Fields, "template") return serializers.NewSerializer(c) } diff --git a/plugins/outputs/graphite/graphite.go b/plugins/outputs/graphite/graphite.go index 717ce06c80b52..926d7e5a033a3 100644 --- a/plugins/outputs/graphite/graphite.go +++ b/plugins/outputs/graphite/graphite.go @@ -16,10 +16,11 @@ import ( type Graphite struct { // URL is only for backwards compatability - Servers []string - Prefix string - Timeout int - conns []net.Conn + Servers []string + Prefix string + Template string + Timeout int + conns []net.Conn } var sampleConfig = ` @@ -27,6 +28,9 @@ var sampleConfig = ` servers = ["localhost:2003"] ## Prefix metrics name prefix = "" + ## Graphite template + ## see https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md + template = "host.tags.measurement.field" ## timeout in seconds for the write connection to graphite timeout = 2 ` @@ -72,7 +76,7 @@ func (g *Graphite) Description() string { func (g *Graphite) Write(metrics []telegraf.Metric) error { // Prepare data var bp []string - s, err := serializers.NewGraphiteSerializer(g.Prefix) + s, err := serializers.NewGraphiteSerializer(g.Prefix, g.Template) if err != nil { return err } diff --git a/plugins/outputs/librato/librato.go b/plugins/outputs/librato/librato.go index f0f03400e587f..690140c55c445 100644 --- a/plugins/outputs/librato/librato.go +++ b/plugins/outputs/librato/librato.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "log" "net/http" + "strings" "github.com/influxdata/telegraf" "github.com/influxdata/telegraf/internal" @@ -152,17 +153,13 @@ func (l *Librato) Description() string { return "Configuration for Librato API to send metrics to." } -func (l *Librato) buildGaugeName(m telegraf.Metric, fieldName string) string { - // Use the GraphiteSerializer - graphiteSerializer := graphite.GraphiteSerializer{} - return graphiteSerializer.SerializeBucketName(m, fieldName) -} - func (l *Librato) buildGauges(m telegraf.Metric) ([]*Gauge, error) { gauges := []*Gauge{} + graphiteSerializer := graphite.GraphiteSerializer{} + bucket := graphiteSerializer.SerializeBucketName(m.Name(), m.Tags()) for fieldName, value := range m.Fields() { gauge := &Gauge{ - Name: l.buildGaugeName(m, fieldName), + Name: strings.Replace(bucket, "FIELDNAME", fieldName, 1), MeasureTime: m.Time().Unix(), } if !gauge.verifyValue(value) { diff --git a/plugins/serializers/graphite/graphite.go b/plugins/serializers/graphite/graphite.go index 7a7fec2f1d374..539c1f2ab6a75 100644 --- a/plugins/serializers/graphite/graphite.go +++ b/plugins/serializers/graphite/graphite.go @@ -8,11 +8,14 @@ import ( "github.com/influxdata/telegraf" ) +const DEFAULT_TEMPLATE = "host.tags.measurement.field" + type GraphiteSerializer struct { - Prefix string + Prefix string + Template string } -var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_") +var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_", "..", ".") func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) { out := []string{} @@ -20,13 +23,16 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) // Convert UnixNano to Unix timestamps timestamp := metric.UnixNano() / 1000000000 + bucket := s.SerializeBucketName(metric.Name(), metric.Tags()) + for field_name, value := range metric.Fields() { // Convert value value_str := fmt.Sprintf("%#v", value) // Write graphite metric var graphitePoint string graphitePoint = fmt.Sprintf("%s %s %d", - s.SerializeBucketName(metric, field_name), + // insert "field" section of template + strings.Replace(bucket, "FIELDNAME", field_name, 1), value_str, timestamp) out = append(out, graphitePoint) @@ -34,51 +40,69 @@ func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) return out, nil } -func (s *GraphiteSerializer) SerializeBucketName(metric telegraf.Metric, field_name string) string { - // Get the metric name - name := metric.Name() +// SerializeBucketName will take the given measurement name and tags and +// produce a graphite bucket. It will use the GraphiteSerializer.Template +// to generate this, or DEFAULT_TEMPLATE. +// +// NOTE: SerializeBucketName replaces the "field" portion of the template with +// FIELDNAME. It is up to the user to replace this. This is so that +// SerializeBucketName can be called just once per measurement, rather than +// once per field. +func (s *GraphiteSerializer) SerializeBucketName( + measurement string, + tags map[string]string, +) string { + if s.Template == "" { + s.Template = DEFAULT_TEMPLATE + } + tagsCopy := make(map[string]string) + for k, v := range tags { + tagsCopy[k] = v + } - // Convert UnixNano to Unix timestamps - tag_str := buildTags(metric) + var out []string + templateParts := strings.Split(s.Template, ".") + for _, templatePart := range templateParts { + switch templatePart { + case "measurement": + out = append(out, measurement) + case "tags": + // we will replace this later + out = append(out, "TAGS") + case "field": + // user of SerializeBucketName needs to replace this + out = append(out, "FIELDNAME") + default: + // This is a tag being applied + if tagvalue, ok := tagsCopy[templatePart]; ok { + out = append(out, strings.Replace(tagvalue, ".", "_", -1)) + delete(tagsCopy, templatePart) + } + } + } - // Write graphite metric - var serializedBucketName string - if name == field_name { - serializedBucketName = fmt.Sprintf("%s.%s", - tag_str, - strings.Replace(name, ".", "_", -1)) - } else { - serializedBucketName = fmt.Sprintf("%s.%s.%s", - tag_str, - strings.Replace(name, ".", "_", -1), - strings.Replace(field_name, ".", "_", -1)) + // insert remaining tags into output name + for i, templatePart := range out { + if templatePart == "TAGS" { + out[i] = buildTags(tagsCopy) + break + } } - if s.Prefix != "" { - serializedBucketName = fmt.Sprintf("%s.%s", s.Prefix, serializedBucketName) + + if s.Prefix == "" { + return sanitizedChars.Replace(strings.Join(out, ".")) } - return serializedBucketName + return sanitizedChars.Replace(s.Prefix + "." + strings.Join(out, ".")) } -func buildTags(metric telegraf.Metric) string { +func buildTags(tags map[string]string) string { var keys []string - tags := metric.Tags() for k := range tags { - if k == "host" { - continue - } keys = append(keys, k) } sort.Strings(keys) var tag_str string - if host, ok := tags["host"]; ok { - if len(keys) > 0 { - tag_str = strings.Replace(host, ".", "_", -1) + "." - } else { - tag_str = strings.Replace(host, ".", "_", -1) - } - } - for i, k := range keys { tag_value := strings.Replace(tags[k], ".", "_", -1) if i == 0 { @@ -87,5 +111,5 @@ func buildTags(metric telegraf.Metric) string { tag_str += "." + tag_value } } - return sanitizedChars.Replace(tag_str) + return tag_str } diff --git a/plugins/serializers/graphite/graphite_test.go b/plugins/serializers/graphite/graphite_test.go index 8d25bf937709a..04a1b63adb0d8 100644 --- a/plugins/serializers/graphite/graphite_test.go +++ b/plugins/serializers/graphite/graphite_test.go @@ -31,12 +31,12 @@ func TestGraphiteTags(t *testing.T) { time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC), ) - tags1 := buildTags(m1) - tags2 := buildTags(m2) - tags3 := buildTags(m3) + tags1 := buildTags(m1.Tags()) + tags2 := buildTags(m2.Tags()) + tags3 := buildTags(m3.Tags()) assert.Equal(t, "192_168_0_1", tags1) - assert.Equal(t, "192_168_0_1.first.second", tags2) + assert.Equal(t, "first.second.192_168_0_1", tags2) assert.Equal(t, "first.second", tags3) } @@ -133,9 +133,9 @@ func TestSerializeBucketNameNoHost(t *testing.T) { assert.NoError(t, err) s := GraphiteSerializer{} - mS := s.SerializeBucketName(m, "usage_idle") + mS := s.SerializeBucketName(m.Name(), m.Tags()) - expS := fmt.Sprintf("cpu0.us-west-2.cpu.usage_idle") + expS := fmt.Sprintf("cpu0.us-west-2.cpu.FIELDNAME") assert.Equal(t, expS, mS) } @@ -153,9 +153,9 @@ func TestSerializeBucketNameHost(t *testing.T) { assert.NoError(t, err) s := GraphiteSerializer{} - mS := s.SerializeBucketName(m, "usage_idle") + mS := s.SerializeBucketName(m.Name(), m.Tags()) - expS := fmt.Sprintf("localhost.cpu0.us-west-2.cpu.usage_idle") + expS := fmt.Sprintf("localhost.cpu0.us-west-2.cpu.FIELDNAME") assert.Equal(t, expS, mS) } @@ -173,8 +173,8 @@ func TestSerializeBucketNamePrefix(t *testing.T) { assert.NoError(t, err) s := GraphiteSerializer{Prefix: "prefix"} - mS := s.SerializeBucketName(m, "usage_idle") + mS := s.SerializeBucketName(m.Name(), m.Tags()) - expS := fmt.Sprintf("prefix.localhost.cpu0.us-west-2.cpu.usage_idle") + expS := fmt.Sprintf("prefix.localhost.cpu0.us-west-2.cpu.FIELDNAME") assert.Equal(t, expS, mS) } diff --git a/plugins/serializers/registry.go b/plugins/serializers/registry.go index ebf79bc59bbb1..0cf8149e38ed2 100644 --- a/plugins/serializers/registry.go +++ b/plugins/serializers/registry.go @@ -30,6 +30,10 @@ type Config struct { // Prefix to add to all measurements, only supports Graphite Prefix string + + // Template for converting telegraf metrics into Graphite + // only supports Graphite + Template string } // NewSerializer a Serializer interface based on the given config. @@ -40,7 +44,7 @@ func NewSerializer(config *Config) (Serializer, error) { case "influx": serializer, err = NewInfluxSerializer() case "graphite": - serializer, err = NewGraphiteSerializer(config.Prefix) + serializer, err = NewGraphiteSerializer(config.Prefix, config.Template) case "json": serializer, err = NewJsonSerializer() } @@ -55,8 +59,9 @@ func NewInfluxSerializer() (Serializer, error) { return &influx.InfluxSerializer{}, nil } -func NewGraphiteSerializer(prefix string) (Serializer, error) { +func NewGraphiteSerializer(prefix, template string) (Serializer, error) { return &graphite.GraphiteSerializer{ - Prefix: prefix, + Prefix: prefix, + Template: template, }, nil }