Create a template system for the graphite serializer

closes #925
closes #879

internal/config/config.go

@@ -850,8 +850,17 @@ func buildSerializer(name string, tbl *ast.Table) (serializers.Serializer, error
 		}
 	}
 
+	if node, ok := tbl.Fields["template"]; ok {
+		if kv, ok := node.(*ast.KeyValue); ok {
+			if str, ok := kv.Value.(*ast.String); ok {
+				c.Template = str.Value
+			}
+		}
+	}
+
 	delete(tbl.Fields, "data_format")
 	delete(tbl.Fields, "prefix")
+	delete(tbl.Fields, "template")
 	return serializers.NewSerializer(c)
 }
 

plugins/outputs/graphite/graphite.go

@@ -16,17 +16,21 @@ import (
 
 type Graphite struct {
 	// URL is only for backwards compatability
-	Servers []string
-	Prefix  string
-	Timeout int
-	conns   []net.Conn
+	Servers  []string
+	Prefix   string
+	Template string
+	Timeout  int
+	conns    []net.Conn
 }
 
 var sampleConfig = `
   ## TCP endpoint for your graphite instance.
   servers = ["localhost:2003"]
   ## Prefix metrics name
   prefix = ""
+  ## Graphite template
+  ## see https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_OUTPUT.md
+  template = "host.tags.measurement.field"
   ## timeout in seconds for the write connection to graphite
   timeout = 2
 `
@@ -72,7 +76,7 @@ func (g *Graphite) Description() string {
 func (g *Graphite) Write(metrics []telegraf.Metric) error {
 	// Prepare data
 	var bp []string
-	s, err := serializers.NewGraphiteSerializer(g.Prefix)
+	s, err := serializers.NewGraphiteSerializer(g.Prefix, g.Template)
 	if err != nil {
 		return err
 	}

plugins/outputs/librato/librato.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"log"
 	"net/http"
+	"strings"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"
@@ -152,17 +153,13 @@ func (l *Librato) Description() string {
 	return "Configuration for Librato API to send metrics to."
 }
 
-func (l *Librato) buildGaugeName(m telegraf.Metric, fieldName string) string {
-	// Use the GraphiteSerializer
-	graphiteSerializer := graphite.GraphiteSerializer{}
-	return graphiteSerializer.SerializeBucketName(m, fieldName)
-}
-
 func (l *Librato) buildGauges(m telegraf.Metric) ([]*Gauge, error) {
 	gauges := []*Gauge{}
+	graphiteSerializer := graphite.GraphiteSerializer{}
+	bucket := graphiteSerializer.SerializeBucketName(m.Name(), m.Tags())
 	for fieldName, value := range m.Fields() {
 		gauge := &Gauge{
-			Name:        l.buildGaugeName(m, fieldName),
+			Name:        strings.Replace(bucket, "FIELDNAME", fieldName, 1),
 			MeasureTime: m.Time().Unix(),
 		}
 		if !gauge.verifyValue(value) {

plugins/serializers/graphite/graphite.go

@@ -8,77 +8,101 @@ import (
 	"github.com/influxdata/telegraf"
 )
 
+const DEFAULT_TEMPLATE = "host.tags.measurement.field"
+
 type GraphiteSerializer struct {
-	Prefix string
+	Prefix   string
+	Template string
 }
 
-var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_")
+var sanitizedChars = strings.NewReplacer("/", "-", "@", "-", " ", "_", "..", ".")
 
 func (s *GraphiteSerializer) Serialize(metric telegraf.Metric) ([]string, error) {
 	out := []string{}
 
 	// Convert UnixNano to Unix timestamps
 	timestamp := metric.UnixNano() / 1000000000
 
+	bucket := s.SerializeBucketName(metric.Name(), metric.Tags())
+
 	for field_name, value := range metric.Fields() {
 		// Convert value
 		value_str := fmt.Sprintf("%#v", value)
 		// Write graphite metric
 		var graphitePoint string
 		graphitePoint = fmt.Sprintf("%s %s %d",
-			s.SerializeBucketName(metric, field_name),
+			// insert "field" section of template
+			strings.Replace(bucket, "FIELDNAME", field_name, 1),
 			value_str,
 			timestamp)
 		out = append(out, graphitePoint)
 	}
 	return out, nil
 }
 
-func (s *GraphiteSerializer) SerializeBucketName(metric telegraf.Metric, field_name string) string {
-	// Get the metric name
-	name := metric.Name()
+// SerializeBucketName will take the given measurement name and tags and
+// produce a graphite bucket. It will use the GraphiteSerializer.Template
+// to generate this, or DEFAULT_TEMPLATE.
+//
+// NOTE: SerializeBucketName replaces the "field" portion of the template with
+// FIELDNAME. It is up to the user to replace this. This is so that
+// SerializeBucketName can be called just once per measurement, rather than
+// once per field.
+func (s *GraphiteSerializer) SerializeBucketName(
+	measurement string,
+	tags map[string]string,
+) string {
+	if s.Template == "" {
+		s.Template = DEFAULT_TEMPLATE
+	}
+	tagsCopy := make(map[string]string)
+	for k, v := range tags {
+		tagsCopy[k] = v
+	}
 
-	// Convert UnixNano to Unix timestamps
-	tag_str := buildTags(metric)
+	var out []string
+	templateParts := strings.Split(s.Template, ".")
+	for _, templatePart := range templateParts {
+		switch templatePart {
+		case "measurement":
+			out = append(out, measurement)
+		case "tags":
+			// we will replace this later
+			out = append(out, "TAGS")
+		case "field":
+			// user of SerializeBucketName needs to replace this
+			out = append(out, "FIELDNAME")
+		default:
+			// This is a tag being applied
+			if tagvalue, ok := tagsCopy[templatePart]; ok {
+				out = append(out, strings.Replace(tagvalue, ".", "_", -1))
+				delete(tagsCopy, templatePart)
+			}
+		}
+	}
 
-	// Write graphite metric
-	var serializedBucketName string
-	if name == field_name {
-		serializedBucketName = fmt.Sprintf("%s.%s",
-			tag_str,
-			strings.Replace(name, ".", "_", -1))
-	} else {
-		serializedBucketName = fmt.Sprintf("%s.%s.%s",
-			tag_str,
-			strings.Replace(name, ".", "_", -1),
-			strings.Replace(field_name, ".", "_", -1))
+	// insert remaining tags into output name
+	for i, templatePart := range out {
+		if templatePart == "TAGS" {
+			out[i] = buildTags(tagsCopy)
+			break
+		}
 	}
-	if s.Prefix != "" {
-		serializedBucketName = fmt.Sprintf("%s.%s", s.Prefix, serializedBucketName)
+
+	if s.Prefix == "" {
+		return sanitizedChars.Replace(strings.Join(out, "."))
 	}
-	return serializedBucketName
+	return sanitizedChars.Replace(s.Prefix + "." + strings.Join(out, "."))
 }
 
-func buildTags(metric telegraf.Metric) string {
+func buildTags(tags map[string]string) string {
 	var keys []string
-	tags := metric.Tags()
 	for k := range tags {
-		if k == "host" {
-			continue
-		}
 		keys = append(keys, k)
 	}
 	sort.Strings(keys)
 
 	var tag_str string
-	if host, ok := tags["host"]; ok {
-		if len(keys) > 0 {
-			tag_str = strings.Replace(host, ".", "_", -1) + "."
-		} else {
-			tag_str = strings.Replace(host, ".", "_", -1)
-		}
-	}
-
 	for i, k := range keys {
 		tag_value := strings.Replace(tags[k], ".", "_", -1)
 		if i == 0 {
@@ -87,5 +111,5 @@ func buildTags(metric telegraf.Metric) string {
 			tag_str += "." + tag_value
 		}
 	}
-	return sanitizedChars.Replace(tag_str)
+	return tag_str
 }

plugins/serializers/graphite/graphite_test.go

@@ -31,12 +31,12 @@ func TestGraphiteTags(t *testing.T) {
 		time.Date(2010, time.November, 10, 23, 0, 0, 0, time.UTC),
 	)
 
-	tags1 := buildTags(m1)
-	tags2 := buildTags(m2)
-	tags3 := buildTags(m3)
+	tags1 := buildTags(m1.Tags())
+	tags2 := buildTags(m2.Tags())
+	tags3 := buildTags(m3.Tags())
 
 	assert.Equal(t, "192_168_0_1", tags1)
-	assert.Equal(t, "192_168_0_1.first.second", tags2)
+	assert.Equal(t, "first.second.192_168_0_1", tags2)
 	assert.Equal(t, "first.second", tags3)
 }
 
@@ -133,9 +133,9 @@ func TestSerializeBucketNameNoHost(t *testing.T) {
 	assert.NoError(t, err)
 
 	s := GraphiteSerializer{}
-	mS := s.SerializeBucketName(m, "usage_idle")
+	mS := s.SerializeBucketName(m.Name(), m.Tags())
 
-	expS := fmt.Sprintf("cpu0.us-west-2.cpu.usage_idle")
+	expS := fmt.Sprintf("cpu0.us-west-2.cpu.FIELDNAME")
 	assert.Equal(t, expS, mS)
 }
 
@@ -153,9 +153,9 @@ func TestSerializeBucketNameHost(t *testing.T) {
 	assert.NoError(t, err)
 
 	s := GraphiteSerializer{}
-	mS := s.SerializeBucketName(m, "usage_idle")
+	mS := s.SerializeBucketName(m.Name(), m.Tags())
 
-	expS := fmt.Sprintf("localhost.cpu0.us-west-2.cpu.usage_idle")
+	expS := fmt.Sprintf("localhost.cpu0.us-west-2.cpu.FIELDNAME")
 	assert.Equal(t, expS, mS)
 }
 
@@ -173,8 +173,8 @@ func TestSerializeBucketNamePrefix(t *testing.T) {
 	assert.NoError(t, err)
 
 	s := GraphiteSerializer{Prefix: "prefix"}
-	mS := s.SerializeBucketName(m, "usage_idle")
+	mS := s.SerializeBucketName(m.Name(), m.Tags())
 
-	expS := fmt.Sprintf("prefix.localhost.cpu0.us-west-2.cpu.usage_idle")
+	expS := fmt.Sprintf("prefix.localhost.cpu0.us-west-2.cpu.FIELDNAME")
 	assert.Equal(t, expS, mS)
 }

plugins/serializers/registry.go

@@ -30,6 +30,10 @@ type Config struct {
 
 	// Prefix to add to all measurements, only supports Graphite
 	Prefix string
+
+	// Template for converting telegraf metrics into Graphite
+	// only supports Graphite
+	Template string
 }
 
 // NewSerializer a Serializer interface based on the given config.
@@ -40,7 +44,7 @@ func NewSerializer(config *Config) (Serializer, error) {
 	case "influx":
 		serializer, err = NewInfluxSerializer()
 	case "graphite":
-		serializer, err = NewGraphiteSerializer(config.Prefix)
+		serializer, err = NewGraphiteSerializer(config.Prefix, config.Template)
 	case "json":
 		serializer, err = NewJsonSerializer()
 	}
@@ -55,8 +59,9 @@ func NewInfluxSerializer() (Serializer, error) {
 	return &influx.InfluxSerializer{}, nil
 }
 
-func NewGraphiteSerializer(prefix string) (Serializer, error) {
+func NewGraphiteSerializer(prefix, template string) (Serializer, error) {
 	return &graphite.GraphiteSerializer{
-		Prefix: prefix,
+		Prefix:   prefix,
+		Template: template,
 	}, nil
 }