Add support for the "extra" field in dnstap payload for JSON and YAML output

JsonFormat.go

@@ -16,9 +16,11 @@ package dnstap
 
 import (
 	"bytes"
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"net"
+	"strconv"
 	"time"
 
 	"github.com/miekg/dns"
@@ -35,6 +37,7 @@ type jsonDnstap struct {
 	Type     string      `json:"type"`
 	Identity string      `json:"identity,omitempty"`
 	Version  string      `json:"version,omitempty"`
+	Extra    string	     `json:"extra,omitempty"`
 	Message  jsonMessage `json:"message"`
 }
 
@@ -121,13 +124,46 @@ func convertJSONMessage(m *Message) jsonMessage {
 
 // JSONFormat renders a Dnstap message in JSON format. Any encapsulated
 // DNS messages are rendered as strings in a format similar to 'dig' output.
+// "extra" field may be escaped if it contains non-printable characters
 func JSONFormat(dt *Dnstap) (out []byte, ok bool) {
+	return jsonFormat(dt, ExtraTextFmt)
+}
+
+// JSONFormatWithHexExtra renders a Dnstap message in JSON format.
+// Similar to JSONFormat, but "extra" field in dnstap payload rendered as hex.
+func JSONFormatWithHexExtra(dt *Dnstap) (out []byte, ok bool) {
+	return jsonFormat(dt, ExtraHexFmt)
+}
+
+// JSONFormatWithBase64Extra renders a Dnstap message in JSON format.
+// Similar to JSONFormat, but "extra" field in dnstap payload rendered as base64.
+func JSONFormatWithBase64Extra(dt *Dnstap) (out []byte, ok bool) {
+	return jsonFormat(dt, ExtraBase64Fmt)
+}
+
+func jsonFormat(dt *Dnstap, extraFormat ExtraFormat) (out []byte, ok bool) {
 	var s bytes.Buffer
 
+	extra := ""
+	if dt.Extra != nil {
+		switch extraFormat {
+		case ExtraTextFmt:
+			// escape non-printable characters
+			extra = strconv.Quote(string(dt.Extra))
+			// remove added quotes
+			extra = extra[1 : len(extra)-1]
+		case ExtraHexFmt:
+			extra = fmt.Sprintf("%x", dt.Extra)
+		case ExtraBase64Fmt:
+			extra = base64.StdEncoding.EncodeToString(dt.Extra)
+		}
+	}
+
 	j, err := json.Marshal(jsonDnstap{
 		Type:     fmt.Sprint(dt.Type),
 		Identity: string(dt.Identity),
 		Version:  string(dt.Version),
+		Extra:    extra,
 		Message:  convertJSONMessage(dt.Message),
 	})
 	if err != nil {

YamlFormat.go

@@ -18,6 +18,7 @@ package dnstap
 
 import (
 	"bytes"
+	"encoding/base64"
 	"fmt"
 	"net"
 	"strconv"
@@ -98,9 +99,34 @@ func yamlConvertMessage(m *Message, s *bytes.Buffer) {
 	s.WriteString("---\n")
 }
 
+type ExtraFormat int
+
+const (
+	ExtraTextFmt ExtraFormat = iota
+	ExtraHexFmt
+	ExtraBase64Fmt
+)
+
 // YamlFormat renders a dnstap message in YAML format. Any encapsulated DNS
 // messages are rendered as strings in a format similar to 'dig' output.
+// "extra" field in dnstap payload may be escaped if it contains non-printable characters
 func YamlFormat(dt *Dnstap) (out []byte, ok bool) {
+	return yamlFormat(dt, ExtraTextFmt)
+}
+
+// YamlFormatWithHexExtra renders a dnstap message in YAML format.
+// Similar to YamlFormat, but "extra" field in dnstap payload rendered as hex.
+func YamlFormatWithHexExtra(dt *Dnstap) (out []byte, ok bool) {
+	return yamlFormat(dt, ExtraHexFmt)
+}
+
+// YamlFormatWithBase64Extra renders a dnstap message in YAML format.
+// Similar to YamlFormat, but "extra" field in dnstap payload rendered as base64.
+func YamlFormatWithBase64Extra(dt *Dnstap) (out []byte, ok bool) {
+	return yamlFormat(dt, ExtraBase64Fmt)
+}
+
+func yamlFormat(dt *Dnstap, extraFormat ExtraFormat) (out []byte, ok bool) {
 	var s bytes.Buffer
 
 	s.WriteString(fmt.Sprint("type: ", dt.Type, "\n"))
@@ -110,6 +136,16 @@ func YamlFormat(dt *Dnstap) (out []byte, ok bool) {
 	if dt.Version != nil {
 		s.WriteString(fmt.Sprint("version: ", strconv.Quote(string(dt.Version)), "\n"))
 	}
+	if dt.Extra != nil {
+		switch extraFormat {
+		case ExtraTextFmt:
+			s.WriteString(fmt.Sprint("extra: ", strconv.Quote(string(dt.Extra)), "\n"))
+		case ExtraHexFmt:
+			s.WriteString(fmt.Sprint("extra: ", fmt.Sprintf("%x", dt.Extra), "\n"))
+		case ExtraBase64Fmt:
+			s.WriteString(fmt.Sprint("extra: ", base64.StdEncoding.EncodeToString(dt.Extra), "\n"))
+		}
+	}
 	if *dt.Type == Dnstap_MESSAGE {
 		s.WriteString("message:\n")
 		yamlConvertMessage(dt.Message, &s)

dnstap/dnstap.8

@@ -135,6 +135,22 @@ form similar to the output of \fBdig(1)\fR.
 At most one text format (\fB-j\fR, \fB-q\fR, or \fB-y\fR) option may be given.
 
 
+.TP
+.B -x \fIformat-type\fR
+Specify output format of the 'extra' field in dnstap payload if exists.
+
+Available options for \fIformat-type\fR includes:
+ - text (default)
+ - hex
+ - base64
+
+For the default text format, if the 'extra' field in payload contains
+non-printable characters, the 'extra' text will be escaped.
+
+Valid only when the YAML or JSON format is specified with the \fB-y\fR
+or \fB-j\fR options. (bianry output does not need a format)
+
+
 .SH EXAMPLES
 
 Listen for Dnstap data from a local name server and print quiet text format

dnstap/main.go

@@ -46,6 +46,7 @@ var (
 	flagQuietText  = flag.Bool("q", false, "use quiet text output")
 	flagYamlText   = flag.Bool("y", false, "use verbose YAML output")
 	flagJSONText   = flag.Bool("j", false, "use verbose JSON output")
+	flagExtraFmt   = flag.String("x", "", "specify the 'extra' field format in output. Available options:\n - text (default)\n - hex\n - base64\nvalid when outputting YAML or JSON.")
 )
 
 func usage() {
@@ -116,10 +117,22 @@ func main() {
 		switch {
 		case *flagYamlText:
 			format = dnstap.YamlFormat
+			switch *flagExtraFmt{
+			case "hex":
+				format = dnstap.YamlFormatWithHexExtra
+			case "base64":
+				format = dnstap.YamlFormatWithBase64Extra
+			}
 		case *flagQuietText:
 			format = dnstap.TextFormat
 		case *flagJSONText:
 			format = dnstap.JSONFormat
+			switch *flagExtraFmt{
+			case "hex":
+				format = dnstap.JSONFormatWithHexExtra
+			case "base64":
+				format = dnstap.JSONFormatWithBase64Extra
+			}
 		}
 
 		o, err := newFileOutput(*flagWriteFile, format, *flagAppendFile)