From ac575cc13dd5f0ac538fedaf54c6127108236da4 Mon Sep 17 00:00:00 2001 From: chenyuheng Date: Tue, 8 Aug 2023 19:55:20 -0400 Subject: [PATCH 1/2] add 'extra' field support for json and yaml output --- JsonFormat.go | 36 ++++++++++++++++++++++++++++++++++++ YamlFormat.go | 36 ++++++++++++++++++++++++++++++++++++ dnstap/dnstap.8 | 13 +++++++++++++ dnstap/main.go | 13 +++++++++++++ 4 files changed, 98 insertions(+) diff --git a/JsonFormat.go b/JsonFormat.go index 092026a..54045c4 100644 --- a/JsonFormat.go +++ b/JsonFormat.go @@ -16,9 +16,11 @@ package dnstap import ( "bytes" + "encoding/base64" "encoding/json" "fmt" "net" + "strconv" "time" "github.com/miekg/dns" @@ -35,6 +37,7 @@ type jsonDnstap struct { Type string `json:"type"` Identity string `json:"identity,omitempty"` Version string `json:"version,omitempty"` + Extra string `json:"extra,omitempty"` Message jsonMessage `json:"message"` } @@ -121,13 +124,46 @@ func convertJSONMessage(m *Message) jsonMessage { // JSONFormat renders a Dnstap message in JSON format. Any encapsulated // DNS messages are rendered as strings in a format similar to 'dig' output. +// "extra" field may be escaped if it contains non-printable characters func JSONFormat(dt *Dnstap) (out []byte, ok bool) { + return jsonFormat(dt, ExtraTextFmt) +} + +// JSONFormatWithHexExtra renders a Dnstap message in JSON format. +// Similar to JSONFormat, but "extra" field in dnstap payload rendered as hex. +func JSONFormatWithHexExtra(dt *Dnstap) (out []byte, ok bool) { + return jsonFormat(dt, ExtraHexFmt) +} + +// JSONFormatWithBase64Extra renders a Dnstap message in JSON format. +// Similar to JSONFormat, but "extra" field in dnstap payload rendered as base64. +func JSONFormatWithBase64Extra(dt *Dnstap) (out []byte, ok bool) { + return jsonFormat(dt, ExtraBase64Fmt) +} + +func jsonFormat(dt *Dnstap, extraFormat ExtraFormat) (out []byte, ok bool) { var s bytes.Buffer + extra := "" + if dt.Extra != nil { + switch extraFormat { + case ExtraTextFmt: + // escape non-printable characters + extra = strconv.Quote(string(dt.Extra)) + // remove added quotes + extra = extra[1 : len(extra)-1] + case ExtraHexFmt: + extra = fmt.Sprintf("%x", dt.Extra) + case ExtraBase64Fmt: + extra = base64.StdEncoding.EncodeToString(dt.Extra) + } + } + j, err := json.Marshal(jsonDnstap{ Type: fmt.Sprint(dt.Type), Identity: string(dt.Identity), Version: string(dt.Version), + Extra: extra, Message: convertJSONMessage(dt.Message), }) if err != nil { diff --git a/YamlFormat.go b/YamlFormat.go index 73f94f7..c795eeb 100644 --- a/YamlFormat.go +++ b/YamlFormat.go @@ -18,6 +18,7 @@ package dnstap import ( "bytes" + "encoding/base64" "fmt" "net" "strconv" @@ -98,9 +99,34 @@ func yamlConvertMessage(m *Message, s *bytes.Buffer) { s.WriteString("---\n") } +type ExtraFormat int + +const ( + ExtraTextFmt ExtraFormat = iota + ExtraHexFmt + ExtraBase64Fmt +) + // YamlFormat renders a dnstap message in YAML format. Any encapsulated DNS // messages are rendered as strings in a format similar to 'dig' output. +// "extra" field in dnstap payload may be escaped if it contains non-printable characters func YamlFormat(dt *Dnstap) (out []byte, ok bool) { + return yamlFormat(dt, ExtraTextFmt) +} + +// YamlFormatWithHexExtra renders a dnstap message in YAML format. +// Similar to YamlFormat, but "extra" field in dnstap payload rendered as hex. +func YamlFormatWithHexExtra(dt *Dnstap) (out []byte, ok bool) { + return yamlFormat(dt, ExtraHexFmt) +} + +// YamlFormatWithBase64Extra renders a dnstap message in YAML format. +// Similar to YamlFormat, but "extra" field in dnstap payload rendered as base64. +func YamlFormatWithBase64Extra(dt *Dnstap) (out []byte, ok bool) { + return yamlFormat(dt, ExtraBase64Fmt) +} + +func yamlFormat(dt *Dnstap, extraFormat ExtraFormat) (out []byte, ok bool) { var s bytes.Buffer s.WriteString(fmt.Sprint("type: ", dt.Type, "\n")) @@ -110,6 +136,16 @@ func YamlFormat(dt *Dnstap) (out []byte, ok bool) { if dt.Version != nil { s.WriteString(fmt.Sprint("version: ", strconv.Quote(string(dt.Version)), "\n")) } + if dt.Extra != nil { + switch extraFormat { + case ExtraTextFmt: + s.WriteString(fmt.Sprint("extra: ", strconv.Quote(string(dt.Extra)), "\n")) + case ExtraHexFmt: + s.WriteString(fmt.Sprint("extra: ", fmt.Sprintf("%x", dt.Extra), "\n")) + case ExtraBase64Fmt: + s.WriteString(fmt.Sprint("extra: ", base64.StdEncoding.EncodeToString(dt.Extra), "\n")) + } + } if *dt.Type == Dnstap_MESSAGE { s.WriteString("message:\n") yamlConvertMessage(dt.Message, &s) diff --git a/dnstap/dnstap.8 b/dnstap/dnstap.8 index 2e80dd3..80948df 100644 --- a/dnstap/dnstap.8 +++ b/dnstap/dnstap.8 @@ -135,6 +135,19 @@ form similar to the output of \fBdig(1)\fR. At most one text format (\fB-j\fR, \fB-q\fR, or \fB-y\fR) option may be given. +.TP +.B -x \fIformat-type\fR +Specify output format of the 'extra' field in dnstap payload if exists. + +Available options for \fIformat-type\fR includes: + - text (default) + - hex + - base64 + +Valid only when the YAML or JSON format is specified with the \fB-y\fR +or \fB-j\fR options. + + .SH EXAMPLES Listen for Dnstap data from a local name server and print quiet text format diff --git a/dnstap/main.go b/dnstap/main.go index 613fabf..a0a90cb 100644 --- a/dnstap/main.go +++ b/dnstap/main.go @@ -46,6 +46,7 @@ var ( flagQuietText = flag.Bool("q", false, "use quiet text output") flagYamlText = flag.Bool("y", false, "use verbose YAML output") flagJSONText = flag.Bool("j", false, "use verbose JSON output") + flagExtraFmt = flag.String("x", "", "specify the 'extra' field format in output. Available options:\n - text (default)\n - hex\n - base64\nvalid when outputting YAML or JSON.") ) func usage() { @@ -116,10 +117,22 @@ func main() { switch { case *flagYamlText: format = dnstap.YamlFormat + switch *flagExtraFmt{ + case "hex": + format = dnstap.YamlFormatWithHexExtra + case "base64": + format = dnstap.YamlFormatWithBase64Extra + } case *flagQuietText: format = dnstap.TextFormat case *flagJSONText: format = dnstap.JSONFormat + switch *flagExtraFmt{ + case "hex": + format = dnstap.JSONFormatWithHexExtra + case "base64": + format = dnstap.JSONFormatWithBase64Extra + } } o, err := newFileOutput(*flagWriteFile, format, *flagAppendFile) From d47c56398b03aab660211049d2ee61e93f8fde73 Mon Sep 17 00:00:00 2001 From: chenyuheng Date: Tue, 8 Aug 2023 20:00:53 -0400 Subject: [PATCH 2/2] update doc for extra field --- dnstap/dnstap.8 | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dnstap/dnstap.8 b/dnstap/dnstap.8 index 80948df..aed3b3d 100644 --- a/dnstap/dnstap.8 +++ b/dnstap/dnstap.8 @@ -144,8 +144,11 @@ Available options for \fIformat-type\fR includes: - hex - base64 +For the default text format, if the 'extra' field in payload contains +non-printable characters, the 'extra' text will be escaped. + Valid only when the YAML or JSON format is specified with the \fB-y\fR -or \fB-j\fR options. +or \fB-j\fR options. (bianry output does not need a format) .SH EXAMPLES