bson: Add decoder

Only supports basic types for now

.vscode/settings.json

@@ -11,6 +11,7 @@
         "BCDU",
         "bgrep",
         "bitio",
+        "bson",
         "bxor",
         "bzip",
         "CCIT",
@@ -138,6 +139,7 @@
         "toimage",
         "tojson",
         "tojvalue",
+        "torepr",
         "tovalue",
         "traf",
         "trex",

README.md

@@ -94,7 +94,7 @@ cp fq /usr/local/bin
 
 [./formats_list.jq]: sh-start
 
-aac_frame, adts, adts_frame, apev2, ar, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bsd_loopback_frame, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip
+aac_frame, adts, adts_frame, apev2, ar, av1_ccr, av1_frame, av1_obu, avc_annexb, avc_au, avc_dcr, avc_nalu, avc_pps, avc_sei, avc_sps, bsd_loopback_frame, bson, bzip2, dns, dns_tcp, elf, ether8023_frame, exif, flac, flac_frame, flac_metadatablock, flac_metadatablocks, flac_picture, flac_streaminfo, gif, gzip, hevc_annexb, hevc_au, hevc_dcr, hevc_nalu, icc_profile, icmp, id3v1, id3v11, id3v2, ipv4_packet, jpeg, json, matroska, mp3, mp3_frame, mp4, mpeg_asc, mpeg_es, mpeg_pes, mpeg_pes_packet, mpeg_spu, mpeg_ts, ogg, ogg_page, opus_packet, pcap, pcapng, png, protobuf, protobuf_widevine, pssh_playready, raw, sll2_packet, sll_packet, tar, tcp_segment, tiff, udp_datagram, vorbis_comment, vorbis_packet, vp8_frame, vp9_cfm, vp9_frame, vpx_ccr, wav, webp, xing, zip
 
 [#]: sh-end
 

doc/formats.md

@@ -20,6 +20,7 @@
 |`avc_sei`             |H.264/AVC&nbsp;Supplemental&nbsp;Enhancement&nbsp;Information |<sub></sub>|
 |`avc_sps`             |H.264/AVC&nbsp;Sequence&nbsp;Parameter&nbsp;Set               |<sub></sub>|
 |`bsd_loopback_frame`  |BSD&nbsp;loopback&nbsp;frame                                  |<sub>`ipv4_packet`</sub>|
+|`bson`                |Binary&nbsp;JSON                                              |<sub></sub>|
 |`bzip2`               |bzip2&nbsp;compression                                        |<sub>`probe`</sub>|
 |`dns`                 |DNS&nbsp;packet                                               |<sub></sub>|
 |`dns_tcp`             |DNS&nbsp;packet&nbsp;(TCP)                                    |<sub></sub>|

format/all/all.go

@@ -6,6 +6,7 @@ import (
 	_ "github.com/wader/fq/format/ape"
 	_ "github.com/wader/fq/format/ar"
 	_ "github.com/wader/fq/format/av1"
+	_ "github.com/wader/fq/format/bson"
 	_ "github.com/wader/fq/format/bzip2"
 	_ "github.com/wader/fq/format/dns"
 	_ "github.com/wader/fq/format/elf"

format/bson/bson.go

@@ -0,0 +1,116 @@
+package bson
+
+// https://bsonspec.org/spec.html
+// TODO: more types
+
+import (
+	"embed"
+
+	"github.com/wader/fq/format"
+	"github.com/wader/fq/format/registry"
+	"github.com/wader/fq/pkg/decode"
+	"github.com/wader/fq/pkg/scalar"
+)
+
+//go:embed *.jq
+var bsonFS embed.FS
+
+func init() {
+	registry.MustRegister(decode.Format{
+		Name:        format.BSON,
+		Description: "Binary JSON",
+		DecodeFn:    decodeBSON,
+		Files:       bsonFS,
+	})
+}
+
+const (
+	elementTypeDouble    = 0x01
+	elementTypeString    = 0x02
+	elementTypeDocument  = 0x03
+	elementTypeArray     = 0x04
+	elementTypeBinary    = 0x05
+	elementTypeUndefined = 0x06
+	elementTypeObjectID  = 0x07
+	elementTypeBoolean   = 0x08
+	elementTypeDatatime  = 0x09
+	elementTypeNull      = 0x0a
+	elementTypeRegexp    = 0x0b
+	elementTypeInt32     = 0x10
+	elementTypeTimestamp = 0x11
+	elementTypeInt64     = 0x12
+)
+
+var elementTypeMap = scalar.UToScalar{
+	elementTypeDouble:    {Sym: "double", Description: "64-bit binary floating point"},
+	elementTypeString:    {Sym: "string", Description: "UTF-8 string"},
+	elementTypeDocument:  {Sym: "document", Description: "Embedded document"},
+	elementTypeArray:     {Sym: "array", Description: "Array"},
+	elementTypeBinary:    {Sym: "binary", Description: "Binary data"},
+	elementTypeUndefined: {Sym: "undefined", Description: "Undefined (deprecated)"},
+	elementTypeObjectID:  {Sym: "object_id", Description: "ObjectId"},
+	elementTypeBoolean:   {Sym: "boolean", Description: "Boolean"},
+	elementTypeDatatime:  {Sym: "datatime", Description: "UTC datetime"},
+	elementTypeNull:      {Sym: "null", Description: "Null value"},
+	elementTypeRegexp:    {Sym: "regexp", Description: "Regular expression"},
+	elementTypeInt32:     {Sym: "int32", Description: "32-bit integer"},
+	elementTypeTimestamp: {Sym: "timestamp", Description: "Timestamp"},
+	elementTypeInt64:     {Sym: "int64", Description: "64-bit integer"},
+}
+
+func decodeBSONDocument(d *decode.D) {
+	size := d.FieldU32("size")
+	d.LenFn(int64(size-4)*8, func(d *decode.D) {
+		d.FieldArray("elements", func(d *decode.D) {
+			for d.BitsLeft() > 8 {
+				d.FieldStruct("element", func(d *decode.D) {
+					typ := d.FieldU8("type", elementTypeMap)
+					d.FieldUTF8Null("name")
+					switch typ {
+					case elementTypeDouble:
+						d.FieldF64("value")
+					case elementTypeString:
+						length := d.FieldU32("length")
+						d.FieldUTF8NullFixedLen("value", int(length))
+					case elementTypeDocument:
+						d.FieldStruct("value", decodeBSONDocument)
+					case elementTypeArray:
+						d.FieldStruct("value", decodeBSONDocument)
+					case elementTypeBinary:
+						length := d.FieldU32("length")
+						d.FieldU8("subtype")
+						d.FieldRawLen("value", int64(length)*8)
+					case elementTypeUndefined:
+					case elementTypeObjectID:
+						d.FieldRawLen("value", 12*8)
+					case elementTypeBoolean:
+						d.FieldU8("value")
+					case elementTypeDatatime:
+						d.FieldS32("value")
+					case elementTypeNull:
+					case elementTypeRegexp:
+						d.FieldUTF8Null("value")
+						d.FieldUTF8Null("options")
+					case elementTypeInt32:
+						d.FieldS32("value")
+					case elementTypeTimestamp:
+						d.FieldU64("value")
+					case elementTypeInt64:
+						d.FieldS64("value")
+					default:
+						d.FieldRawLen("value", d.BitsLeft())
+					}
+				})
+			}
+		})
+		d.FieldU8("terminator", d.ValidateU(0))
+	})
+}
+
+func decodeBSON(d *decode.D, in interface{}) interface{} {
+	d.Endian = decode.LittleEndian
+
+	decodeBSONDocument(d)
+
+	return nil
+}

format/bson/bson.jq

@@ -0,0 +1,18 @@
+def bson_torepr:
+  def _torepr:
+   ( if .type == null or .type == "array" then
+      ( .value.elements
+      | map(_torepr)
+      )
+     elif .type == "document" then
+      ( .value.elements
+      | map({key: .name, value: _torepr})
+      | from_entries
+      )
+     elif .type == "boolean" then .value != 0
+     else .value
+     end
+   );
+  ( {type: "document", value: .}
+  | _torepr
+  );