bson: support all non-deprecated types and fix int/uint bugs

doc/formats.md

@@ -434,6 +434,10 @@ bplist> from_ns_keyed_archiver(1)
 
 ## bson
 
+### Limitations:
+
+- The decimal128 type is not supported for decoding, will just be treated as binary
+
 ### Convert represented value to JSON
 
 ```

format/bson/bson.go

@@ -1,7 +1,6 @@
 package bson
 
 // https://bsonspec.org/spec.html
-// TODO: more types
 
 import (
 	"embed"
@@ -27,42 +26,50 @@ func init() {
 }
 
 const (
-	elementTypeDouble    = 0x01
-	elementTypeString    = 0x02
-	elementTypeDocument  = 0x03
-	elementTypeArray     = 0x04
-	elementTypeBinary    = 0x05
-	elementTypeUndefined = 0x06
-	elementTypeObjectID  = 0x07
-	elementTypeBoolean   = 0x08
-	elementTypeDatatime  = 0x09
-	elementTypeNull      = 0x0a
-	elementTypeRegexp    = 0x0b
-	elementTypeInt32     = 0x10
-	elementTypeTimestamp = 0x11
-	elementTypeInt64     = 0x12
+	elementTypeDouble     = 0x01
+	elementTypeString     = 0x02
+	elementTypeDocument   = 0x03
+	elementTypeArray      = 0x04
+	elementTypeBinary     = 0x05
+	elementTypeUndefined  = 0x06
+	elementTypeObjectID   = 0x07
+	elementTypeBoolean    = 0x08
+	elementTypeDatetime   = 0x09
+	elementTypeNull       = 0x0a
+	elementTypeRegexp     = 0x0b
+	elementTypeJavaScript = 0x0d
+	elementTypeInt32      = 0x10
+	elementTypeTimestamp  = 0x11
+	elementTypeInt64      = 0x12
+	elementTypeDecimal128 = 0x13
+	elementTypeMinKey     = 0xFF
+	elementTypeMaxKey     = 0x7f
 )
 
 var elementTypeMap = scalar.UintMap{
-	elementTypeDouble:    {Sym: "double", Description: "64-bit binary floating point"},
-	elementTypeString:    {Sym: "string", Description: "UTF-8 string"},
-	elementTypeDocument:  {Sym: "document", Description: "Embedded document"},
-	elementTypeArray:     {Sym: "array", Description: "Array"},
-	elementTypeBinary:    {Sym: "binary", Description: "Binary data"},
-	elementTypeUndefined: {Sym: "undefined", Description: "Undefined (deprecated)"},
-	elementTypeObjectID:  {Sym: "object_id", Description: "ObjectId"},
-	elementTypeBoolean:   {Sym: "boolean", Description: "Boolean"},
-	elementTypeDatatime:  {Sym: "datatime", Description: "UTC datetime"},
-	elementTypeNull:      {Sym: "null", Description: "Null value"},
-	elementTypeRegexp:    {Sym: "regexp", Description: "Regular expression"},
-	elementTypeInt32:     {Sym: "int32", Description: "32-bit integer"},
-	elementTypeTimestamp: {Sym: "timestamp", Description: "Timestamp"},
-	elementTypeInt64:     {Sym: "int64", Description: "64-bit integer"},
+	elementTypeDouble:     {Sym: "double", Description: "64-bit binary floating point"},
+	elementTypeString:     {Sym: "string", Description: "UTF-8 string"},
+	elementTypeDocument:   {Sym: "document", Description: "Embedded document"},
+	elementTypeArray:      {Sym: "array", Description: "Array"},
+	elementTypeBinary:     {Sym: "binary", Description: "Binary data"},
+	elementTypeUndefined:  {Sym: "undefined", Description: "Undefined (deprecated)"},
+	elementTypeObjectID:   {Sym: "object_id", Description: "ObjectId"},
+	elementTypeBoolean:    {Sym: "boolean", Description: "Boolean"},
+	elementTypeDatetime:   {Sym: "datetime", Description: "UTC datetime"},
+	elementTypeNull:       {Sym: "null", Description: "Null value"},
+	elementTypeRegexp:     {Sym: "regexp", Description: "Regular expression"},
+	elementTypeJavaScript: {Sym: "javascript", Description: "JavaScript code"},
+	elementTypeInt32:      {Sym: "int32", Description: "32-bit integer"},
+	elementTypeTimestamp:  {Sym: "timestamp", Description: "Timestamp"},
+	elementTypeInt64:      {Sym: "int64", Description: "64-bit integer"},
+	elementTypeDecimal128: {Sym: "decimal128", Description: "128-bit decimal floating point"},
+	elementTypeMinKey:     {Sym: "minkey", Description: "Min key"},
+	elementTypeMaxKey:     {Sym: "maxkey", Description: "Max key"},
 }
 
 func decodeBSONDocument(d *decode.D) {
-	size := d.FieldU32("size")
-	d.FramedFn(int64(size-4)*8, func(d *decode.D) {
+	size := d.FieldS32("size")
+	d.FramedFn((size-4)*8, func(d *decode.D) {
 		d.FieldArray("elements", func(d *decode.D) {
 			for d.BitsLeft() > 8 {
 				d.FieldStruct("element", func(d *decode.D) {
@@ -79,28 +86,38 @@ func decodeBSONDocument(d *decode.D) {
 					case elementTypeArray:
 						d.FieldStruct("value", decodeBSONDocument)
 					case elementTypeBinary:
-						length := d.FieldU32("length")
+						length := d.FieldS32("length")
 						d.FieldU8("subtype")
-						d.FieldRawLen("value", int64(length)*8)
+						d.FieldRawLen("value", length*8)
 					case elementTypeUndefined:
 						//deprecated
 					case elementTypeObjectID:
 						d.FieldRawLen("value", 12*8)
 					case elementTypeBoolean:
 						d.FieldU8("value")
-					case elementTypeDatatime:
-						d.FieldS32("value")
+					case elementTypeDatetime:
+						d.FieldS64("value")
 					case elementTypeNull:
 						d.FieldValueAny("value", nil)
 					case elementTypeRegexp:
 						d.FieldUTF8Null("value")
 						d.FieldUTF8Null("options")
+					case elementTypeJavaScript:
+						length := d.FieldS32("length")
+						d.FieldUTF8NullFixedLen("value", int(length))
 					case elementTypeInt32:
 						d.FieldS32("value")
 					case elementTypeTimestamp:
 						d.FieldU64("value")
 					case elementTypeInt64:
 						d.FieldS64("value")
+					case elementTypeDecimal128:
+						// TODO: Parse the IEEE 754 decimal128 value.
+						d.FieldRawLen("value", 128)
+					case elementTypeMinKey:
+						d.FieldValueAny("value", nil)
+					case elementTypeMaxKey:
+						d.FieldValueAny("value", nil)
 					default:
 						d.FieldRawLen("value", d.BitsLeft())
 					}

format/bson/bson.md

@@ -1,3 +1,7 @@
+### Limitations
+
+- The decimal128 type is not supported for decoding, will just be treated as binary
+
 ### Convert represented value to JSON
 
 ```
@@ -10,5 +14,9 @@ $ fq -d bson torepr file.bson
 $ fq -d bson 'torepr | select(.name=="bob")' file.bson
 ```
 
+### Authors
+- Mattias Wadman mattias.wadman@gmail.com, original author
+- Matt Dale [@matthewdale](https://github.com/matthewdale), additional types and bug fixes
+
 ### References
 - https://bsonspec.org/spec.html

format/bson/testdata/generate_testbson/go.mod

@@ -0,0 +1,5 @@
+module generate_testbson
+
+go 1.18
+
+require go.mongodb.org/mongo-driver v1.11.4

format/bson/testdata/generate_testbson/go.sum

@@ -0,0 +1,40 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=
+github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
+github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g=
+github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8=
+github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
+go.mongodb.org/mongo-driver v1.11.4 h1:4ayjakA013OdpGyL2K3ZqylTac/rMjrJOMZ1EHizXas=
+go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g=
+golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

format/bson/testdata/generate_testbson/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"os"
+
+	"go.mongodb.org/mongo-driver/bson/bsontype"
+	"go.mongodb.org/mongo-driver/bson/primitive"
+	"go.mongodb.org/mongo-driver/x/bsonx/bsoncore"
+)
+
+// Writes a BSON document to STDOUT that contains at least one element of every non-deprecated field
+// type.
+
+func main() {
+	oid, err := primitive.ObjectIDFromHex("644b1619251be740e85522d6")
+	if err != nil {
+		panic(err)
+	}
+
+	d128, err := primitive.ParseDecimal128("123.456")
+	if err != nil {
+		panic(err)
+	}
+
+	b := bsoncore.NewDocumentBuilder().
+		AppendDouble("dou", 98.765).
+		AppendString("str", "my string").
+		AppendDocument("doc", bsoncore.NewDocumentBuilder().
+			AppendString("nstr", "nested string").
+			AppendArray("narr", bsoncore.NewArrayBuilder().
+				AppendInt32(-123).
+				Build()).
+			AppendDouble("ndou", 98.765).
+			Build()).
+		AppendArray("arr", bsoncore.NewArrayBuilder().
+			AppendString("arr string").
+			AppendDocument(bsoncore.NewDocumentBuilder().
+				AppendInt32("ni32", -123).
+				Build()).
+			AppendDouble(98.765).
+			Build()).
+		AppendBinary("bin", bsontype.BinaryGeneric, []byte{0, 1, 2, 3, 4}).
+		AppendObjectID("_id", oid).
+		AppendBoolean("boo", true).
+		AppendDateTime("dat", 1682642622682).
+		AppendNull("nul").
+		AppendRegex("reg", "my pattern", "ix").
+		AppendJavaScript("jav", "var x = 5;").
+		AppendInt32("i32", -123).
+		AppendTimestamp("tim", 123, 1682642846).
+		AppendInt64("i64", -456).
+		AppendDecimal128("dec", d128).
+		AppendMinKey("min").
+		AppendMaxKey("max").
+		Build()
+
+	_, err = os.Stdout.Write(b)
+	if err != nil {
+		panic(err)
+	}
+}

format/bson/testdata/help_bson.fqtest

@@ -9,6 +9,11 @@ Decode examples
   # Decode value as bson
   ... | bson
 
+Limitations
+===========
+
+- The decimal128 type is not supported for decoding, will just be treated as binary
+
 Convert represented value to JSON
 =================================
 
@@ -19,6 +24,12 @@ Filter represented value
 
   $ fq -d bson 'torepr | select(.name=="bob")' file.bson
 
+Authors
+=======
+
+- Mattias Wadman mattias.wadman@gmail.com, original author
+- Matt Dale @matthewdale (https://github.com/matthewdale), additional types and bug fixes
+
 References
 ==========