From 3097d9a05d8cce1cbeefa85905950e907df5b030 Mon Sep 17 00:00:00 2001 From: Alex Date: Sun, 14 Feb 2021 15:11:12 -0600 Subject: [PATCH] Added requireRecordFields option to Newtonsoft.Json --- src/FsCodec.NewtonsoftJson/Codec.fs | 52 +++++++++++++------ .../FsCodec.NewtonsoftJson.fsproj | 1 - .../VerbatimUtf8Converter.fs | 6 ++- .../FsCodec.SystemTextJson.fsproj | 1 - src/FsCodec/FsCodec.fsproj | 8 +++ src/FsCodec/Internal.fs | 43 +++++++++++++++ .../VerbatimUtf8ConverterTests.fs | 29 +++++++++-- 7 files changed, 117 insertions(+), 23 deletions(-) create mode 100644 src/FsCodec/Internal.fs diff --git a/src/FsCodec.NewtonsoftJson/Codec.fs b/src/FsCodec.NewtonsoftJson/Codec.fs index f979580..e1bfc68 100755 --- a/src/FsCodec.NewtonsoftJson/Codec.fs +++ b/src/FsCodec.NewtonsoftJson/Codec.fs @@ -44,10 +44,27 @@ module Core = // TOCONSIDER as noted in the comments on RecyclableMemoryStream.ToArray, ideally we'd be continuing the rental and passing out a Span ms.ToArray() - member __.Decode(json : byte[]) = + member __.Decode(json : byte[]) : 'a = use ms = Utf8BytesEncoder.wrapAsStream json use jsonReader = Utf8BytesEncoder.makeJsonReader ms - serializer.Deserialize<'T>(jsonReader) + let returnType = typeof<'a> + if returnType = typeof then + json + |> System.Text.Encoding.ASCII.GetString + |> Guid.Parse + |> unbox + elif returnType = typeof then + json + |> System.Text.Encoding.ASCII.GetString + |> Boolean.Parse + |> unbox + elif returnType = typeof then + json + |> System.Text.Encoding.UTF8.GetChars + |> Seq.head + |> unbox + else + serializer.Deserialize<'a> jsonReader /// Provides Codecs that render to a UTF-8 array suitable for storage in Event Stores based using Newtonsoft.Json and the conventions implied by using /// TypeShape.UnionContract.UnionContractEncoder - if you need full control and/or have have your own codecs, see FsCodec.Codec.Create instead @@ -73,19 +90,18 @@ type Codec private () = /// Configuration to be used by the underlying Newtonsoft.Json Serializer when encoding/decoding. Defaults to same as Settings.Create() [] ?settings, /// Enables one to fail encoder generation if union contains nullary cases. Defaults to false, i.e. permitting them - [] ?rejectNullaryCases) + [] ?rejectNullaryCases, + [] ?requireRecordFields) : FsCodec.IEventCodec<'Event, byte[], 'Context> = let settings = match settings with Some x -> x | None -> defaultSettings.Value - let bytesEncoder : TypeShape.UnionContract.IEncoder<_> = new Core.BytesEncoder(settings) :> _ + let bytesEncoder : TypeShape.UnionContract.IEncoder<_> = Core.BytesEncoder(settings) :> _ + let requireRecordFields = defaultArg requireRecordFields true + Internal.checkIfSupported<'Contract> requireRecordFields let dataCodec = TypeShape.UnionContract.UnionContractEncoder.Create<'Contract, byte[]>( bytesEncoder, - // For now, we hard wire in disabling of non-record bodies as: - // a) it's extra yaks to shave - // b) it's questionable whether allowing one to define event contracts that preclude adding extra fields is a useful idea in the first instance - // See VerbatimUtf8EncoderTests.fs and InteropTests.fs - there are edge cases when `d` fields have null / zero-length / missing values - requireRecordFields = true, + requireRecordFields = requireRecordFields, allowNullaryCases = not (defaultArg rejectNullaryCases false)) { new FsCodec.IEventCodec<'Event, byte[], 'Context> with @@ -119,14 +135,16 @@ type Codec private () = /// Configuration to be used by the underlying Newtonsoft.Json Serializer when encoding/decoding. Defaults to same as Settings.Create() [] ?settings, /// Enables one to fail encoder generation if union contains nullary cases. Defaults to false, i.e. permitting them - [] ?rejectNullaryCases) + [] ?rejectNullaryCases, + /// Enables unions to contain a Guid or most primitives. Defaults to true, i.e. preventing Guids and primitives + [] ?requireRecordFields) : FsCodec.IEventCodec<'Event, byte[], 'Context> = let down (context, union) = let c, m, t = down union let m', eventId, correlationId, causationId = mapCausation (context, m) c, m', eventId, correlationId, causationId, t - Codec.Create(up = up, down = down, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases) + Codec.Create(up = up, down = down, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases, ?requireRecordFields = requireRecordFields) /// Generate an IEventCodec using the supplied Newtonsoft.Json settings. /// Uses up and down and mapCausation functions to facilitate upconversion/downconversion and correlation/causationId mapping @@ -145,11 +163,13 @@ type Codec private () = /// Configuration to be used by the underlying Newtonsoft.Json Serializer when encoding/decoding. Defaults to same as Settings.Create() [] ?settings, /// Enables one to fail encoder generation if union contains nullary cases. Defaults to false, i.e. permitting them - [] ?rejectNullaryCases) + [] ?rejectNullaryCases, + /// Enables unions to contain a Guid or most primitives. Defaults to true, i.e. preventing Guids and primitives + [] ?requireRecordFields) : FsCodec.IEventCodec<'Event, byte[], obj> = let mapCausation (_context : obj, m : 'Meta option) = m, Guid.NewGuid(), null, null - Codec.Create(up = up, down = down, mapCausation = mapCausation, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases) + Codec.Create(up = up, down = down, mapCausation = mapCausation, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases, ?requireRecordFields = requireRecordFields) /// Generate an IEventCodec using the supplied Newtonsoft.Json settings. /// The Event Type Names are inferred based on either explicit DataMember(Name= Attributes, or (if unspecified) the Discriminated Union Case Name @@ -158,9 +178,11 @@ type Codec private () = ( // Configuration to be used by the underlying Newtonsoft.Json Serializer when encoding/decoding. Defaults to same as Settings.Create() [] ?settings, /// Enables one to fail encoder generation if union contains nullary cases. Defaults to false, i.e. permitting them - [] ?rejectNullaryCases) + [] ?rejectNullaryCases, + /// Enables unions to contain a Guid or most primitives. Defaults to true, i.e. preventing Guids and primitives + [] ?requireRecordFields) : FsCodec.IEventCodec<'Union, byte[], obj> = let up : FsCodec.ITimelineEvent<_> * 'Union -> 'Union = snd let down (event : 'Union) = event, None, None - Codec.Create(up = up, down = down, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases) + Codec.Create(up = up, down = down, ?settings = settings, ?rejectNullaryCases = rejectNullaryCases, ?requireRecordFields = requireRecordFields) diff --git a/src/FsCodec.NewtonsoftJson/FsCodec.NewtonsoftJson.fsproj b/src/FsCodec.NewtonsoftJson/FsCodec.NewtonsoftJson.fsproj index 8b9515c..f133184 100644 --- a/src/FsCodec.NewtonsoftJson/FsCodec.NewtonsoftJson.fsproj +++ b/src/FsCodec.NewtonsoftJson/FsCodec.NewtonsoftJson.fsproj @@ -29,7 +29,6 @@ - diff --git a/src/FsCodec.NewtonsoftJson/VerbatimUtf8Converter.fs b/src/FsCodec.NewtonsoftJson/VerbatimUtf8Converter.fs index fd7b1d3..bedaa8f 100755 --- a/src/FsCodec.NewtonsoftJson/VerbatimUtf8Converter.fs +++ b/src/FsCodec.NewtonsoftJson/VerbatimUtf8Converter.fs @@ -20,5 +20,7 @@ type VerbatimUtf8JsonConverter() = override __.ReadJson(reader : JsonReader, _ : Type, _ : obj, _ : JsonSerializer) = let token = JToken.Load reader - if token.Type = JTokenType.Null then null - else token |> string |> enc.GetBytes |> box + match token.Type with + | JTokenType.Null -> null + | JTokenType.Float -> reader.Value :?> double |> fun x -> x.ToString "r" |> enc.GetBytes |> box + | _ -> token |> string |> enc.GetBytes |> box diff --git a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj index fd22c06..c88c268 100644 --- a/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj +++ b/src/FsCodec.SystemTextJson/FsCodec.SystemTextJson.fsproj @@ -27,7 +27,6 @@ - diff --git a/src/FsCodec/FsCodec.fsproj b/src/FsCodec/FsCodec.fsproj index 2b9a4c8..8c7e82a 100644 --- a/src/FsCodec/FsCodec.fsproj +++ b/src/FsCodec/FsCodec.fsproj @@ -9,6 +9,7 @@ + @@ -21,6 +22,13 @@ + + + + + + <_Parameter1>FsCodec.NewtonsoftJson + \ No newline at end of file diff --git a/src/FsCodec/Internal.fs b/src/FsCodec/Internal.fs new file mode 100644 index 0000000..3b2957b --- /dev/null +++ b/src/FsCodec/Internal.fs @@ -0,0 +1,43 @@ +module internal Internal + +open TypeShape.Core + +let checkIfSupported<'Contract> requireRecordFields = + if not requireRecordFields then + let shape = + match shapeof<'Contract> with + | Shape.FSharpUnion (:? ShapeFSharpUnion<'Contract> as s) -> s + | _ -> + sprintf "Type '%O' is not an F# union" typeof<'Contract> + |> invalidArg "Union" + let isAllowed (scase : ShapeFSharpUnionCase<_>) = + match scase.Fields with + | [| field |] -> + match field.Member with + // non-primitives + | Shape.FSharpRecord _ + | Shape.Guid _ + + // primitives + | Shape.Bool _ + | Shape.Byte _ + | Shape.SByte _ + | Shape.Int16 _ + | Shape.Int32 _ + | Shape.Int64 _ + //| Shape.IntPtr _ // unsupported + | Shape.UInt16 _ + | Shape.UInt32 _ + | Shape.UInt64 _ + //| Shape.UIntPtr _ // unsupported + | Shape.Single _ + | Shape.Double _ + | Shape.Char _ -> true + | _ -> false + | [||] -> true // allows all nullary cases, but a subsequent check is done by UnionContractEncoder.Create with `allowNullaryCases` + | _ -> false + shape.UnionCases + |> Array.tryFind (not << isAllowed) + |> function + | None -> () + | Some x -> failwithf "The '%s' case has an unsupported type: '%s'" x.CaseInfo.Name x.Fields.[0].Member.Type.FullName diff --git a/tests/FsCodec.NewtonsoftJson.Tests/VerbatimUtf8ConverterTests.fs b/tests/FsCodec.NewtonsoftJson.Tests/VerbatimUtf8ConverterTests.fs index 78e7c35..289890b 100644 --- a/tests/FsCodec.NewtonsoftJson.Tests/VerbatimUtf8ConverterTests.fs +++ b/tests/FsCodec.NewtonsoftJson.Tests/VerbatimUtf8ConverterTests.fs @@ -24,8 +24,24 @@ type U = //| DT of DateTime // Have not analyzed but seems to be same issue as DTO | EDto of EmbeddedDateTimeOffset | ES of EmbeddedString - //| I of int // works but removed as no other useful top level values work + | Guid of Guid | N + + // primitives + | Boolean of bool + | Byte of byte + | SByte of sbyte + | Int16 of int16 + | UInt16 of uint16 + | Int32 of int32 + | UInt32 of uint32 + | Int64 of int64 + | UInt64 of uint64 + //| IntPtr of IntPtr // unsupported + //| UIntPtr of UIntPtr // unsupported + | Char of char + | Double of double + | Single of single interface TypeShape.UnionContract.IUnionContract type [] @@ -57,7 +73,7 @@ let mkBatch (encoded : FsCodec.IEventData) : Batch = module VerbatimUtf8Tests = // not a module or CI will fail for net461 - let eventCodec = Codec.Create() + let eventCodec = Codec.Create(requireRecordFields=false) let [] ``encodes correctly`` () = let input = Union.A { embed = "\"" } @@ -71,7 +87,7 @@ module VerbatimUtf8Tests = // not a module or CI will fail for net461 input =! decoded let defaultSettings = Settings.CreateDefault() - let defaultEventCodec = Codec.Create(defaultSettings) + let defaultEventCodec = Codec.Create(defaultSettings, requireRecordFields=false) let [] ``round-trips diverse bodies correctly`` (x: U) = let encoded = defaultEventCodec.Encode(None,x) @@ -80,7 +96,12 @@ module VerbatimUtf8Tests = // not a module or CI will fail for net461 let des = JsonConvert.DeserializeObject(ser, defaultSettings) let loaded = FsCodec.Core.TimelineEvent.Create(-1L, des.e.[0].c, des.e.[0].d) let decoded = defaultEventCodec.TryDecode loaded |> Option.get - x =! decoded + match x, decoded with + | U.Double x, U.Double d when Double.IsNaN x && Double.IsNaN d -> () + | U.Single x, U.Single d when Single.IsNaN x && Single.IsNaN d -> () + | U.Double x, U.Double d -> Assert.Equal( x, d, 10) + | U.Single x, U.Single d -> Assert.Equal(double x, double d, 10) + | _ -> x =! decoded // https://github.com/JamesNK/Newtonsoft.Json/issues/862 // doesnt apply to this case let [] ``Codec does not fall prey to Date-strings being mutilated`` () =