Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix incorrect JsonProvider numeric inference #1222

Closed
wants to merge 6 commits into from
Closed

Fix incorrect JsonProvider numeric inference #1222

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
63961a2
Add failing test cases
tachyus-ryan Oct 23, 2018
437cacd
Fix expected test values
tachyus-ryan Oct 23, 2018
48054ea
added complex test
tachyus-ryan Oct 23, 2018
1ba8d65
fix asserts
tachyus-ryan Oct 23, 2018
1f923ee
exponentials are preserved for floats
baronfel Jan 15, 2019
8c63dd2
Merge pull request #3 from baronfel/bugfix/json-numeric-inference
panesofglass Jan 15, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions docs/content/ja/tutorials/JsonAnonymizer.fsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ open System
open System.Globalization
open FSharp.Data

type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =
type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =

let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip [])
let valuesToSkip = Set.ofList (defaultArg valuesToSkip [])
Expand Down Expand Up @@ -56,31 +56,31 @@ type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =
if typ = typeof<Guid> then Guid.NewGuid().ToString()
elif typ = typeof<Runtime.StructuralTypes.Bit0> || typ = typeof<Runtime.StructuralTypes.Bit1> then s
elif typ = typeof<DateTime> then s
else
else
let prefix, s =
if s.StartsWith "http://" then "http://", s.Substring("http://".Length)
elif s.StartsWith "https://" then "https://", s.Substring("https://".Length)
else "", s
prefix + randomize s
|> JsonValue.String
| JsonValue.Number d ->
| JsonValue.Number d ->
let typ = Runtime.StructuralInference.inferPrimitiveType CultureInfo.InvariantCulture (d.ToString())
if typ = typeof<Runtime.StructuralTypes.Bit0> || typ = typeof<Runtime.StructuralTypes.Bit1>
then json
else d.ToString() |> randomize |> Decimal.Parse |> JsonValue.Number
| JsonValue.Float f ->
| JsonValue.Float (f, isExponential) ->
f.ToString()
|> randomize
|> Double.Parse
|> JsonValue.Float
|> randomize
|> Double.Parse
|> fun f -> JsonValue.Float(f, isExponential)
| JsonValue.Boolean _ | JsonValue.Null -> json
| JsonValue.Record props ->
props
| JsonValue.Record props ->
props
|> Array.map (fun (key, value) -> key, if propertiesToSkip.Contains key then value else anonymize value)
|> JsonValue.Record
| JsonValue.Array array ->
array
|> Array.map anonymize
| JsonValue.Array array ->
array
|> Array.map anonymize
|> JsonValue.Array

member __.Anonymize json = anonymize json
Expand All @@ -101,4 +101,4 @@ printfn "%O" anonymizedJson
型安全な方法でJSONデータにアクセスする機能を持った
F# 型プロバイダーについて説明しています。

*)
*)
26 changes: 13 additions & 13 deletions docs/content/ja/tutorials/JsonToXml.fsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ JSONの配列やレコードは無名です(ただしレコードには
[lang=js]
{ "version": "1.0",
"title": { "text": "Sample input" },
"items": [ { "value": "First" },
"items": [ { "value": "First" },
{ "value": "Second" } ] }

このように、 `item` 要素は自動的に `items` という複数形に変換されて、
Expand All @@ -80,7 +80,7 @@ JSONの配列やレコードは無名です(ただしレコードには
let rec fromXml (xml:XElement) =

// すべての属性に対してキー値ペアのコレクションを作成する
let attrs =
let attrs =
[ for attr in xml.Attributes() ->
(attr.Name.LocalName, JsonValue.String attr.Value) ]

Expand All @@ -94,13 +94,13 @@ let rec fromXml (xml:XElement) =
// 単一要素のグループを(再帰的に)レコードへと変換し、
// 複数要素のグループをcreateArrayでJSON配列に変換する
let children =
xml.Elements()
xml.Elements()
|> Seq.groupBy (fun x -> x.Name.LocalName)
|> Seq.map (fun (key, childs) ->
match Seq.toList childs with
| [child] -> key, fromXml child
| children -> key + "s", createArray children )

// 子要素および属性用に生成された要素を連結する
Array.append (Array.ofList attrs) (Array.ofSeq children)
|> JsonValue.Record
Expand Down Expand Up @@ -145,9 +145,9 @@ JSONからXMLへ変換する場合、同じようなミスマッチが起こり
/// (ただしトップレベルの値がオブジェクトまたは配列の場合のみ機能する)
let toXml(x:JsonValue) =
// XML属性やXML要素を作成するためのヘルパ関数
let attr name value =
let attr name value =
XAttribute(XName.Get name, value) :> XObject
let elem name (value:obj) =
let elem name (value:obj) =
XElement(XName.Get name, value) :> XObject

// 変換機能を実装している内部用再帰関数
Expand All @@ -156,25 +156,25 @@ let toXml(x:JsonValue) =
| JsonValue.Null -> null
| JsonValue.Boolean b -> b :> obj
| JsonValue.Number number -> number :> obj
| JsonValue.Float number -> number :> obj
| JsonValue.Float(number, _) -> number :> obj
| JsonValue.String s -> s :> obj

// JSONオブジェクトは(プリミティブであれば)XML属性か、
// あるいは子要素になる
// attributes (for primitives) or child elements
| JsonValue.Record properties ->
properties
| JsonValue.Record properties ->
properties
|> Array.map (fun (key, value) ->
match value with
| JsonValue.String s -> attr key s
| JsonValue.Boolean b -> attr key b
| JsonValue.Number n -> attr key n
| JsonValue.Float n -> attr key n
| JsonValue.Float(n, _) -> attr key n
| _ -> elem key (toXml value)) :> obj

// JSON配列は <item> 要素のシーケンスになる
| JsonValue.Array elements ->
elements |> Array.map (fun item ->
| JsonValue.Array elements ->
elements |> Array.map (fun item ->
elem "item" (toXml item)) :> obj

// 変換を実行して、結果をオブジェクトのシーケンスにキャストする
Expand All @@ -195,4 +195,4 @@ let toXml(x:JsonValue) =
型安全な方法でXMLデータにアクセスする機能を持った
F# 型プロバイダーについて説明しています。

*)
*)
50 changes: 25 additions & 25 deletions docs/content/tutorials/JsonAnonymizer.fsx
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
(**
# F# Data: Anonymizing JSON
# F# Data: Anonymizing JSON

This tutorial shows how to implement an anonymizer for a JSON document (represented using
the `JsonValue` type discussed in [JSON parser article](JsonValue.html))
This functionality is not directly available in the F# Data library, but it can
be very easily implemented by recursively walking over the JSON document.

If you want to use the JSON anonymizer in your code, you can copy the
If you want to use the JSON anonymizer in your code, you can copy the
[source from GitHub][jsonanonymizer] and just include it in your project. If you use these
functions often and would like to see them in the F# Data library, please submit
a [feature request][issues].
Expand All @@ -23,7 +23,7 @@ open System
open System.Globalization
open FSharp.Data

type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =
type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =

let propertiesToSkip = Set.ofList (defaultArg propertiesToSkip [])
let valuesToSkip = Set.ofList (defaultArg valuesToSkip [])
Expand All @@ -49,45 +49,45 @@ type JsonAnonymizer(?propertiesToSkip, ?valuesToSkip) =
match json with
| JsonValue.String s when valuesToSkip.Contains s -> json
| JsonValue.String s ->
let typ =
Runtime.StructuralInference.inferPrimitiveType
let typ =
Runtime.StructuralInference.inferPrimitiveType
CultureInfo.InvariantCulture s

( if typ = typeof<Guid> then Guid.NewGuid().ToString()
elif typ = typeof<Runtime.StructuralTypes.Bit0> ||
elif typ = typeof<Runtime.StructuralTypes.Bit0> ||
typ = typeof<Runtime.StructuralTypes.Bit1> then s
elif typ = typeof<DateTime> then s
else
else
let prefix, s =
if s.StartsWith "http://" then
if s.StartsWith "http://" then
"http://", s.Substring("http://".Length)
elif s.StartsWith "https://" then
elif s.StartsWith "https://" then
"https://", s.Substring("https://".Length)
else "", s
prefix + randomize s )
|> JsonValue.String
| JsonValue.Number d ->
let typ =
Runtime.StructuralInference.inferPrimitiveType
| JsonValue.Number d ->
let typ =
Runtime.StructuralInference.inferPrimitiveType
CultureInfo.InvariantCulture (d.ToString())
if typ = typeof<Runtime.StructuralTypes.Bit0> ||
if typ = typeof<Runtime.StructuralTypes.Bit0> ||
typ = typeof<Runtime.StructuralTypes.Bit1> then json
else d.ToString() |> randomize |> Decimal.Parse |> JsonValue.Number
| JsonValue.Float f ->
| JsonValue.Float (f, isExponential) ->
f.ToString()
|> randomize
|> Double.Parse
|> JsonValue.Float
|> randomize
|> Double.Parse
|> fun f -> JsonValue.Float(f, isExponential)
| JsonValue.Boolean _ | JsonValue.Null -> json
| JsonValue.Record props ->
props
|> Array.map (fun (key, value) -> key,
if propertiesToSkip.Contains key then value
| JsonValue.Record props ->
props
|> Array.map (fun (key, value) -> key,
if propertiesToSkip.Contains key then value
else anonymize value)
|> JsonValue.Record
| JsonValue.Array array ->
array
|> Array.map anonymize
| JsonValue.Array array ->
array
|> Array.map anonymize
|> JsonValue.Array

member __.Anonymize json = anonymize json
Expand All @@ -107,4 +107,4 @@ printfn "%O" anonymizedJson
* [F# Data: JSON Type Provider](../library/JsonProvider.html) - discusses F# type provider
that provides type-safe access to JSON data.

*)
*)
48 changes: 24 additions & 24 deletions docs/content/tutorials/JsonToXml.fsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ XML document (represented as `XElement`) and the other way round.
This functionality is not directly available in the F# Data library, but it can
be very easily implemented by recursively walking over the JSON (or XML) document.

If you want to use the JSON to/from XML conversion in your code, you can copy the
If you want to use the JSON to/from XML conversion in your code, you can copy the
[source from GitHub][jsontoxml] and just include it in your project. If you use these
functions often and would like to see them in the F# Data library, please submit
a [feature request][issues].
Expand All @@ -31,7 +31,7 @@ open FSharp.Data

In this script, we create a conversion that returns an easy to process value, but the
conversion is not reversible (e.g. converting JSON to XML and then back to JSON will
produce a different value).
produce a different value).

Converting XML to JSON
----------------------
Expand All @@ -55,24 +55,24 @@ If an element appears multiple times, it is turned into an array:
[lang=js]
{ "version": "1.0",
"title": { "text": "Sample input" },
"items": [ { "value": "First" },
"items": [ { "value": "First" },
{ "value": "Second" } ] }

As you can see, the `item` element has been automatically pluralized to `items` and the
array contains two record values that consist of the `value` attribute.

The conversion function is a recursive function that takes a `XElement` and produces
The conversion function is a recursive function that takes a `XElement` and produces
`JsonValue`. It builds JSON records (using `JsonValue.Record`) and arrays (using
`JsonValue.Array`). All attribute values are turned into `JsonValue.String` - the
sample does not imlement more sophisticated conversion that would turn numeric
`JsonValue.Array`). All attribute values are turned into `JsonValue.String` - the
sample does not imlement more sophisticated conversion that would turn numeric
attributes to a corresponding JSON type:
*)

/// Creates a JSON representation of a XML element
let rec fromXml (xml:XElement) =

// Create a collection of key/value pairs for all attributes
let attrs =
// Create a collection of key/value pairs for all attributes
let attrs =
[ for attr in xml.Attributes() ->
(attr.Name.LocalName, JsonValue.String attr.Value) ]

Expand All @@ -86,13 +86,13 @@ let rec fromXml (xml:XElement) =
// element groups into a record (recursively) and all multi-
// element groups into a JSON array using createArray
let children =
xml.Elements()
xml.Elements()
|> Seq.groupBy (fun x -> x.Name.LocalName)
|> Seq.map (fun (key, childs) ->
match Seq.toList childs with
| [child] -> key, fromXml child
| children -> key + "s", createArray children )

// Concatenate elements produced for child elements & attributes
Array.append (Array.ofList attrs) (Array.ofSeq children)
|> JsonValue.Record
Expand All @@ -111,7 +111,7 @@ When converting JSON value to XML, we fact the same mismatch. Consider the follo

The top-level record does not have a name, so our conversion produces a list of `XObject`
values that can be wrapped into an `XElement` by the user (who has to specify the root
name). Record fields that are a primitive value are turned into attributes, while
name). Record fields that are a primitive value are turned into attributes, while
complex values (array or record) become objects:

[lang=xml]
Expand All @@ -123,21 +123,21 @@ complex values (array or record) become objects:
<paging current="1" />
</root>

The conversion function is, again, implemented as a recursive function. This time, we use
The conversion function is, again, implemented as a recursive function. This time, we use
pattern matching to distinguish between the different possible cases of `JsonValue`.
The cases representing a primitive value simply return the value as `obj`, while array
and record construct nested element(s) or attribute:

*)

/// Creates an XML representation of a JSON value (works
/// Creates an XML representation of a JSON value (works
/// only when the top-level value is an object or an array)
let toXml(x:JsonValue) =
// Helper functions for constructing XML
// Helper functions for constructing XML
// attributes and XML elements
let attr name value =
let attr name value =
XAttribute(XName.Get name, value) :> XObject
let elem name (value:obj) =
let elem name (value:obj) =
XElement(XName.Get name, value) :> XObject

// Inner recursive function that implements the conversion
Expand All @@ -146,25 +146,25 @@ let toXml(x:JsonValue) =
| JsonValue.Null -> null
| JsonValue.Boolean b -> b :> obj
| JsonValue.Number number -> number :> obj
| JsonValue.Float number -> number :> obj
| JsonValue.Float (number,_) -> number :> obj
| JsonValue.String s -> s :> obj

// JSON object becomes a collection of XML
// attributes (for primitives) or child elements
| JsonValue.Record properties ->
properties
| JsonValue.Record properties ->
properties
|> Array.map (fun (key, value) ->
match value with
| JsonValue.String s -> attr key s
| JsonValue.Boolean b -> attr key b
| JsonValue.Number n -> attr key n
| JsonValue.Float n -> attr key n
| JsonValue.Float (n, _) -> attr key n
| _ -> elem key (toXml value)) :> obj

// JSON array is turned into a
// JSON array is turned into a
// sequence of <item> elements
| JsonValue.Array elements ->
elements |> Array.map (fun item ->
| JsonValue.Array elements ->
elements |> Array.map (fun item ->
elem "item" (toXml item)) :> obj

// Perform the conversion and cast the result to sequence
Expand All @@ -183,4 +183,4 @@ let toXml(x:JsonValue) =
* [F# Data: XML Type Provider](../library/XmlProvider.html) - discusses the F# type provider
that provides type-safe access to XML data.

*)
*)
Loading